suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2024-11-12 05:38:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

collect object operator!

Browse Source
This commit is contained in:
Mike Farah 2020-10-21 13:54:51 +11:00
parent 65e6e492cd
commit badd476730
11 changed files with 206 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@ -4,6 +4,7 @@ require (
github.com/elliotchance/orderedmap v1.3.0 // indirect
github.com/fatih/color v1.9.0
github.com/goccy/go-yaml v1.8.1
github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/mattn/go-colorable v0.1.7 // indirect
github.com/pkg/errors v0.9.1

2
go.sum
View File

@ -49,6 +49,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a h1:zPPuIq2jAWWPTrGt70eK/BSch+gFAGrNzecsoENgu2o=
github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=

12
pkg/yqlib/treeops/candidate_node.go
View File

@ -5,6 +5,7 @@ import (
"strconv"
"strings"
"github.com/jinzhu/copier"
"gopkg.in/yaml.v3"
)
@ -18,6 +19,12 @@ func (n *CandidateNode) GetKey() string {
return fmt.Sprintf("%v - %v - %v", n.Document, n.Path, n.Node.Value)
}
func (n *CandidateNode) Copy() *CandidateNode {
clone := &CandidateNode{}
copier.Copy(clone, n)
return clone
}
// updates this candidate from the given candidate node
func (n *CandidateNode) UpdateFrom(other *CandidateNode) {
n.UpdateAttributesFrom(other)
@ -34,7 +41,10 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode) {
}
n.Node.Kind = other.Node.Kind
n.Node.Tag = other.Node.Tag
n.Node.Style = other.Node.Style
// not sure if this ever should happen here...
// if other.Node.Style != 0 {
// n.Node.Style = other.Node.Style
// }
n.Node.FootComment = other.Node.FootComment
n.Node.HeadComment = other.Node.HeadComment
n.Node.LineComment = other.Node.LineComment

37
pkg/yqlib/treeops/operation_collection_object_test.go
View File

@ -1,2 +1,39 @@
package treeops
import (
"testing"
)
var collectObjectOperatorScenarios = []expressionScenario{
{
document: `{name: Mike, age: 32}`,
expression: `{.name: .age}`,
expected: []string{
"D0, P[0], (!!map)::Mike: 32\n",
},
},
{
document: `{name: Mike, pets: [cat, dog]}`,
expression: `{.name: .pets[]}`,
expected: []string{
"D0, P[0], (!!map)::Mike: cat\n",
"D0, P[1], (!!map)::Mike: dog\n",
},
},
{
document: `{name: Mike, pets: [cat, dog], food: [hotdog, burger]}`,
expression: `{.name: .pets[], "f":.food[]}`,
expected: []string{
"D0, P[], (!!map)::Mike: cat\nf: hotdog\n",
"D0, P[], (!!map)::Mike: cat\nf: burger\n",
"D0, P[], (!!map)::Mike: dog\nf: hotdog\n",
"D0, P[], (!!map)::Mike: dog\nf: burger\n",
},
},
}
func TestCollectObjectOperatorScenarios(t *testing.T) {
for _, tt := range collectObjectOperatorScenarios {
testScenario(t, &tt)
}
}

53
pkg/yqlib/treeops/operator_collect_object.go
View File

@ -1,8 +1,57 @@
package treeops
import "container/list"
import (
"container/list"
)
/*
[Mike: cat, Bob: dog]
[Thing: rabbit, peter: sam]
==> cross multiply
{Mike: cat, Thing: rabbit}
{Mike: cat, peter: sam}
...
*/
func CollectObjectOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
log.Debugf("-- collectObjectOperation")
return nil, nil
return collect(d, list.New(), matchMap)
}
func collect(d *dataTreeNavigator, aggregate *list.List, remainingMatches *list.List) (*list.List, error) {
if remainingMatches.Len() == 0 {
return aggregate, nil
}
candidate := remainingMatches.Remove(remainingMatches.Front()).(*CandidateNode)
splatted, err := Splat(d, nodeToMap(candidate))
if err != nil {
return nil, err
}
if aggregate.Len() == 0 {
return collect(d, splatted, remainingMatches)
}
newAgg := list.New()
for el := aggregate.Front(); el != nil; el = el.Next() {
aggCandidate := el.Value.(*CandidateNode)
for splatEl := splatted.Front(); splatEl != nil; splatEl = splatEl.Next() {
splatCandidate := splatEl.Value.(*CandidateNode)
newCandidate := aggCandidate.Copy()
newCandidate.Path = nil
newCandidate, err := multiply(d, newCandidate, splatCandidate)
if err != nil {
return nil, err
}
newAgg.PushBack(newCandidate)
}
}
return collect(d, newAgg, remainingMatches)
}

8
pkg/yqlib/treeops/operator_create_map_test.go
View File

@ -19,6 +19,14 @@ var createMapOperatorScenarios = []expressionScenario{
"D0, P[], (!!seq)::- Mike: cat\n- Mike: dog\n",
},
},
{
document: `{name: Mike, pets: [cat, dog], food: [hotdog, burger]}`,
expression: `.name: .pets[], "f":.food[]`,
expected: []string{
"D0, P[], (!!seq)::- Mike: cat\n- Mike: dog\n",
"D0, P[], (!!seq)::- f: hotdog\n- f: burger\n",
},
},
}
func TestCreateMapOperatorScenarios(t *testing.T) {

120
pkg/yqlib/treeops/operator_multiply_test.go
View File

@ -6,67 +6,65 @@ import (
var multiplyOperatorScenarios = []expressionScenario{
{
// document: `{a: {also: [1]}, b: {also: me}}`,
// expression: `.a * .b`,
// expected: []string{
// "D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\n",
// },
// }, {
// document: `{a: {also: me}, b: {also: [1]}}`,
// expression: `.a * .b`,
// expected: []string{
// "D0, P[], (!!map)::{a: {also: [1]}, b: {also: [1]}}\n",
// },
// }, {
// document: `{a: {also: me}, b: {also: {g: wizz}}}`,
// expression: `.a * .b`,
// expected: []string{
// "D0, P[], (!!map)::{a: {also: {g: wizz}}, b: {also: {g: wizz}}}\n",
// },
// }, {
// document: `{a: {also: {g: wizz}}, b: {also: me}}`,
// expression: `.a * .b`,
// expected: []string{
// "D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\n",
// },
// }, {
// document: `{a: {also: {g: wizz}}, b: {also: [1]}}`,
// expression: `.a * .b`,
// expected: []string{
// "D0, P[], (!!map)::{a: {also: [1]}, b: {also: [1]}}\n",
// },
// }, {
// document: `{a: {also: [1]}, b: {also: {g: wizz}}}`,
// expression: `.a * .b`,
// expected: []string{
// "D0, P[], (!!map)::{a: {also: {g: wizz}}, b: {also: {g: wizz}}}\n",
// },
// }, {
// document: `{a: {things: great}, b: {also: me}}`,
// expression: `.a * .b`,
// expected: []string{
// "D0, P[], (!!map)::{a: {things: great, also: me}, b: {also: me}}\n",
// },
// }, {
// document: `a: {things: great}
// b:
// also: "me"
// `,
// expression: `(.a * .b)`,
// expected: []string{
// `D0, P[], (!!map)::a:
// things: great
// also: "me"
// b:
// also: "me"
// `,
// },
// }, {
// document: `{a: [1,2,3], b: [3,4,5]}`,
// expression: `.a * .b`,
// expected: []string{
// "D0, P[], (!!map)::{a: [3, 4, 5], b: [3, 4, 5]}\n",
// },
document: `{a: {also: [1]}, b: {also: me}}`,
expression: `. * {"a" : .b}`,
expected: []string{
"D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\n",
},
}, {
document: `{a: {also: me}, b: {also: [1]}}`,
expression: `. * {"a":.b}`,
expected: []string{
"D0, P[], (!!map)::{a: {also: [1]}, b: {also: [1]}}\n",
},
}, {
document: `{a: {also: me}, b: {also: {g: wizz}}}`,
expression: `. * {"a":.b}`,
expected: []string{
"D0, P[], (!!map)::{a: {also: {g: wizz}}, b: {also: {g: wizz}}}\n",
},
}, {
document: `{a: {also: {g: wizz}}, b: {also: me}}`,
expression: `. * {"a":.b}`,
expected: []string{
"D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\n",
},
}, {
document: `{a: {also: {g: wizz}}, b: {also: [1]}}`,
expression: `. * {"a":.b}`,
expected: []string{
"D0, P[], (!!map)::{a: {also: [1]}, b: {also: [1]}}\n",
},
}, {
document: `{a: {also: [1]}, b: {also: {g: wizz}}}`,
expression: `. * {"a":.b}`,
expected: []string{
"D0, P[], (!!map)::{a: {also: {g: wizz}}, b: {also: {g: wizz}}}\n",
},
}, {
document: `{a: {things: great}, b: {also: me}}`,
expression: `. * {"a":.b}`,
expected: []string{
"D0, P[], (!!map)::{a: {things: great, also: me}, b: {also: me}}\n",
},
}, {
document: `a: {things: great}
b:
also: "me"
`,
expression: `. * {"a":.b}`,
expected: []string{
`D0, P[], (!!map)::a: {things: great, also: me}
b:
also: "me"
`,
},
}, {
document: `{a: [1,2,3], b: [3,4,5]}`,
expression: `. * {"a":.b}`,
expected: []string{
"D0, P[], (!!map)::{a: [3, 4, 5], b: [3, 4, 5]}\n",
},
},
}

5
pkg/yqlib/treeops/operator_recursive_descent.go
View File

@ -16,14 +16,11 @@ func RecursiveDescentOperator(d *dataTreeNavigator, matchMap *list.List, pathNod
}
func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.List) error {
splatOperation := &Operation{OperationType: TraversePath, Value: "[]"}
splatTreeNode := &PathTreeNode{Operation: splatOperation}
for el := matchMap.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
results.PushBack(candidate)
children, err := TraversePathOperator(d, nodeToMap(candidate), splatTreeNode)
children, err := Splat(d, nodeToMap(candidate))
if err != nil {
return err

6
pkg/yqlib/treeops/operator_traverse_path.go
View File

@ -8,6 +8,12 @@ import (
"gopkg.in/yaml.v3"
)
func Splat(d *dataTreeNavigator, matches *list.List) (*list.List, error) {
splatOperation := &Operation{OperationType: TraversePath, Value: "[]"}
splatTreeNode := &PathTreeNode{Operation: splatOperation}
return TraversePathOperator(d, matches, splatTreeNode)
}
func TraversePathOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
log.Debugf("-- Traversing")
var matchingNodeMap = list.New()

20
pkg/yqlib/treeops/path_parse_test.go
View File

@ -59,6 +59,11 @@ var pathTests = []struct {
append(make([]interface{}, 0), "[", "true (bool)", "]"),
append(make([]interface{}, 0), "true (bool)", "COLLECT", "PIPE"),
},
{
`[true, false]`,
append(make([]interface{}, 0), "[", "true (bool)", "UNION", "false (bool)", "]"),
append(make([]interface{}, 0), "true (bool)", "false (bool)", "UNION", "COLLECT", "PIPE"),
},
{
`"mike": .a`,
append(make([]interface{}, 0), "mike (string)", "CREATE_MAP", "a"),
@ -69,6 +74,21 @@ var pathTests = []struct {
append(make([]interface{}, 0), "a", "CREATE_MAP", "mike (string)"),
append(make([]interface{}, 0), "a", "mike (string)", "CREATE_MAP"),
},
{
`{"mike": .a}`,
append(make([]interface{}, 0), "{", "mike (string)", "CREATE_MAP", "a", "}"),
append(make([]interface{}, 0), "mike (string)", "a", "CREATE_MAP", "COLLECT_OBJECT", "PIPE"),
},
{
`{.a: "mike"}`,
append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "mike (string)", "}"),
append(make([]interface{}, 0), "a", "mike (string)", "CREATE_MAP", "COLLECT_OBJECT", "PIPE"),
},
{
`{.a: .c, .b[]: .f.g[]}`,
append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "c", "UNION", "b", "PIPE", "[]", "CREATE_MAP", "f", "PIPE", "g", "PIPE", "[]", "}"),
append(make([]interface{}, 0), "a", "c", "CREATE_MAP", "b", "[]", "PIPE", "f", "g", "PIPE", "[]", "PIPE", "CREATE_MAP", "UNION", "COLLECT_OBJECT", "PIPE"),
},
// {".animals | .==cat", append(make([]interface{}, 0), "animals", "TRAVERSE", "SELF", "EQUALS", "cat")},
// {".animals | (. == cat)", append(make([]interface{}, 0), "animals", "TRAVERSE", "(", "SELF", "EQUALS", "cat", ")")},

14
pkg/yqlib/treeops/path_postfix.go
View File

@ -32,10 +32,16 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*Operation, er
for _, token := range tokens {
log.Debugf("postfix processing token %v, %v", token.toString(), token.Operation)
switch token.TokenType {
case OpenBracket, OpenCollect:
case OpenBracket, OpenCollect, OpenCollectObject:
opStack = append(opStack, token)
case CloseCollect:
for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != OpenCollect {
case CloseCollect, CloseCollectObject:
var opener TokenType = OpenCollect
var collectOperator *OperationType = Collect
if token.TokenType == CloseCollectObject {
opener = OpenCollectObject
collectOperator = CollectObject
}
for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != opener {
opStack, result = popOpToResult(opStack, result)
}
if len(opStack) == 0 {
@ -45,7 +51,7 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*Operation, er
opStack = opStack[0 : len(opStack)-1]
//and append a collect to the opStack
opStack = append(opStack, &Token{TokenType: OperationToken, Operation: &Operation{OperationType: Pipe}})
opStack = append(opStack, &Token{TokenType: OperationToken, Operation: &Operation{OperationType: Collect}})
opStack = append(opStack, &Token{TokenType: OperationToken, Operation: &Operation{OperationType: collectOperator}})
case CloseBracket:
for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != OpenBracket {
opStack, result = popOpToResult(opStack, result)