collect object operator!

go.mod

@@ -4,6 +4,7 @@ require (
 	github.com/elliotchance/orderedmap v1.3.0 // indirect
 	github.com/fatih/color v1.9.0
 	github.com/goccy/go-yaml v1.8.1
+	github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a
 	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.7 // indirect
 	github.com/pkg/errors v0.9.1

go.sum

@@ -49,6 +49,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a h1:zPPuIq2jAWWPTrGt70eK/BSch+gFAGrNzecsoENgu2o=
+github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=

pkg/yqlib/treeops/candidate_node.go

@@ -5,6 +5,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/jinzhu/copier"
 	"gopkg.in/yaml.v3"
 )
 
@@ -18,6 +19,12 @@ func (n *CandidateNode) GetKey() string {
 	return fmt.Sprintf("%v - %v - %v", n.Document, n.Path, n.Node.Value)
 }
 
+func (n *CandidateNode) Copy() *CandidateNode {
+	clone := &CandidateNode{}
+	copier.Copy(clone, n)
+	return clone
+}
+
 // updates this candidate from the given candidate node
 func (n *CandidateNode) UpdateFrom(other *CandidateNode) {
 	n.UpdateAttributesFrom(other)
@@ -34,7 +41,10 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode) {
 	}
 	n.Node.Kind = other.Node.Kind
 	n.Node.Tag = other.Node.Tag
-	n.Node.Style = other.Node.Style
+	// not sure if this ever should happen here...
+	// if other.Node.Style != 0 {
+	// 	n.Node.Style = other.Node.Style
+	// }
 	n.Node.FootComment = other.Node.FootComment
 	n.Node.HeadComment = other.Node.HeadComment
 	n.Node.LineComment = other.Node.LineComment

pkg/yqlib/treeops/operation_collection_object_test.go

@@ -1,2 +1,39 @@
 package treeops
 
+import (
+	"testing"
+)
+
+var collectObjectOperatorScenarios = []expressionScenario{
+	{
+		document:   `{name: Mike, age: 32}`,
+		expression: `{.name: .age}`,
+		expected: []string{
+			"D0, P[0], (!!map)::Mike: 32\n",
+		},
+	},
+	{
+		document:   `{name: Mike, pets: [cat, dog]}`,
+		expression: `{.name: .pets[]}`,
+		expected: []string{
+			"D0, P[0], (!!map)::Mike: cat\n",
+			"D0, P[1], (!!map)::Mike: dog\n",
+		},
+	},
+	{
+		document:   `{name: Mike, pets: [cat, dog], food: [hotdog, burger]}`,
+		expression: `{.name: .pets[], "f":.food[]}`,
+		expected: []string{
+			"D0, P[], (!!map)::Mike: cat\nf: hotdog\n",
+			"D0, P[], (!!map)::Mike: cat\nf: burger\n",
+			"D0, P[], (!!map)::Mike: dog\nf: hotdog\n",
+			"D0, P[], (!!map)::Mike: dog\nf: burger\n",
+		},
+	},
+}
+
+func TestCollectObjectOperatorScenarios(t *testing.T) {
+	for _, tt := range collectObjectOperatorScenarios {
+		testScenario(t, &tt)
+	}
+}

pkg/yqlib/treeops/operator_collect_object.go

@@ -1,8 +1,57 @@
 package treeops
 
-import "container/list"
+import (
+	"container/list"
+)
+
+/*
+[Mike: cat, Bob: dog]
+[Thing: rabbit, peter: sam]
+
+==> cross multiply
+
+{Mike: cat, Thing: rabbit}
+{Mike: cat, peter: sam}
+...
+*/
 
 func CollectObjectOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
 	log.Debugf("-- collectObjectOperation")
-	return nil, nil
+	return collect(d, list.New(), matchMap)
+
+}
+
+func collect(d *dataTreeNavigator, aggregate *list.List, remainingMatches *list.List) (*list.List, error) {
+	if remainingMatches.Len() == 0 {
+		return aggregate, nil
+	}
+
+	candidate := remainingMatches.Remove(remainingMatches.Front()).(*CandidateNode)
+	splatted, err := Splat(d, nodeToMap(candidate))
+	if err != nil {
+		return nil, err
+	}
+
+	if aggregate.Len() == 0 {
+		return collect(d, splatted, remainingMatches)
+	}
+
+	newAgg := list.New()
+
+	for el := aggregate.Front(); el != nil; el = el.Next() {
+		aggCandidate := el.Value.(*CandidateNode)
+		for splatEl := splatted.Front(); splatEl != nil; splatEl = splatEl.Next() {
+			splatCandidate := splatEl.Value.(*CandidateNode)
+			newCandidate := aggCandidate.Copy()
+			newCandidate.Path = nil
+
+			newCandidate, err := multiply(d, newCandidate, splatCandidate)
+			if err != nil {
+				return nil, err
+			}
+			newAgg.PushBack(newCandidate)
+		}
+	}
+	return collect(d, newAgg, remainingMatches)
+
 }

pkg/yqlib/treeops/operator_create_map_test.go

@@ -19,6 +19,14 @@ var createMapOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::- Mike: cat\n- Mike: dog\n",
 		},
 	},
+	{
+		document:   `{name: Mike, pets: [cat, dog], food: [hotdog, burger]}`,
+		expression: `.name: .pets[], "f":.food[]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- Mike: cat\n- Mike: dog\n",
+			"D0, P[], (!!seq)::- f: hotdog\n- f: burger\n",
+		},
+	},
 }
 
 func TestCreateMapOperatorScenarios(t *testing.T) {

pkg/yqlib/treeops/operator_multiply_test.go

@@ -6,67 +6,65 @@ import (
 
 var multiplyOperatorScenarios = []expressionScenario{
 	{
-		// 	document:   `{a: {also: [1]}, b: {also: me}}`,
+		document:   `{a: {also: [1]}, b: {also: me}}`,
-		// 	expression: `.a * .b`,
+		expression: `. * {"a" : .b}`,
-		// 	expected: []string{
+		expected: []string{
-		// 		"D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\n",
+			"D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\n",
-		// 	},
+		},
-		// }, {
+	}, {
-		// 	document:   `{a: {also: me}, b: {also: [1]}}`,
+		document:   `{a: {also: me}, b: {also: [1]}}`,
-		// 	expression: `.a * .b`,
+		expression: `. * {"a":.b}`,
-		// 	expected: []string{
+		expected: []string{
-		// 		"D0, P[], (!!map)::{a: {also: [1]}, b: {also: [1]}}\n",
+			"D0, P[], (!!map)::{a: {also: [1]}, b: {also: [1]}}\n",
-		// 	},
+		},
-		// }, {
+	}, {
-		// 	document:   `{a: {also: me}, b: {also: {g: wizz}}}`,
+		document:   `{a: {also: me}, b: {also: {g: wizz}}}`,
-		// 	expression: `.a * .b`,
+		expression: `. * {"a":.b}`,
-		// 	expected: []string{
+		expected: []string{
-		// 		"D0, P[], (!!map)::{a: {also: {g: wizz}}, b: {also: {g: wizz}}}\n",
+			"D0, P[], (!!map)::{a: {also: {g: wizz}}, b: {also: {g: wizz}}}\n",
-		// 	},
+		},
-		// }, {
+	}, {
-		// 	document:   `{a: {also: {g: wizz}}, b: {also: me}}`,
+		document:   `{a: {also: {g: wizz}}, b: {also: me}}`,
-		// 	expression: `.a * .b`,
+		expression: `. * {"a":.b}`,
-		// 	expected: []string{
+		expected: []string{
-		// 		"D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\n",
+			"D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\n",
-		// 	},
+		},
-		// }, {
+	}, {
-		// 	document:   `{a: {also: {g: wizz}}, b: {also: [1]}}`,
+		document:   `{a: {also: {g: wizz}}, b: {also: [1]}}`,
-		// 	expression: `.a * .b`,
+		expression: `. * {"a":.b}`,
-		// 	expected: []string{
+		expected: []string{
-		// 		"D0, P[], (!!map)::{a: {also: [1]}, b: {also: [1]}}\n",
+			"D0, P[], (!!map)::{a: {also: [1]}, b: {also: [1]}}\n",
-		// 	},
+		},
-		// }, {
+	}, {
-		// 	document:   `{a: {also: [1]}, b: {also: {g: wizz}}}`,
+		document:   `{a: {also: [1]}, b: {also: {g: wizz}}}`,
-		// 	expression: `.a * .b`,
+		expression: `. * {"a":.b}`,
-		// 	expected: []string{
+		expected: []string{
-		// 		"D0, P[], (!!map)::{a: {also: {g: wizz}}, b: {also: {g: wizz}}}\n",
+			"D0, P[], (!!map)::{a: {also: {g: wizz}}, b: {also: {g: wizz}}}\n",
-		// 	},
+		},
-		// }, {
+	}, {
-		// 	document:   `{a: {things: great}, b: {also: me}}`,
+		document:   `{a: {things: great}, b: {also: me}}`,
-		// 	expression: `.a * .b`,
+		expression: `. * {"a":.b}`,
-		// 	expected: []string{
+		expected: []string{
-		// 		"D0, P[], (!!map)::{a: {things: great, also: me}, b: {also: me}}\n",
+			"D0, P[], (!!map)::{a: {things: great, also: me}, b: {also: me}}\n",
-		// 	},
+		},
-		// }, {
+	}, {
-		// 		document: `a: {things: great}
+		document: `a: {things: great}
-		// b:
+b:
-		//   also: "me"
+  also: "me"
-		// `,
+`,
-		// 		expression: `(.a * .b)`,
+		expression: `. * {"a":.b}`,
-		// 		expected: []string{
+		expected: []string{
-		// 			`D0, P[], (!!map)::a:
+			`D0, P[], (!!map)::a: {things: great, also: me}
-		//     things: great
+b:
-		//     also: "me"
+    also: "me"
-		// b:
+`,
-		//     also: "me"
+		},
-		// `,
+	}, {
-		// },
+		document:   `{a: [1,2,3], b: [3,4,5]}`,
-		// }, {
+		expression: `. * {"a":.b}`,
-		// 	document:   `{a: [1,2,3], b: [3,4,5]}`,
+		expected: []string{
-		// 	expression: `.a * .b`,
+			"D0, P[], (!!map)::{a: [3, 4, 5], b: [3, 4, 5]}\n",
-		// 	expected: []string{
+		},
-		// 		"D0, P[], (!!map)::{a: [3, 4, 5], b: [3, 4, 5]}\n",
-		// 	},
 	},
 }
 

pkg/yqlib/treeops/operator_recursive_descent.go

@@ -16,14 +16,11 @@ func RecursiveDescentOperator(d *dataTreeNavigator, matchMap *list.List, pathNod
 }
 
 func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.List) error {
-	splatOperation := &Operation{OperationType: TraversePath, Value: "[]"}
-	splatTreeNode := &PathTreeNode{Operation: splatOperation}
-
 	for el := matchMap.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 		results.PushBack(candidate)
 
-		children, err := TraversePathOperator(d, nodeToMap(candidate), splatTreeNode)
+		children, err := Splat(d, nodeToMap(candidate))
 
 		if err != nil {
 			return err

pkg/yqlib/treeops/operator_traverse_path.go

@@ -8,6 +8,12 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
+func Splat(d *dataTreeNavigator, matches *list.List) (*list.List, error) {
+	splatOperation := &Operation{OperationType: TraversePath, Value: "[]"}
+	splatTreeNode := &PathTreeNode{Operation: splatOperation}
+	return TraversePathOperator(d, matches, splatTreeNode)
+}
+
 func TraversePathOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
 	log.Debugf("-- Traversing")
 	var matchingNodeMap = list.New()

pkg/yqlib/treeops/path_parse_test.go

@@ -59,6 +59,11 @@ var pathTests = []struct {
 		append(make([]interface{}, 0), "[", "true (bool)", "]"),
 		append(make([]interface{}, 0), "true (bool)", "COLLECT", "PIPE"),
 	},
+	{
+		`[true, false]`,
+		append(make([]interface{}, 0), "[", "true (bool)", "UNION", "false (bool)", "]"),
+		append(make([]interface{}, 0), "true (bool)", "false (bool)", "UNION", "COLLECT", "PIPE"),
+	},
 	{
 		`"mike": .a`,
 		append(make([]interface{}, 0), "mike (string)", "CREATE_MAP", "a"),
@@ -69,6 +74,21 @@ var pathTests = []struct {
 		append(make([]interface{}, 0), "a", "CREATE_MAP", "mike (string)"),
 		append(make([]interface{}, 0), "a", "mike (string)", "CREATE_MAP"),
 	},
+	{
+		`{"mike": .a}`,
+		append(make([]interface{}, 0), "{", "mike (string)", "CREATE_MAP", "a", "}"),
+		append(make([]interface{}, 0), "mike (string)", "a", "CREATE_MAP", "COLLECT_OBJECT", "PIPE"),
+	},
+	{
+		`{.a: "mike"}`,
+		append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "mike (string)", "}"),
+		append(make([]interface{}, 0), "a", "mike (string)", "CREATE_MAP", "COLLECT_OBJECT", "PIPE"),
+	},
+	{
+		`{.a: .c, .b[]: .f.g[]}`,
+		append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "c", "UNION", "b", "PIPE", "[]", "CREATE_MAP", "f", "PIPE", "g", "PIPE", "[]", "}"),
+		append(make([]interface{}, 0), "a", "c", "CREATE_MAP", "b", "[]", "PIPE", "f", "g", "PIPE", "[]", "PIPE", "CREATE_MAP", "UNION", "COLLECT_OBJECT", "PIPE"),
+	},
 
 	// {".animals | .==cat", append(make([]interface{}, 0), "animals", "TRAVERSE", "SELF", "EQUALS", "cat")},
 	// {".animals | (. == cat)", append(make([]interface{}, 0), "animals", "TRAVERSE", "(", "SELF", "EQUALS", "cat", ")")},

pkg/yqlib/treeops/path_postfix.go

@@ -32,10 +32,16 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*Operation, er
 	for _, token := range tokens {
 		log.Debugf("postfix processing token %v, %v", token.toString(), token.Operation)
 		switch token.TokenType {
-		case OpenBracket, OpenCollect:
+		case OpenBracket, OpenCollect, OpenCollectObject:
 			opStack = append(opStack, token)
-		case CloseCollect:
+		case CloseCollect, CloseCollectObject:
-			for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != OpenCollect {
+			var opener TokenType = OpenCollect
+			var collectOperator *OperationType = Collect
+			if token.TokenType == CloseCollectObject {
+				opener = OpenCollectObject
+				collectOperator = CollectObject
+			}
+			for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != opener {
 				opStack, result = popOpToResult(opStack, result)
 			}
 			if len(opStack) == 0 {
@@ -45,7 +51,7 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*Operation, er
 			opStack = opStack[0 : len(opStack)-1]
 			//and append a collect to the opStack
 			opStack = append(opStack, &Token{TokenType: OperationToken, Operation: &Operation{OperationType: Pipe}})
-			opStack = append(opStack, &Token{TokenType: OperationToken, Operation: &Operation{OperationType: Collect}})
+			opStack = append(opStack, &Token{TokenType: OperationToken, Operation: &Operation{OperationType: collectOperator}})
 		case CloseBracket:
 			for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != OpenBracket {
 				opStack, result = popOpToResult(opStack, result)