wip

pkg/yqlib/doc/Multiply.md

@@ -66,9 +66,9 @@ b:
 Given a sample.yml file of:
 ```yaml
 a: {things: great}
-b:
+	b:
-  also: "me"
+	  also: "me"
-
+	
 ```
 then
 ```bash
@@ -76,9 +76,6 @@ yq eval '. * {"a":.b}' sample.yml
 ```
 will output
 ```yaml
-a: {things: great, also: "me"}
-b:
-  also: "me"
 ```
 
 ## Merge arrays
@@ -109,6 +106,38 @@ b:
   - 5
 ```
 
+## Merge, appending arrays
+Given a sample.yml file of:
+```yaml
+a:
+  array:
+    - 1
+    - 2
+    - animal: dog
+  value: coconut
+b:
+  array:
+    - 3
+    - 4
+    - animal: cat
+  value: banana
+```
+then
+```bash
+yq eval '.a *+ .b' sample.yml
+```
+will output
+```yaml
+array:
+  - 1
+  - 2
+  - animal: dog
+  - 3
+  - 4
+  - animal: cat
+value: banana
+```
+
 ## Merge to prefix an element
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/operator_add.go

@@ -8,6 +8,10 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
+type AddPreferences struct {
+	InPlace bool
+}
+
 func toNodes(candidates *list.List) []*yaml.Node {
 
 	if candidates.Len() == 0 {
@@ -41,26 +45,35 @@ func AddOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathT
 		return nil, err
 	}
 
+	preferences := pathNode.Operation.Preferences
+	inPlace := preferences != nil && preferences.(*AddPreferences).InPlace
+
 	for el := lhs.Front(); el != nil; el = el.Next() {
 		lhsCandidate := el.Value.(*CandidateNode)
 		lhsNode := UnwrapDoc(lhsCandidate.Node)
 
-		var newBlank = &CandidateNode{
+		var target *CandidateNode
-			Path:     lhsCandidate.Path,
+
-			Document: lhsCandidate.Document,
+		if inPlace {
-			Filename: lhsCandidate.Filename,
+			target = lhsCandidate
-			Node:     &yaml.Node{},
+		} else {
+			target = &CandidateNode{
+				Path:     lhsCandidate.Path,
+				Document: lhsCandidate.Document,
+				Filename: lhsCandidate.Filename,
+				Node:     &yaml.Node{},
+			}
 		}
 
 		switch lhsNode.Kind {
 		case yaml.MappingNode:
 			return nil, fmt.Errorf("Maps not yet supported for addition")
 		case yaml.SequenceNode:
-			newBlank.Node.Kind = yaml.SequenceNode
+			target.Node.Kind = yaml.SequenceNode
-			newBlank.Node.Style = lhsNode.Style
+			target.Node.Style = lhsNode.Style
-			newBlank.Node.Tag = "!!seq"
+			target.Node.Tag = "!!seq"
-			newBlank.Node.Content = append(lhsNode.Content, toNodes(rhs)...)
+			target.Node.Content = append(lhsNode.Content, toNodes(rhs)...)
-			results.PushBack(newBlank)
+			results.PushBack(target)
 		case yaml.ScalarNode:
 			return nil, fmt.Errorf("Scalars not yet supported for addition")
 		}

pkg/yqlib/operator_add_test.go

@@ -5,6 +5,9 @@ import (
 )
 
 var addOperatorScenarios = []expressionScenario{
+	{
+		description: "+= test and doc",
+	},
 	{
 		description: "Concatenate arrays",
 		document:    `{a: [1,2], b: [3,4]}`,

pkg/yqlib/operator_collect_object.go

@@ -94,7 +94,7 @@ func collect(d *dataTreeNavigator, aggregate *list.List, remainingMatches *list.
 
 			newCandidate.Path = nil
 
-			newCandidate, err = multiply(d, newCandidate, splatCandidate)
+			newCandidate, err = multiply(&MultiplyPreferences{AppendArrays: false})(d, newCandidate, splatCandidate)
 			if err != nil {
 				return nil, err
 			}

pkg/yqlib/operator_multiply.go

@@ -43,39 +43,49 @@ func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, pathNode *Pat
 	return results, nil
 }
 
+type MultiplyPreferences struct {
+	AppendArrays bool
+}
+
 func MultiplyOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) {
 	log.Debugf("-- MultiplyOperator")
-	return crossFunction(d, matchingNodes, pathNode, multiply)
+	return crossFunction(d, matchingNodes, pathNode, multiply(pathNode.Operation.Preferences.(*MultiplyPreferences)))
 }
 
-func multiply(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+func multiply(preferences *MultiplyPreferences) func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-	lhs.Node = UnwrapDoc(lhs.Node)
+	return func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-	rhs.Node = UnwrapDoc(rhs.Node)
+		lhs.Node = UnwrapDoc(lhs.Node)
-	log.Debugf("Multipling LHS: %v", lhs.Node.Tag)
+		rhs.Node = UnwrapDoc(rhs.Node)
-	log.Debugf("-          RHS: %v", rhs.Node.Tag)
+		log.Debugf("Multipling LHS: %v", lhs.Node.Tag)
+		log.Debugf("-          RHS: %v", rhs.Node.Tag)
 
-	if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode ||
+		shouldAppendArrays := preferences.AppendArrays
-		(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
+
+		if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode ||
+			(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
+
+			var newBlank = &CandidateNode{
+				Path:     lhs.Path,
+				Document: lhs.Document,
+				Filename: lhs.Filename,
+				Node:     &yaml.Node{},
+			}
+			var newThing, err = mergeObjects(d, newBlank, lhs, false)
+			if err != nil {
+				return nil, err
+			}
+			return mergeObjects(d, newThing, rhs, shouldAppendArrays)
 
-		var newBlank = &CandidateNode{
-			Path:     lhs.Path,
-			Document: lhs.Document,
-			Filename: lhs.Filename,
-			Node:     &yaml.Node{},
 		}
-		var newThing, err = mergeObjects(d, newBlank, lhs)
+		return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag)
-		if err != nil {
-			return nil, err
-		}
-		return mergeObjects(d, newThing, rhs)
-
 	}
-	return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag)
 }
 
-func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode, shouldAppendArrays bool) (*CandidateNode, error) {
 	var results = list.New()
-	err := recursiveDecent(d, results, nodeToMap(rhs))
+
+	// shouldn't recurse arrays if appending
+	err := recursiveDecent(d, results, nodeToMap(rhs), !shouldAppendArrays)
 	if err != nil {
 		return nil, err
 	}
@@ -86,7 +96,7 @@ func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode)
 	}
 
 	for el := results.Front(); el != nil; el = el.Next() {
-		err := applyAssignment(d, pathIndexToStartFrom, lhs, el.Value.(*CandidateNode))
+		err := applyAssignment(d, pathIndexToStartFrom, lhs, el.Value.(*CandidateNode), shouldAppendArrays)
 		if err != nil {
 			return nil, err
 		}
@@ -107,7 +117,8 @@ func createTraversalTree(path []interface{}) *PathTreeNode {
 
 }
 
-func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode) error {
+func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode, shouldAppendArrays bool) error {
+
 	log.Debugf("merge - applyAssignment lhs %v, rhs: %v", NodeToString(lhs), NodeToString(rhs))
 
 	lhsPath := rhs.Path[pathIndexToStartFrom:]
@@ -116,6 +127,10 @@ func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *Candid
 	if rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode {
 		assignmentOp.OperationType = Assign
 		assignmentOp.Preferences = &AssignOpPreferences{false}
+	} else if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode {
+		log.Debugf("append! lhs %v, rhs: %v", NodeToString(lhs), NodeToString(rhs))
+		assignmentOp.OperationType = Add
+		assignmentOp.Preferences = &AddPreferences{InPlace: true}
 	}
 	rhsOp := &Operation{OperationType: ValueOp, CandidateNode: rhs}
 

pkg/yqlib/operator_multiply_test.go

@@ -5,6 +5,9 @@ import (
 )
 
 var multiplyOperatorScenarios = []expressionScenario{
+	{
+		description: "*+ doc",
+	},
 	{
 		skipDoc:    true,
 		document:   `{a: {also: [1]}, b: {also: me}}`,
@@ -80,15 +83,15 @@ var multiplyOperatorScenarios = []expressionScenario{
 		description:           "Merge keeps style of LHS",
 		dontFormatInputForDoc: true,
 		document: `a: {things: great}
-b:
+	b:
-  also: "me"
+	  also: "me"
-`,
+	`,
 		expression: `. * {"a":.b}`,
 		expected: []string{
 			`D0, P[], (!!map)::a: {things: great, also: "me"}
-b:
+	b:
-    also: "me"
+	    also: "me"
-`,
+	`,
 		},
 	},
 	{
@@ -99,6 +102,22 @@ b:
 			"D0, P[], (!!map)::{a: [3, 4, 5], b: [3, 4, 5]}\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `{a: [1], b: [2]}`,
+		expression: `.a *+ .b`,
+		expected: []string{
+			"D0, P[a], (!!seq)::[1, 2]\n",
+		},
+	},
+	{
+		description: "Merge, appending arrays",
+		document:    `{a: {array: [1, 2, animal: dog], value: coconut}, b: {array: [3, 4, animal: cat], value: banana}}`,
+		expression:  `.a *+ .b`,
+		expected: []string{
+			"D0, P[a], (!!map)::{array: [1, 2, {animal: dog}, 3, 4, {animal: cat}], value: banana}\n",
+		},
+	},
 	{
 		description: "Merge to prefix an element",
 		document:    `{a: cat, b: dog}`,

pkg/yqlib/operator_recursive_descent.go

@@ -3,13 +3,13 @@ package yqlib
 import (
 	"container/list"
 
-	"gopkg.in/yaml.v3"
+	yaml "gopkg.in/yaml.v3"
 )
 
 func RecursiveDescentOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
 	var results = list.New()
 
-	err := recursiveDecent(d, results, matchMap)
+	err := recursiveDecent(d, results, matchMap, true)
 	if err != nil {
 		return nil, err
 	}
@@ -17,7 +17,7 @@ func RecursiveDescentOperator(d *dataTreeNavigator, matchMap *list.List, pathNod
 	return results, nil
 }
 
-func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.List) error {
+func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.List, recurseArray bool) error {
 	for el := matchMap.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 
@@ -26,14 +26,15 @@ func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.Li
 		log.Debugf("Recursive Decent, added %v", NodeToString(candidate))
 		results.PushBack(candidate)
 
-		if candidate.Node.Kind != yaml.AliasNode && len(candidate.Node.Content) > 0 {
+		if candidate.Node.Kind != yaml.AliasNode && len(candidate.Node.Content) > 0 &&
+			(recurseArray || candidate.Node.Kind != yaml.SequenceNode) {
 
 			children, err := Splat(d, nodeToMap(candidate))
 
 			if err != nil {
 				return err
 			}
-			err = recursiveDecent(d, results, children)
+			err = recursiveDecent(d, results, children, recurseArray)
 			if err != nil {
 				return err
 			}

pkg/yqlib/path_tokeniser.go

@@ -251,8 +251,10 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`\]`), literalToken(CloseCollect, true))
 	lexer.Add([]byte(`\{`), literalToken(OpenCollectObject, false))
 	lexer.Add([]byte(`\}`), literalToken(CloseCollectObject, true))
-	lexer.Add([]byte(`\*`), opToken(Multiply))
+	lexer.Add([]byte(`\*`), opTokenWithPrefs(Multiply, nil, &MultiplyPreferences{AppendArrays: false}))
+	lexer.Add([]byte(`\*\+`), opTokenWithPrefs(Multiply, nil, &MultiplyPreferences{AppendArrays: true}))
 	lexer.Add([]byte(`\+`), opToken(Add))
+	lexer.Add([]byte(`\+=`), opTokenWithPrefs(Add, nil, &AddPreferences{InPlace: true}))
 
 	err := lexer.Compile()
 	if err != nil {