Arrays no longer deeply merge by defauly, like jq

pkg/yqlib/doc/Multiply.md

@@ -1,19 +1,34 @@
-Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently only objects are supported, which have the effect of merging the RHS into the LHS.
+Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported.
 
-To concatenate arrays when merging objects, use the *+ form (see examples below). This will recursively merge objects, appending arrays when it encounters them.
+## Objects and arrays - merging
+Objects are merged deeply matching on matching keys. By default, array values override and are not deeply merged.
 
-To merge only existing fields, use the *? form. Note that this can be used with the concatenate arrays too *+?.
 Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
 
-Multiplication of strings and numbers are not yet supported.
+### Merge Flags
+You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`.  See examples below
 
-## Merging files
+- `+` to append arrays
+- `?` to only merge existing fields
+- `d` to deeply merge arrays
+
+### Merging files
 Note the use of `eval-all` to ensure all documents are loaded into memory.
 
 ```bash
 yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' file1.yaml file2.yaml
 ```
 
+## Multiply integers
+Running
+```bash
+yq eval --null-input '3 * 4'
+```
+will output
+```yaml
+12
+```
+
 ## Merge objects together, returning merged result only
 Given a sample.yml file of:
 ```yaml
@@ -190,6 +205,32 @@ thing:
   - 4
 ```
 
+## Merge, deeply merging arrays
+Merging arrays deeply means arrays are merge like objects, with indexes as their key. In this case, we merge the first item in the array, and do nothing with the second.
+
+Given a sample.yml file of:
+```yaml
+a:
+  - name: fred
+    age: 12
+  - name: bob
+    age: 32
+b:
+  - name: fred
+    age: 34
+```
+then
+```bash
+yq eval '.a *d .b' sample.yml
+```
+will output
+```yaml
+- name: fred
+  age: 34
+- name: bob
+  age: 32
+```
+
 ## Merge to prefix an element
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/headers/Multiply.md

@@ -1,13 +1,18 @@
-Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently only objects are supported, which have the effect of merging the RHS into the LHS.
+Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported.
 
-To concatenate arrays when merging objects, use the *+ form (see examples below). This will recursively merge objects, appending arrays when it encounters them.
+## Objects and arrays - merging
+Objects are merged deeply matching on matching keys. By default, array values override and are not deeply merged.
 
-To merge only existing fields, use the *? form. Note that this can be used with the concatenate arrays too *+?.
 Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
 
-Multiplication of strings and numbers are not yet supported.
+### Merge Flags
+You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`.  See examples below
 
-## Merging files
+- `+` to append arrays
+- `?` to only merge existing fields
+- `d` to deeply merge arrays
+
+### Merging files
 Note the use of `eval-all` to ensure all documents are loaded into memory.
 
 ```bash

pkg/yqlib/expression_tokeniser.go

@@ -100,6 +100,9 @@ func multiplyWithPrefs() lex.Action {
 		if strings.Contains(options, "?") {
 			prefs.TraversePrefs = traversePreferences{DontAutoCreate: true}
 		}
+		if strings.Contains(options, "d") {
+			prefs.DeepMergeArrays = true
+		}
 		op := &Operation{OperationType: multiplyOpType, Value: multiplyOpType.Type, StringValue: options, Preferences: prefs}
 		return &token{TokenType: operationToken, Operation: op}, nil
 	}
@@ -319,7 +322,7 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`\]`), literalToken(closeCollect, true))
 	lexer.Add([]byte(`\{`), literalToken(openCollectObject, false))
 	lexer.Add([]byte(`\}`), literalToken(closeCollectObject, true))
-	lexer.Add([]byte(`\*[\+|\?]*`), multiplyWithPrefs())
+	lexer.Add([]byte(`\*[\+|\?d]*`), multiplyWithPrefs())
 	lexer.Add([]byte(`\+`), opToken(addOpType))
 	lexer.Add([]byte(`\+=`), opToken(addAssignOpType))
 	lexer.Add([]byte(`\$[a-zA-Z_-0-9]+`), getVariableOpToken())

pkg/yqlib/operator_multiply.go

@@ -10,8 +10,9 @@ import (
 )
 
 type multiplyPreferences struct {
-	AppendArrays  bool
+	AppendArrays    bool
-	TraversePrefs traversePreferences
+	DeepMergeArrays bool
+	TraversePrefs   traversePreferences
 }
 
 func multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
@@ -37,11 +38,31 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
 			return mergeObjects(d, context, newThing, rhs, preferences)
 		} else if lhs.Node.Tag == "!!int" && rhs.Node.Tag == "!!int" {
 			return multiplyIntegers(lhs, rhs)
+		} else if (lhs.Node.Tag == "!!int" || lhs.Node.Tag == "!!float") && (rhs.Node.Tag == "!!int" || rhs.Node.Tag == "!!float") {
+			return multiplyFloats(lhs, rhs)
 		}
 		return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag)
 	}
 }
 
+func multiplyFloats(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	target := lhs.CreateChild(nil, &yaml.Node{})
+	target.Node.Kind = yaml.ScalarNode
+	target.Node.Style = lhs.Node.Style
+	target.Node.Tag = "!!float"
+
+	lhsNum, err := strconv.ParseFloat(lhs.Node.Value, 64)
+	if err != nil {
+		return nil, err
+	}
+	rhsNum, err := strconv.ParseFloat(rhs.Node.Value, 64)
+	if err != nil {
+		return nil, err
+	}
+	target.Node.Value = fmt.Sprintf("%v", lhsNum*rhsNum)
+	return target, nil
+}
+
 func multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	target := lhs.CreateChild(nil, &yaml.Node{})
 	target.Node.Kind = yaml.ScalarNode
@@ -98,11 +119,12 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom
 	lhsPath := rhs.Path[pathIndexToStartFrom:]
 
 	assignmentOp := &Operation{OperationType: assignAttributesOpType}
-	if rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode {
+	if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode {
+		assignmentOp.OperationType = addAssignOpType
+	} else if !preferences.DeepMergeArrays && rhs.Node.Kind == yaml.SequenceNode ||
+		(rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode) {
 		assignmentOp.OperationType = assignOpType
 		assignmentOp.UpdateAssign = false
-	} else if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode {
-		assignmentOp.OperationType = addAssignOpType
 	}
 	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs}
 

pkg/yqlib/operator_multiply_test.go

@@ -5,6 +5,27 @@ import (
 )
 
 var multiplyOperatorScenarios = []expressionScenario{
+	{
+		description: "Multiply integers",
+		expression:  `3 * 4`,
+		expected: []string{
+			"D0, P[], (!!int)::12\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `3 * 4.5`,
+		expected: []string{
+			"D0, P[], (!!float)::13.5\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `4.5 * 3`,
+		expected: []string{
+			"D0, P[], (!!float)::13.5\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `{a: {also: [1]}, b: {also: me}}`,
@@ -157,7 +178,7 @@ var multiplyOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{a: [{thing: one}], b: [{missing: two, thing: two}]}`,
-		expression: `.a *? .b`,
+		expression: `.a *?d .b`,
 		expected: []string{
 			"D0, P[a], (!!seq)::[{thing: two}]\n",
 		},
@@ -186,6 +207,15 @@ var multiplyOperatorScenarios = []expressionScenario{
 			"D0, P[a], (!!map)::{thing: [1, 2, 3, 4]}\n",
 		},
 	},
+	{
+		description:    "Merge, deeply merging arrays",
+		subdescription: "Merging arrays deeply means arrays are merge like objects, with indexes as their key. In this case, we merge the first item in the array, and do nothing with the second.",
+		document:       `{a: [{name: fred, age: 12}, {name: bob, age: 32}], b: [{name: fred, age: 34}]}`,
+		expression:     `.a *d .b`,
+		expected: []string{
+			"D0, P[a], (!!seq)::[{name: fred, age: 34}, {name: bob, age: 32}]\n",
+		},
+	},
 	{
 		description: "Merge to prefix an element",
 		document:    `{a: cat, b: dog}`,