From 81136ad57e4338c17b777a5421eb4383f323f64d Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Thu, 18 Feb 2021 11:16:54 +1100 Subject: [PATCH] Arrays no longer deeply merge by defauly, like jq --- pkg/yqlib/doc/Multiply.md | 51 ++++++++++++++++++++++++++--- pkg/yqlib/doc/headers/Multiply.md | 15 ++++++--- pkg/yqlib/expression_tokeniser.go | 5 ++- pkg/yqlib/operator_multiply.go | 32 +++++++++++++++--- pkg/yqlib/operator_multiply_test.go | 32 +++++++++++++++++- 5 files changed, 118 insertions(+), 17 deletions(-) diff --git a/pkg/yqlib/doc/Multiply.md b/pkg/yqlib/doc/Multiply.md index 0bbc0626..7bff0786 100644 --- a/pkg/yqlib/doc/Multiply.md +++ b/pkg/yqlib/doc/Multiply.md @@ -1,19 +1,34 @@ -Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently only objects are supported, which have the effect of merging the RHS into the LHS. +Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported. -To concatenate arrays when merging objects, use the *+ form (see examples below). This will recursively merge objects, appending arrays when it encounters them. +## Objects and arrays - merging +Objects are merged deeply matching on matching keys. By default, array values override and are not deeply merged. -To merge only existing fields, use the *? form. Note that this can be used with the concatenate arrays too *+?. Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below. -Multiplication of strings and numbers are not yet supported. +### Merge Flags +You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`. See examples below -## Merging files +- `+` to append arrays +- `?` to only merge existing fields +- `d` to deeply merge arrays + +### Merging files Note the use of `eval-all` to ensure all documents are loaded into memory. ```bash yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' file1.yaml file2.yaml ``` +## Multiply integers +Running +```bash +yq eval --null-input '3 * 4' +``` +will output +```yaml +12 +``` + ## Merge objects together, returning merged result only Given a sample.yml file of: ```yaml @@ -190,6 +205,32 @@ thing: - 4 ``` +## Merge, deeply merging arrays +Merging arrays deeply means arrays are merge like objects, with indexes as their key. In this case, we merge the first item in the array, and do nothing with the second. + +Given a sample.yml file of: +```yaml +a: + - name: fred + age: 12 + - name: bob + age: 32 +b: + - name: fred + age: 34 +``` +then +```bash +yq eval '.a *d .b' sample.yml +``` +will output +```yaml +- name: fred + age: 34 +- name: bob + age: 32 +``` + ## Merge to prefix an element Given a sample.yml file of: ```yaml diff --git a/pkg/yqlib/doc/headers/Multiply.md b/pkg/yqlib/doc/headers/Multiply.md index 24c5c557..37730575 100644 --- a/pkg/yqlib/doc/headers/Multiply.md +++ b/pkg/yqlib/doc/headers/Multiply.md @@ -1,13 +1,18 @@ -Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently only objects are supported, which have the effect of merging the RHS into the LHS. +Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported. -To concatenate arrays when merging objects, use the *+ form (see examples below). This will recursively merge objects, appending arrays when it encounters them. +## Objects and arrays - merging +Objects are merged deeply matching on matching keys. By default, array values override and are not deeply merged. -To merge only existing fields, use the *? form. Note that this can be used with the concatenate arrays too *+?. Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below. -Multiplication of strings and numbers are not yet supported. +### Merge Flags +You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`. See examples below -## Merging files +- `+` to append arrays +- `?` to only merge existing fields +- `d` to deeply merge arrays + +### Merging files Note the use of `eval-all` to ensure all documents are loaded into memory. ```bash diff --git a/pkg/yqlib/expression_tokeniser.go b/pkg/yqlib/expression_tokeniser.go index de0f9ad1..372476b3 100644 --- a/pkg/yqlib/expression_tokeniser.go +++ b/pkg/yqlib/expression_tokeniser.go @@ -100,6 +100,9 @@ func multiplyWithPrefs() lex.Action { if strings.Contains(options, "?") { prefs.TraversePrefs = traversePreferences{DontAutoCreate: true} } + if strings.Contains(options, "d") { + prefs.DeepMergeArrays = true + } op := &Operation{OperationType: multiplyOpType, Value: multiplyOpType.Type, StringValue: options, Preferences: prefs} return &token{TokenType: operationToken, Operation: op}, nil } @@ -319,7 +322,7 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`\]`), literalToken(closeCollect, true)) lexer.Add([]byte(`\{`), literalToken(openCollectObject, false)) lexer.Add([]byte(`\}`), literalToken(closeCollectObject, true)) - lexer.Add([]byte(`\*[\+|\?]*`), multiplyWithPrefs()) + lexer.Add([]byte(`\*[\+|\?d]*`), multiplyWithPrefs()) lexer.Add([]byte(`\+`), opToken(addOpType)) lexer.Add([]byte(`\+=`), opToken(addAssignOpType)) lexer.Add([]byte(`\$[a-zA-Z_-0-9]+`), getVariableOpToken()) diff --git a/pkg/yqlib/operator_multiply.go b/pkg/yqlib/operator_multiply.go index f09c689d..573160c1 100644 --- a/pkg/yqlib/operator_multiply.go +++ b/pkg/yqlib/operator_multiply.go @@ -10,8 +10,9 @@ import ( ) type multiplyPreferences struct { - AppendArrays bool - TraversePrefs traversePreferences + AppendArrays bool + DeepMergeArrays bool + TraversePrefs traversePreferences } func multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { @@ -37,11 +38,31 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex return mergeObjects(d, context, newThing, rhs, preferences) } else if lhs.Node.Tag == "!!int" && rhs.Node.Tag == "!!int" { return multiplyIntegers(lhs, rhs) + } else if (lhs.Node.Tag == "!!int" || lhs.Node.Tag == "!!float") && (rhs.Node.Tag == "!!int" || rhs.Node.Tag == "!!float") { + return multiplyFloats(lhs, rhs) } return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag) } } +func multiplyFloats(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { + target := lhs.CreateChild(nil, &yaml.Node{}) + target.Node.Kind = yaml.ScalarNode + target.Node.Style = lhs.Node.Style + target.Node.Tag = "!!float" + + lhsNum, err := strconv.ParseFloat(lhs.Node.Value, 64) + if err != nil { + return nil, err + } + rhsNum, err := strconv.ParseFloat(rhs.Node.Value, 64) + if err != nil { + return nil, err + } + target.Node.Value = fmt.Sprintf("%v", lhsNum*rhsNum) + return target, nil +} + func multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { target := lhs.CreateChild(nil, &yaml.Node{}) target.Node.Kind = yaml.ScalarNode @@ -98,11 +119,12 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom lhsPath := rhs.Path[pathIndexToStartFrom:] assignmentOp := &Operation{OperationType: assignAttributesOpType} - if rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode { + if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode { + assignmentOp.OperationType = addAssignOpType + } else if !preferences.DeepMergeArrays && rhs.Node.Kind == yaml.SequenceNode || + (rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode) { assignmentOp.OperationType = assignOpType assignmentOp.UpdateAssign = false - } else if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode { - assignmentOp.OperationType = addAssignOpType } rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs} diff --git a/pkg/yqlib/operator_multiply_test.go b/pkg/yqlib/operator_multiply_test.go index 76f0e1d9..a903966c 100644 --- a/pkg/yqlib/operator_multiply_test.go +++ b/pkg/yqlib/operator_multiply_test.go @@ -5,6 +5,27 @@ import ( ) var multiplyOperatorScenarios = []expressionScenario{ + { + description: "Multiply integers", + expression: `3 * 4`, + expected: []string{ + "D0, P[], (!!int)::12\n", + }, + }, + { + skipDoc: true, + expression: `3 * 4.5`, + expected: []string{ + "D0, P[], (!!float)::13.5\n", + }, + }, + { + skipDoc: true, + expression: `4.5 * 3`, + expected: []string{ + "D0, P[], (!!float)::13.5\n", + }, + }, { skipDoc: true, document: `{a: {also: [1]}, b: {also: me}}`, @@ -157,7 +178,7 @@ var multiplyOperatorScenarios = []expressionScenario{ { skipDoc: true, document: `{a: [{thing: one}], b: [{missing: two, thing: two}]}`, - expression: `.a *? .b`, + expression: `.a *?d .b`, expected: []string{ "D0, P[a], (!!seq)::[{thing: two}]\n", }, @@ -186,6 +207,15 @@ var multiplyOperatorScenarios = []expressionScenario{ "D0, P[a], (!!map)::{thing: [1, 2, 3, 4]}\n", }, }, + { + description: "Merge, deeply merging arrays", + subdescription: "Merging arrays deeply means arrays are merge like objects, with indexes as their key. In this case, we merge the first item in the array, and do nothing with the second.", + document: `{a: [{name: fred, age: 12}, {name: bob, age: 32}], b: [{name: fred, age: 34}]}`, + expression: `.a *d .b`, + expected: []string{ + "D0, P[a], (!!seq)::[{name: fred, age: 34}, {name: bob, age: 32}]\n", + }, + }, { description: "Merge to prefix an element", document: `{a: cat, b: dog}`,