suyiiyii
/
yq
mirror of https://github.com/mikefarah/yq.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Arrays no longer deeply merge by defauly, like jq

This commit is contained in:
Mike Farah 2021-02-18 11:16:54 +11:00
parent a6cd250987
commit 81136ad57e
5 changed files with 118 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
pkg/yqlib/doc/Multiply.md
View File

@ -1,19 +1,34 @@
Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently only objects are supported, which have the effect of merging the RHS into the LHS.
Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported.
To concatenate arrays when merging objects, use the *+ form (see examples below). This will recursively merge objects, appending arrays when it encounters them.
## Objects and arrays - merging
Objects are merged deeply matching on matching keys. By default, array values override and are not deeply merged.
To merge only existing fields, use the *? form. Note that this can be used with the concatenate arrays too *+?.
Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
Multiplication of strings and numbers are not yet supported.
### Merge Flags
You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`. See examples below
## Merging files
- `+` to append arrays
- `?` to only merge existing fields
- `d` to deeply merge arrays
### Merging files
Note the use of `eval-all` to ensure all documents are loaded into memory.
```bash
yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' file1.yaml file2.yaml
```
## Multiply integers
Running
```bash
yq eval --null-input '3 * 4'
```
will output
```yaml
12
```
## Merge objects together, returning merged result only
Given a sample.yml file of:
```yaml
@ -190,6 +205,32 @@ thing:
- 4
```
## Merge, deeply merging arrays
Merging arrays deeply means arrays are merge like objects, with indexes as their key. In this case, we merge the first item in the array, and do nothing with the second.
Given a sample.yml file of:
```yaml
a:
- name: fred
age: 12
- name: bob
age: 32
b:
- name: fred
age: 34
```
then
```bash
yq eval '.a *d .b' sample.yml
```
will output
```yaml
- name: fred
age: 34
- name: bob
age: 32
```
## Merge to prefix an element
Given a sample.yml file of:
```yaml

15
pkg/yqlib/doc/headers/Multiply.md
View File

@ -1,13 +1,18 @@
Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently only objects are supported, which have the effect of merging the RHS into the LHS.
Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported.
To concatenate arrays when merging objects, use the *+ form (see examples below). This will recursively merge objects, appending arrays when it encounters them.
## Objects and arrays - merging
Objects are merged deeply matching on matching keys. By default, array values override and are not deeply merged.
To merge only existing fields, use the *? form. Note that this can be used with the concatenate arrays too *+?.
Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
Multiplication of strings and numbers are not yet supported.
### Merge Flags
You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`. See examples below
## Merging files
- `+` to append arrays
- `?` to only merge existing fields
- `d` to deeply merge arrays
### Merging files
Note the use of `eval-all` to ensure all documents are loaded into memory.
```bash

5
pkg/yqlib/expression_tokeniser.go
View File

@ -100,6 +100,9 @@ func multiplyWithPrefs() lex.Action {
if strings.Contains(options, "?") {
prefs.TraversePrefs = traversePreferences{DontAutoCreate: true}
}
if strings.Contains(options, "d") {
prefs.DeepMergeArrays = true
}
op := &Operation{OperationType: multiplyOpType, Value: multiplyOpType.Type, StringValue: options, Preferences: prefs}
return &token{TokenType: operationToken, Operation: op}, nil
}
@ -319,7 +322,7 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`\]`), literalToken(closeCollect, true))
lexer.Add([]byte(`\{`), literalToken(openCollectObject, false))
lexer.Add([]byte(`\}`), literalToken(closeCollectObject, true))
lexer.Add([]byte(`\*[\+|\?]*`), multiplyWithPrefs())
lexer.Add([]byte(`\*[\+|\?d]*`), multiplyWithPrefs())
lexer.Add([]byte(`\+`), opToken(addOpType))
lexer.Add([]byte(`\+=`), opToken(addAssignOpType))
lexer.Add([]byte(`\$[a-zA-Z_-0-9]+`), getVariableOpToken())

32
pkg/yqlib/operator_multiply.go
View File

@ -10,8 +10,9 @@ import (
)
type multiplyPreferences struct {
AppendArrays bool
TraversePrefs traversePreferences
AppendArrays bool
DeepMergeArrays bool
TraversePrefs traversePreferences
}
func multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
@ -37,11 +38,31 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
return mergeObjects(d, context, newThing, rhs, preferences)
} else if lhs.Node.Tag == "!!int" && rhs.Node.Tag == "!!int" {
return multiplyIntegers(lhs, rhs)
} else if (lhs.Node.Tag == "!!int" || lhs.Node.Tag == "!!float") && (rhs.Node.Tag == "!!int" || rhs.Node.Tag == "!!float") {
return multiplyFloats(lhs, rhs)
}
return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag)
}
}
func multiplyFloats(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
target := lhs.CreateChild(nil, &yaml.Node{})
target.Node.Kind = yaml.ScalarNode
target.Node.Style = lhs.Node.Style
target.Node.Tag = "!!float"
lhsNum, err := strconv.ParseFloat(lhs.Node.Value, 64)
if err != nil {
return nil, err
}
rhsNum, err := strconv.ParseFloat(rhs.Node.Value, 64)
if err != nil {
return nil, err
}
target.Node.Value = fmt.Sprintf("%v", lhsNum*rhsNum)
return target, nil
}
func multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
target := lhs.CreateChild(nil, &yaml.Node{})
target.Node.Kind = yaml.ScalarNode
@ -98,11 +119,12 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom
lhsPath := rhs.Path[pathIndexToStartFrom:]
assignmentOp := &Operation{OperationType: assignAttributesOpType}
if rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode {
if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode {
assignmentOp.OperationType = addAssignOpType
} else if !preferences.DeepMergeArrays && rhs.Node.Kind == yaml.SequenceNode ||
(rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode) {
assignmentOp.OperationType = assignOpType
assignmentOp.UpdateAssign = false
} else if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode {
assignmentOp.OperationType = addAssignOpType
}
rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs}

32
pkg/yqlib/operator_multiply_test.go
View File

@ -5,6 +5,27 @@ import (
)
var multiplyOperatorScenarios = []expressionScenario{
{
description: "Multiply integers",
expression: `3 * 4`,
expected: []string{
"D0, P[], (!!int)::12\n",
},
},
{
skipDoc: true,
expression: `3 * 4.5`,
expected: []string{
"D0, P[], (!!float)::13.5\n",
},
},
{
skipDoc: true,
expression: `4.5 * 3`,
expected: []string{
"D0, P[], (!!float)::13.5\n",
},
},
{
skipDoc: true,
document: `{a: {also: [1]}, b: {also: me}}`,
@ -157,7 +178,7 @@ var multiplyOperatorScenarios = []expressionScenario{
{
skipDoc: true,
document: `{a: [{thing: one}], b: [{missing: two, thing: two}]}`,
expression: `.a *? .b`,
expression: `.a *?d .b`,
expected: []string{
"D0, P[a], (!!seq)::[{thing: two}]\n",
},
@ -186,6 +207,15 @@ var multiplyOperatorScenarios = []expressionScenario{
"D0, P[a], (!!map)::{thing: [1, 2, 3, 4]}\n",
},
},
{
description: "Merge, deeply merging arrays",
subdescription: "Merging arrays deeply means arrays are merge like objects, with indexes as their key. In this case, we merge the first item in the array, and do nothing with the second.",
document: `{a: [{name: fred, age: 12}, {name: bob, age: 32}], b: [{name: fred, age: 34}]}`,
expression: `.a *d .b`,
expected: []string{
"D0, P[a], (!!seq)::[{name: fred, age: 34}, {name: bob, age: 32}]\n",
},
},
{
description: "Merge to prefix an element",
document: `{a: cat, b: dog}`,