From 8c94a96ee0461640706c12818783ee5b67f22492 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sat, 15 Jan 2022 15:48:34 +1100 Subject: [PATCH] New merge flag (n) to only merge in new fields (#1038) --- .../doc/operators/headers/multiply-merge.md | 7 +++-- pkg/yqlib/doc/operators/multiply-merge.md | 28 +++++++++++++++++-- pkg/yqlib/expression_tokeniser.go | 5 +++- pkg/yqlib/operator_assign.go | 10 +++++-- pkg/yqlib/operator_multiply.go | 9 ++++-- pkg/yqlib/operator_multiply_test.go | 26 +++++++++++++++++ pkg/yqlib/operators.go | 2 +- 7 files changed, 74 insertions(+), 13 deletions(-) diff --git a/pkg/yqlib/doc/operators/headers/multiply-merge.md b/pkg/yqlib/doc/operators/headers/multiply-merge.md index 5f6e079a..a2e69df1 100644 --- a/pkg/yqlib/doc/operators/headers/multiply-merge.md +++ b/pkg/yqlib/doc/operators/headers/multiply-merge.md @@ -10,9 +10,10 @@ Note that when merging objects, this operator returns the merged object (not the ### Merge Flags You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`. See examples below -- `+` to append arrays -- `?` to only merge existing fields -- `d` to deeply merge arrays +- `+` append arrays +- `d` deeply merge arrays +- `?` only merge _existing_ fields +- `n` only merge _new_ fields ### Merging files Note the use of `eval-all` to ensure all documents are loaded into memory. diff --git a/pkg/yqlib/doc/operators/multiply-merge.md b/pkg/yqlib/doc/operators/multiply-merge.md index b06a6834..1fea5319 100644 --- a/pkg/yqlib/doc/operators/multiply-merge.md +++ b/pkg/yqlib/doc/operators/multiply-merge.md @@ -10,9 +10,10 @@ Note that when merging objects, this operator returns the merged object (not the ### Merge Flags You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`. See examples below -- `+` to append arrays -- `?` to only merge existing fields -- `d` to deeply merge arrays +- `+` append arrays +- `d` deeply merge arrays +- `?` only merge _existing_ fields +- `n` only merge _new_ fields ### Merging files Note the use of `eval-all` to ensure all documents are loaded into memory. @@ -148,6 +149,27 @@ thing: two cat: frog ``` +## Merge, only new fields +Given a sample.yml file of: +```yaml +a: + thing: one + cat: frog +b: + missing: two + thing: two +``` +then +```bash +yq eval '.a *n .b' sample.yml +``` +will output +```yaml +thing: one +cat: frog +missing: two +``` + ## Merge, appending arrays Given a sample.yml file of: ```yaml diff --git a/pkg/yqlib/expression_tokeniser.go b/pkg/yqlib/expression_tokeniser.go index 465bf5a5..caaf6d15 100644 --- a/pkg/yqlib/expression_tokeniser.go +++ b/pkg/yqlib/expression_tokeniser.go @@ -110,6 +110,9 @@ func multiplyWithPrefs() lex.Action { if strings.Contains(options, "?") { prefs.TraversePrefs = traversePreferences{DontAutoCreate: true} } + if strings.Contains(options, "n") { + prefs.AssignPrefs = assignPreferences{OnlyWriteNull: true} + } if strings.Contains(options, "d") { prefs.DeepMergeArrays = true } @@ -473,7 +476,7 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`\]\??`), literalToken(closeCollect, true)) lexer.Add([]byte(`\{`), literalToken(openCollectObject, false)) lexer.Add([]byte(`\}`), literalToken(closeCollectObject, true)) - lexer.Add([]byte(`\*[\+|\?d]*`), multiplyWithPrefs()) + lexer.Add([]byte(`\*[\+|\?dn]*`), multiplyWithPrefs()) lexer.Add([]byte(`\+`), opToken(addOpType)) lexer.Add([]byte(`\+=`), opToken(addAssignOpType)) lexer.Add([]byte(`\-`), opToken(subtractOpType)) diff --git a/pkg/yqlib/operator_assign.go b/pkg/yqlib/operator_assign.go index 61417df7..b120c5f8 100644 --- a/pkg/yqlib/operator_assign.go +++ b/pkg/yqlib/operator_assign.go @@ -2,13 +2,15 @@ package yqlib type assignPreferences struct { DontOverWriteAnchor bool + OnlyWriteNull bool } func assignUpdateFunc(prefs assignPreferences) crossFunctionCalculation { return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { rhs.Node = unwrapDoc(rhs.Node) - - lhs.UpdateFrom(rhs, prefs) + if !prefs.OnlyWriteNull || lhs.Node.Tag == "!!null" { + lhs.UpdateFrom(rhs, prefs) + } return lhs, nil } } @@ -76,7 +78,9 @@ func assignAttributesOperator(d *dataTreeNavigator, context Context, expressionN if expressionNode.Operation.Preferences != nil { prefs = expressionNode.Operation.Preferences.(assignPreferences) } - candidate.UpdateAttributesFrom(first.Value.(*CandidateNode), prefs) + if !prefs.OnlyWriteNull || candidate.Node.Tag == "!!null" { + candidate.UpdateAttributesFrom(first.Value.(*CandidateNode), prefs) + } } } return context, nil diff --git a/pkg/yqlib/operator_multiply.go b/pkg/yqlib/operator_multiply.go index d22e2612..8575c9ea 100644 --- a/pkg/yqlib/operator_multiply.go +++ b/pkg/yqlib/operator_multiply.go @@ -13,6 +13,7 @@ type multiplyPreferences struct { AppendArrays bool DeepMergeArrays bool TraversePrefs traversePreferences + AssignPrefs assignPreferences } func multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { @@ -140,7 +141,7 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom lhsPath := rhs.Path[pathIndexToStartFrom:] log.Debugf("merge - lhsPath %v", lhsPath) - assignmentOp := &Operation{OperationType: assignAttributesOpType} + assignmentOp := &Operation{OperationType: assignAttributesOpType, Preferences: preferences.AssignPrefs} if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode { assignmentOp.OperationType = addAssignOpType log.Debugf("merge - assignmentOp.OperationType = addAssignOpType") @@ -157,7 +158,11 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom } rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs} - assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath, preferences.TraversePrefs, rhs.IsMapKey), Rhs: &ExpressionNode{Operation: rhsOp}} + assignmentOpNode := &ExpressionNode{ + Operation: assignmentOp, + Lhs: createTraversalTree(lhsPath, preferences.TraversePrefs, rhs.IsMapKey), + Rhs: &ExpressionNode{Operation: rhsOp}, + } _, err := d.GetMatchingNodes(context.SingleChildContext(lhs), assignmentOpNode) diff --git a/pkg/yqlib/operator_multiply_test.go b/pkg/yqlib/operator_multiply_test.go index ab136f13..b3711048 100644 --- a/pkg/yqlib/operator_multiply_test.go +++ b/pkg/yqlib/operator_multiply_test.go @@ -344,6 +344,14 @@ var multiplyOperatorScenarios = []expressionScenario{ "D0, P[a], (!!map)::{thing: two, cat: frog}\n", }, }, + { + description: "Merge, only new fields", + document: `{a: {thing: one, cat: frog}, b: {missing: two, thing: two}}`, + expression: `.a *n .b`, + expected: []string{ + "D0, P[a], (!!map)::{thing: one, cat: frog, missing: two}\n", + }, + }, { skipDoc: true, document: `{a: [{thing: one}], b: [{missing: two, thing: two}]}`, @@ -352,6 +360,14 @@ var multiplyOperatorScenarios = []expressionScenario{ "D0, P[a], (!!seq)::[{thing: two}]\n", }, }, + { + skipDoc: true, + document: `{a: [{thing: one}], b: [{missing: two, thing: two}]}`, + expression: `.a *nd .b`, + expected: []string{ + "D0, P[a], (!!seq)::[{thing: one, missing: two}]\n", + }, + }, { skipDoc: true, document: `{a: {array: [1]}, b: {}}`, @@ -376,6 +392,16 @@ var multiplyOperatorScenarios = []expressionScenario{ "D0, P[a], (!!map)::{thing: [1, 2, 3, 4]}\n", }, }, + { + description: "Merge, only new fields, appending arrays", + subdescription: "Append (+) with (n) has no effect.", + skipDoc: true, + document: `{a: {thing: [1,2]}, b: {thing: [3,4], another: [1]}}`, + expression: `.a *n+ .b`, + expected: []string{ + "D0, P[a], (!!map)::{thing: [1, 2], another: [1]}\n", + }, + }, { description: "Merge, deeply merging arrays", subdescription: "Merging arrays deeply means arrays are merge like objects, with indexes as their key. In this case, we merge the first item in the array, and do nothing with the second.", diff --git a/pkg/yqlib/operators.go b/pkg/yqlib/operators.go index a88d9872..5358eee2 100644 --- a/pkg/yqlib/operators.go +++ b/pkg/yqlib/operators.go @@ -18,7 +18,7 @@ func compoundAssignFunction(d *dataTreeNavigator, context Context, expressionNod return Context{}, err } - assignmentOp := &Operation{OperationType: assignOpType} + assignmentOp := &Operation{OperationType: assignOpType, Preferences: expressionNode.Operation.Preferences} valueOp := &Operation{OperationType: valueOpType} for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {