New merge flag (n) to only merge in new fields (#1038)

pkg/yqlib/doc/operators/headers/multiply-merge.md

@@ -10,9 +10,10 @@ Note that when merging objects, this operator returns the merged object (not the
 ### Merge Flags
 You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`.  See examples below
 
-- `+` to append arrays
+- `+` append arrays
-- `?` to only merge existing fields
+- `d` deeply merge arrays
-- `d` to deeply merge arrays
+- `?` only merge _existing_ fields
+- `n` only merge _new_ fields
 
 ### Merging files
 Note the use of `eval-all` to ensure all documents are loaded into memory.

pkg/yqlib/doc/operators/multiply-merge.md

@@ -10,9 +10,10 @@ Note that when merging objects, this operator returns the merged object (not the
 ### Merge Flags
 You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`.  See examples below
 
-- `+` to append arrays
+- `+` append arrays
-- `?` to only merge existing fields
+- `d` deeply merge arrays
-- `d` to deeply merge arrays
+- `?` only merge _existing_ fields
+- `n` only merge _new_ fields
 
 ### Merging files
 Note the use of `eval-all` to ensure all documents are loaded into memory.
@@ -148,6 +149,27 @@ thing: two
 cat: frog
 ```
 
+## Merge, only new fields
+Given a sample.yml file of:
+```yaml
+a:
+  thing: one
+  cat: frog
+b:
+  missing: two
+  thing: two
+```
+then
+```bash
+yq eval '.a *n .b' sample.yml
+```
+will output
+```yaml
+thing: one
+cat: frog
+missing: two
+```
+
 ## Merge, appending arrays
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/expression_tokeniser.go

@@ -110,6 +110,9 @@ func multiplyWithPrefs() lex.Action {
 		if strings.Contains(options, "?") {
 			prefs.TraversePrefs = traversePreferences{DontAutoCreate: true}
 		}
+		if strings.Contains(options, "n") {
+			prefs.AssignPrefs = assignPreferences{OnlyWriteNull: true}
+		}
 		if strings.Contains(options, "d") {
 			prefs.DeepMergeArrays = true
 		}
@@ -473,7 +476,7 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`\]\??`), literalToken(closeCollect, true))
 	lexer.Add([]byte(`\{`), literalToken(openCollectObject, false))
 	lexer.Add([]byte(`\}`), literalToken(closeCollectObject, true))
-	lexer.Add([]byte(`\*[\+|\?d]*`), multiplyWithPrefs())
+	lexer.Add([]byte(`\*[\+|\?dn]*`), multiplyWithPrefs())
 	lexer.Add([]byte(`\+`), opToken(addOpType))
 	lexer.Add([]byte(`\+=`), opToken(addAssignOpType))
 	lexer.Add([]byte(`\-`), opToken(subtractOpType))

pkg/yqlib/operator_assign.go

@@ -2,13 +2,15 @@ package yqlib
 
 type assignPreferences struct {
 	DontOverWriteAnchor bool
+	OnlyWriteNull       bool
 }
 
 func assignUpdateFunc(prefs assignPreferences) crossFunctionCalculation {
 	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 		rhs.Node = unwrapDoc(rhs.Node)
-
+		if !prefs.OnlyWriteNull || lhs.Node.Tag == "!!null" {
-		lhs.UpdateFrom(rhs, prefs)
+			lhs.UpdateFrom(rhs, prefs)
+		}
 		return lhs, nil
 	}
 }
@@ -76,7 +78,9 @@ func assignAttributesOperator(d *dataTreeNavigator, context Context, expressionN
 			if expressionNode.Operation.Preferences != nil {
 				prefs = expressionNode.Operation.Preferences.(assignPreferences)
 			}
-			candidate.UpdateAttributesFrom(first.Value.(*CandidateNode), prefs)
+			if !prefs.OnlyWriteNull || candidate.Node.Tag == "!!null" {
+				candidate.UpdateAttributesFrom(first.Value.(*CandidateNode), prefs)
+			}
 		}
 	}
 	return context, nil

pkg/yqlib/operator_multiply.go

@@ -13,6 +13,7 @@ type multiplyPreferences struct {
 	AppendArrays    bool
 	DeepMergeArrays bool
 	TraversePrefs   traversePreferences
+	AssignPrefs     assignPreferences
 }
 
 func multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
@@ -140,7 +141,7 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom
 	lhsPath := rhs.Path[pathIndexToStartFrom:]
 	log.Debugf("merge - lhsPath %v", lhsPath)
 
-	assignmentOp := &Operation{OperationType: assignAttributesOpType}
+	assignmentOp := &Operation{OperationType: assignAttributesOpType, Preferences: preferences.AssignPrefs}
 	if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode {
 		assignmentOp.OperationType = addAssignOpType
 		log.Debugf("merge - assignmentOp.OperationType = addAssignOpType")
@@ -157,7 +158,11 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom
 	}
 	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs}
 
-	assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath, preferences.TraversePrefs, rhs.IsMapKey), Rhs: &ExpressionNode{Operation: rhsOp}}
+	assignmentOpNode := &ExpressionNode{
+		Operation: assignmentOp,
+		Lhs:       createTraversalTree(lhsPath, preferences.TraversePrefs, rhs.IsMapKey),
+		Rhs:       &ExpressionNode{Operation: rhsOp},
+	}
 
 	_, err := d.GetMatchingNodes(context.SingleChildContext(lhs), assignmentOpNode)
 

pkg/yqlib/operator_multiply_test.go

@@ -344,6 +344,14 @@ var multiplyOperatorScenarios = []expressionScenario{
 			"D0, P[a], (!!map)::{thing: two, cat: frog}\n",
 		},
 	},
+	{
+		description: "Merge, only new fields",
+		document:    `{a: {thing: one, cat: frog}, b: {missing: two, thing: two}}`,
+		expression:  `.a *n .b`,
+		expected: []string{
+			"D0, P[a], (!!map)::{thing: one, cat: frog, missing: two}\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `{a: [{thing: one}], b: [{missing: two, thing: two}]}`,
@@ -352,6 +360,14 @@ var multiplyOperatorScenarios = []expressionScenario{
 			"D0, P[a], (!!seq)::[{thing: two}]\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `{a: [{thing: one}], b: [{missing: two, thing: two}]}`,
+		expression: `.a *nd .b`,
+		expected: []string{
+			"D0, P[a], (!!seq)::[{thing: one, missing: two}]\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `{a: {array: [1]}, b: {}}`,
@@ -376,6 +392,16 @@ var multiplyOperatorScenarios = []expressionScenario{
 			"D0, P[a], (!!map)::{thing: [1, 2, 3, 4]}\n",
 		},
 	},
+	{
+		description:    "Merge, only new fields, appending arrays",
+		subdescription: "Append (+) with (n) has no effect.",
+		skipDoc:        true,
+		document:       `{a: {thing: [1,2]}, b: {thing: [3,4], another: [1]}}`,
+		expression:     `.a *n+ .b`,
+		expected: []string{
+			"D0, P[a], (!!map)::{thing: [1, 2], another: [1]}\n",
+		},
+	},
 	{
 		description:    "Merge, deeply merging arrays",
 		subdescription: "Merging arrays deeply means arrays are merge like objects, with indexes as their key. In this case, we merge the first item in the array, and do nothing with the second.",

pkg/yqlib/operators.go

@@ -18,7 +18,7 @@ func compoundAssignFunction(d *dataTreeNavigator, context Context, expressionNod
 		return Context{}, err
 	}
 
-	assignmentOp := &Operation{OperationType: assignOpType}
+	assignmentOp := &Operation{OperationType: assignOpType, Preferences: expressionNode.Operation.Preferences}
 	valueOp := &Operation{OperationType: valueOpType}
 
 	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {