Refactored merge - will allow more sophisticated mergin

cmd/merge.go

@@ -43,7 +43,7 @@ If append flag is set then existing arrays will be merged with the arrays from e
  */
 func createReadFunctionForMerge() func(*yaml.Node) ([]*yqlib.NodeContext, error) {
 	return func(dataBucket *yaml.Node) ([]*yqlib.NodeContext, error) {
-		return lib.Get(dataBucket, "**", !appendFlag)
+		return lib.GetForMerge(dataBucket, "**", !appendFlag)
 	}
 }
 
@@ -63,9 +63,21 @@ func mergeProperties(cmd *cobra.Command, args []string) error {
 			if errorProcessingFile != nil {
 				return errorProcessingFile
 			}
+			log.Debugf("finished reading for merge!")
+			for _, matchingNode := range matchingNodes {
+				log.Debugf("matched node %v", lib.PathStackToString(matchingNode.PathStack))
+				yqlib.DebugNode(matchingNode.Node)
+			}
 			for _, matchingNode := range matchingNodes {
 				mergePath := lib.MergePathStackToString(matchingNode.PathStack, appendFlag)
-				updateCommands = append(updateCommands, yqlib.UpdateCommand{Command: "update", Path: mergePath, Value: matchingNode.Node, Overwrite: overwriteFlag})
+				updateCommands = append(updateCommands, yqlib.UpdateCommand{
+					Command:   "merge",
+					Path:      mergePath,
+					Value:     matchingNode.Node,
+					Overwrite: overwriteFlag,
+					// dont update the content for nodes midway, only leaf nodes
+					DontUpdateNodeContent: matchingNode.IsMiddleNode,
+				})
 			}
 		}
 	}

cmd/merge_test.go

@@ -129,7 +129,11 @@ func TestMergeArraysCmd(t *testing.T) {
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
-	expectedOutput := `[1, 2, 3, 4, 5]
+	expectedOutput := `- 1
+- 2
+- 3
+- 4
+- 5
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
@@ -145,9 +149,7 @@ func TestMergeCmd_Multi(t *testing.T) {
 another:
   document: here
 a: simple # just the best
-b:
+b: [1, 2]
-  - 1
-  - 2
 c:
   test: 1
 ---
@@ -316,9 +318,7 @@ func TestMergeAllowEmptyTargetCmd(t *testing.T) {
 		t.Error(result.Error)
 	}
 	expectedOutput := `a: simple # just the best
-b:
+b: [1, 2]
-  - 1
-  - 2
 c:
   test: 1
 `

cmd/utils.go

@@ -17,7 +17,7 @@ type readDataFn func(dataBucket *yaml.Node) ([]*yqlib.NodeContext, error)
 
 func createReadFunction(path string) func(*yaml.Node) ([]*yqlib.NodeContext, error) {
 	return func(dataBucket *yaml.Node) ([]*yqlib.NodeContext, error) {
-		return lib.Get(dataBucket, path, true)
+		return lib.Get(dataBucket, path)
 	}
 }
 

pkg/yqlib/data_navigator.go

@@ -73,6 +73,7 @@ func (n *navigator) recurse(value *yaml.Node, head interface{}, tail []interface
 	nodeContext := NewNodeContext(value, head, tail, pathStack)
 
 	if head == "**" && !n.navigationStrategy.ShouldOnlyDeeplyVisitLeaves(nodeContext) {
+		nodeContext.IsMiddleNode = true
 		errorVisitingDeeply := n.navigationStrategy.Visit(nodeContext)
 		if errorVisitingDeeply != nil {
 			return errorVisitingDeeply

pkg/yqlib/lib.go

@@ -13,11 +13,12 @@ import (
 var log = logging.MustGetLogger("yq")
 
 type UpdateCommand struct {
-	Command             string
+	Command               string
-	Path                string
+	Path                  string
-	Value               *yaml.Node
+	Value                 *yaml.Node
-	Overwrite           bool
+	Overwrite             bool
-	DontUpdateNodeValue bool
+	DontUpdateNodeValue   bool
+	DontUpdateNodeContent bool
 }
 
 func KindString(kind yaml.Kind) string {
@@ -49,7 +50,10 @@ func DebugNode(value *yaml.Node) {
 		}
 		encoder.Close()
 		log.Debug("Tag: %v, Kind: %v, Anchor: %v", value.Tag, KindString(value.Kind), value.Anchor)
-		log.Debug("%v", buf.String())
+		log.Debug("Head Comment: %v", value.HeadComment)
+		log.Debug("Line Comment: %v", value.LineComment)
+		log.Debug("FootComment Comment: %v", value.FootComment)
+		log.Debug("\n%v", buf.String())
 	}
 }
 
@@ -131,7 +135,8 @@ func guessKind(head interface{}, tail []interface{}, guess yaml.Kind) yaml.Kind
 }
 
 type YqLib interface {
-	Get(rootNode *yaml.Node, path string, deeplyTraverseArrays bool) ([]*NodeContext, error)
+	Get(rootNode *yaml.Node, path string) ([]*NodeContext, error)
+	GetForMerge(rootNode *yaml.Node, path string, deeplyTraverseArrays bool) ([]*NodeContext, error)
 	Update(rootNode *yaml.Node, updateCommand UpdateCommand, autoCreate bool) error
 	New(path string) yaml.Node
 
@@ -149,13 +154,20 @@ func NewYqLib() YqLib {
 	}
 }
 
-func (l *lib) Get(rootNode *yaml.Node, path string, deeplyTraverseArrays bool) ([]*NodeContext, error) {
+func (l *lib) Get(rootNode *yaml.Node, path string) ([]*NodeContext, error) {
 	var paths = l.parser.ParsePath(path)
-	navigationStrategy := ReadNavigationStrategy(deeplyTraverseArrays)
+	navigationStrategy := ReadNavigationStrategy()
 	navigator := NewDataNavigator(navigationStrategy)
 	error := navigator.Traverse(rootNode, paths)
 	return navigationStrategy.GetVisitedNodes(), error
+}
 
+func (l *lib) GetForMerge(rootNode *yaml.Node, path string, deeplyTraverseArrays bool) ([]*NodeContext, error) {
+	var paths = l.parser.ParsePath(path)
+	navigationStrategy := ReadForMergeNavigationStrategy(deeplyTraverseArrays)
+	navigator := NewDataNavigator(navigationStrategy)
+	error := navigator.Traverse(rootNode, paths)
+	return navigationStrategy.GetVisitedNodes(), error
 }
 
 func (l *lib) PathStackToString(pathStack []interface{}) string {
@@ -179,6 +191,10 @@ func (l *lib) Update(rootNode *yaml.Node, updateCommand UpdateCommand, autoCreat
 		var paths = l.parser.ParsePath(updateCommand.Path)
 		navigator := NewDataNavigator(UpdateNavigationStrategy(updateCommand, autoCreate))
 		return navigator.Traverse(rootNode, paths)
+	case "merge":
+		var paths = l.parser.ParsePath(updateCommand.Path)
+		navigator := NewDataNavigator(MergeNavigationStrategy(updateCommand, autoCreate))
+		return navigator.Traverse(rootNode, paths)
 	case "delete":
 		var paths = l.parser.ParsePath(updateCommand.Path)
 		lastBit, newTail := paths[len(paths)-1], paths[:len(paths)-1]

pkg/yqlib/merge_navigation_strategy.go

@@ -0,0 +1,63 @@
+package yqlib
+
+import "gopkg.in/yaml.v3"
+
+func MergeNavigationStrategy(updateCommand UpdateCommand, autoCreate bool) NavigationStrategy {
+	return &NavigationStrategyImpl{
+		visitedNodes: []*NodeContext{},
+		pathParser:   NewPathParser(),
+		followAlias: func(nodeContext NodeContext) bool {
+			return false
+		},
+		autoCreateMap: func(nodeContext NodeContext) bool {
+			return autoCreate
+		},
+		visit: func(nodeContext NodeContext) error {
+			node := nodeContext.Node
+			changesToApply := updateCommand.Value
+
+			if node.Kind == yaml.DocumentNode && changesToApply.Kind != yaml.DocumentNode {
+				// when the path is empty, it matches both the top level pseudo document node
+				// and the actual top level node (e.g. map/sequence/whatever)
+				// so when we are updating with no path, make sure we update the right node.
+				node = node.Content[0]
+			}
+
+			if updateCommand.Overwrite || node.Value == "" {
+				log.Debug("going to update")
+				DebugNode(node)
+				log.Debug("with")
+				DebugNode(changesToApply)
+				node.Value = changesToApply.Value
+				node.Tag = changesToApply.Tag
+				node.Kind = changesToApply.Kind
+				node.Style = changesToApply.Style
+				node.Anchor = changesToApply.Anchor
+				node.Alias = changesToApply.Alias
+				node.HeadComment = changesToApply.HeadComment
+				node.LineComment = changesToApply.LineComment
+				node.FootComment = changesToApply.FootComment
+
+				if !updateCommand.DontUpdateNodeContent {
+					node.Content = changesToApply.Content
+				}
+
+				// // TODO: mergeComments flag
+				// if node.HeadComment != "" && changesToApply.HeadComment != "" {
+				// 	node.HeadComment = node.HeadComment + "\n" + changesToApply.HeadComment
+				// 	log.Debug("merged comments with a space, %v", node.HeadComment)
+				// } else {
+				// 	node.HeadComment = node.HeadComment + changesToApply.HeadComment
+				// 	if node.HeadComment != "" {
+				// 		log.Debug("merged comments with no space, %v", node.HeadComment)
+				// 	}
+				// }
+				// node.LineComment = node.LineComment + changesToApply.LineComment
+				// node.FootComment = node.FootComment + changesToApply.FootComment
+			} else {
+				log.Debug("skipping update as node already has value %v and overwriteFlag is ", node.Value, updateCommand.Overwrite)
+			}
+			return nil
+		},
+	}
+}

pkg/yqlib/navigation_strategy.go

@@ -11,6 +11,9 @@ type NodeContext struct {
 	Head      interface{}
 	Tail      []interface{}
 	PathStack []interface{}
+	// middle nodes are nodes that match along the original path, but not a
+	// target match of the path. This is only relevant when ShouldOnlyDeeplyVisitLeaves is false.
+	IsMiddleNode bool
 }
 
 func NewNodeContext(node *yaml.Node, head interface{}, tail []interface{}, pathStack []interface{}) NodeContext {

pkg/yqlib/read_for_merge_navigation_strategy.go

@@ -0,0 +1,30 @@
+package yqlib
+
+func ReadForMergeNavigationStrategy(deeplyTraverseArrays bool) NavigationStrategy {
+	return &NavigationStrategyImpl{
+		visitedNodes: []*NodeContext{},
+		pathParser:   NewPathParser(),
+		followAlias: func(nodeContext NodeContext) bool {
+			return false
+		},
+		shouldOnlyDeeplyVisitLeaves: func(nodeContext NodeContext) bool {
+			return false
+		},
+		visit: func(nodeContext NodeContext) error {
+			return nil
+		},
+		shouldDeeplyTraverse: func(nodeContext NodeContext) bool {
+			var isInArray = false
+			if len(nodeContext.PathStack) > 0 {
+				var lastElement = nodeContext.PathStack[len(nodeContext.PathStack)-1]
+				switch lastElement.(type) {
+				case int:
+					isInArray = true
+				default:
+					isInArray = false
+				}
+			}
+			return deeplyTraverseArrays || !isInArray
+		},
+	}
+}

pkg/yqlib/read_navigation_strategy.go

@@ -1,24 +1,11 @@
 package yqlib
 
-func ReadNavigationStrategy(deeplyTraverseArrays bool) NavigationStrategy {
+func ReadNavigationStrategy() NavigationStrategy {
 	return &NavigationStrategyImpl{
 		visitedNodes: []*NodeContext{},
 		pathParser:   NewPathParser(),
 		visit: func(nodeContext NodeContext) error {
 			return nil
 		},
-		shouldDeeplyTraverse: func(nodeContext NodeContext) bool {
-			var isInArray = false
-			if len(nodeContext.PathStack) > 0 {
-				var lastElement = nodeContext.PathStack[len(nodeContext.PathStack)-1]
-				switch lastElement.(type) {
-				case int:
-					isInArray = true
-				default:
-					isInArray = false
-				}
-			}
-			return deeplyTraverseArrays || !isInArray
-		},
 	}
 }

pkg/yqlib/update_navigation_strategy.go

@@ -24,7 +24,9 @@ func UpdateNavigationStrategy(updateCommand UpdateCommand, autoCreate bool) Navi
 				node.Tag = changesToApply.Tag
 				node.Kind = changesToApply.Kind
 				node.Style = changesToApply.Style
-				node.Content = changesToApply.Content
+				if !updateCommand.DontUpdateNodeContent {
+					node.Content = changesToApply.Content
+				}
 				node.Anchor = changesToApply.Anchor
 				node.Alias = changesToApply.Alias
 				node.HeadComment = changesToApply.HeadComment