Merge! wip

commands_test.go

@@ -1297,32 +1297,45 @@ something: else`
 	test.AssertResult(t, expectedOutput, strings.Trim(result.Output, "\n "))
 }
 
-func xTestMergeCmd(t *testing.T) {
+func TestMergeCmd(t *testing.T) {
 	cmd := getRootCommand()
 	result := test.RunCmd(cmd, "merge examples/data1.yaml examples/data2.yaml")
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
 	expectedOutput := `a: simple
-b:
+b: [1, 2]
-- 1
+c:
-- 2
+  test: 1
+  toast: leave
+  tell: 1
+  taco: cool
+`
+	test.AssertResult(t, expectedOutput, result.Output)
+}
+
+func TestMergeNoAutoCreateCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := test.RunCmd(cmd, "merge -c=false examples/data1.yaml examples/data2.yaml")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	expectedOutput := `a: simple
+b: [1, 2]
 c:
   test: 1
 `
 	test.AssertResult(t, expectedOutput, result.Output)
 }
 
-func xTestMergeOverwriteCmd(t *testing.T) {
+func TestMergeOverwriteCmd(t *testing.T) {
 	cmd := getRootCommand()
-	result := test.RunCmd(cmd, "merge --overwrite examples/data1.yaml examples/data2.yaml")
+	result := test.RunCmd(cmd, "merge -c=false --overwrite examples/data1.yaml examples/data2.yaml")
 	if result.Error != nil {
 		t.Error(result.Error)
 	}
 	expectedOutput := `a: other
-b:
+b: [3, 4]
-- 3
-- 4
 c:
   test: 1
 `

examples/data1.yaml

@@ -1,2 +1,4 @@
 a: simple
 b: [1, 2]
+c:
+  test: 1

examples/sample2.yaml

@@ -1,10 +1,2 @@
-a: Easy! as one two three
 b:
   c: things
-  d: whatever
-things:
- borg: snorg
- thing1:
-  cat: 'fred'
- thing2:
-  cat: 'sam'

pkg/yqlib/lib.go

@@ -12,10 +12,12 @@ import (
 
 var log = logging.MustGetLogger("yq")
 
+// TODO: enumerate
 type UpdateCommand struct {
-	Command string
+	Command   string
-	Path    string
+	Path      string
-	Value   *yaml.Node
+	Value     *yaml.Node
+	Overwrite bool
 }
 
 func DebugNode(value *yaml.Node) {
@@ -76,7 +78,7 @@ func guessKind(head string, tail []string, guess yaml.Kind) yaml.Kind {
 
 type YqLib interface {
 	Get(rootNode *yaml.Node, path string) ([]*NodeContext, error)
-	Update(rootNode *yaml.Node, updateCommand UpdateCommand) error
+	Update(rootNode *yaml.Node, updateCommand UpdateCommand, autoCreate bool) error
 	New(path string) yaml.Node
 }
 
@@ -106,12 +108,12 @@ func (l *lib) New(path string) yaml.Node {
 	return newNode
 }
 
-func (l *lib) Update(rootNode *yaml.Node, updateCommand UpdateCommand) error {
+func (l *lib) Update(rootNode *yaml.Node, updateCommand UpdateCommand, autoCreate bool) error {
 	log.Debugf("%v to %v", updateCommand.Command, updateCommand.Path)
 	switch updateCommand.Command {
 	case "update":
 		var paths = l.parser.ParsePath(updateCommand.Path)
-		navigator := NewDataNavigator(UpdateNavigationStrategy(updateCommand.Value))
+		navigator := NewDataNavigator(UpdateNavigationStrategy(updateCommand, autoCreate))
 		return navigator.Traverse(rootNode, paths)
 	case "delete":
 		var paths = l.parser.ParsePath(updateCommand.Path)

pkg/yqlib/update_navigation_strategy.go

@@ -1,32 +1,33 @@
 package yqlib
 
-import (
+func UpdateNavigationStrategy(updateCommand UpdateCommand, autoCreate bool) NavigationStrategy {
-	yaml "gopkg.in/yaml.v3"
-)
-
-func UpdateNavigationStrategy(changesToApply *yaml.Node) NavigationStrategy {
 	return &NavigationStrategyImpl{
 		visitedNodes: []*NodeContext{},
 		followAlias: func(nodeContext NodeContext) bool {
 			return false
 		},
 		autoCreateMap: func(nodeContext NodeContext) bool {
-			return true
+			return autoCreate
 		},
 		visit: func(nodeContext NodeContext) error {
 			node := nodeContext.Node
-			log.Debug("going to update")
+			changesToApply := updateCommand.Value
-			DebugNode(node)
+			if updateCommand.Overwrite == true || node.Value == "" {
-			log.Debug("with")
+				log.Debug("going to update")
-			DebugNode(changesToApply)
+				DebugNode(node)
-			node.Value = changesToApply.Value
+				log.Debug("with")
-			node.Tag = changesToApply.Tag
+				DebugNode(changesToApply)
-			node.Kind = changesToApply.Kind
+				node.Value = changesToApply.Value
-			node.Style = changesToApply.Style
+				node.Tag = changesToApply.Tag
-			node.Content = changesToApply.Content
+				node.Kind = changesToApply.Kind
-			node.HeadComment = changesToApply.HeadComment
+				node.Style = changesToApply.Style
-			node.LineComment = changesToApply.LineComment
+				node.Content = changesToApply.Content
-			node.FootComment = changesToApply.FootComment
+				node.HeadComment = changesToApply.HeadComment
+				node.LineComment = changesToApply.LineComment
+				node.FootComment = changesToApply.FootComment
+			} else {
+				log.Debug("skipping update as node already has value %v and overwriteFlag is ", node.Value, updateCommand.Overwrite)
+			}
 			return nil
 		},
 	}

yq.go

@@ -22,6 +22,7 @@ var printMode = "v"
 var writeInplace = false
 var writeScript = ""
 var overwriteFlag = false
+var autoCreateFlag = true
 var allowEmptyFlag = false
 var appendFlag = false
 var verbose = false
@@ -235,7 +236,8 @@ Note that if you set both flags only overwrite will take effect.
 		RunE: mergeProperties,
 	}
 	cmdMerge.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace")
-	// cmdMerge.PersistentFlags().BoolVarP(&overwriteFlag, "overwrite", "x", false, "update the yaml file by overwriting existing values")
+	cmdMerge.PersistentFlags().BoolVarP(&overwriteFlag, "overwrite", "x", false, "update the yaml file by overwriting existing values")
+	cmdMerge.PersistentFlags().BoolVarP(&autoCreateFlag, "autocreate", "c", true, "automatically create any missing entries")
 	// cmdMerge.PersistentFlags().BoolVarP(&appendFlag, "append", "a", false, "update the yaml file by appending array values")
 	// cmdMerge.PersistentFlags().BoolVarP(&allowEmptyFlag, "allow-empty", "e", false, "allow empty yaml files")
 	cmdMerge.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)")
@@ -256,10 +258,20 @@ func readProperty(cmd *cobra.Command, args []string) error {
 		return errorParsingDocIndex
 	}
 
+	matchingNodes, errorReadingStream := readYamlFile(args[0], path, updateAll, docIndexInt)
+
+	if errorReadingStream != nil {
+		return errorReadingStream
+	}
+
+	return printResults(matchingNodes, cmd)
+}
+
+func readYamlFile(filename string, path string, updateAll bool, docIndexInt int) ([]*yqlib.NodeContext, error) {
 	var matchingNodes []*yqlib.NodeContext
 
 	var currentIndex = 0
-	var errorReadingStream = readStream(args[0], func(decoder *yaml.Decoder) error {
+	var errorReadingStream = readStream(filename, func(decoder *yaml.Decoder) error {
 		for {
 			var dataBucket yaml.Node
 			errorReading := decoder.Decode(&dataBucket)
@@ -275,12 +287,7 @@ func readProperty(cmd *cobra.Command, args []string) error {
 			currentIndex = currentIndex + 1
 		}
 	})
-
+	return matchingNodes, errorReadingStream
-	if errorReadingStream != nil {
-		return errorReadingStream
-	}
-
-	return printResults(matchingNodes, cmd)
 }
 
 func handleEOF(updateAll bool, docIndexInt int, currentIndex int) error {
@@ -419,7 +426,21 @@ func writeProperty(cmd *cobra.Command, args []string) error {
 
 func mergeProperties(cmd *cobra.Command, args []string) error {
 	// first generate update commands from the file
-	return nil
+	var filesToMerge = args[1:]
+	var updateCommands []yqlib.UpdateCommand = make([]yqlib.UpdateCommand, 0)
+
+	for _, fileToMerge := range filesToMerge {
+		matchingNodes, errorProcessingFile := readYamlFile(fileToMerge, "**", false, 0)
+		if errorProcessingFile != nil {
+			return errorProcessingFile
+		}
+		for _, matchingNode := range matchingNodes {
+			mergePath := yqlib.PathStackToString(matchingNode.PathStack)
+			updateCommands = append(updateCommands, yqlib.UpdateCommand{Command: "update", Path: mergePath, Value: matchingNode.Node, Overwrite: overwriteFlag})
+		}
+	}
+
+	return updateDoc(args[0], updateCommands, cmd.OutOrStdout())
 }
 
 func newProperty(cmd *cobra.Command, args []string) error {
@@ -431,7 +452,7 @@ func newProperty(cmd *cobra.Command, args []string) error {
 
 	for _, updateCommand := range updateCommands {
 
-		errorUpdating := lib.Update(&newNode, updateCommand)
+		errorUpdating := lib.Update(&newNode, updateCommand, true)
 
 		if errorUpdating != nil {
 			return errorUpdating
@@ -474,7 +495,7 @@ func prefixDocument(updateAll bool, docIndexInt int, currentIndex int, dataBucke
 		newNode := lib.New(updateCommand.Path)
 		dataBucket.Content[0] = &newNode
 
-		errorUpdating := lib.Update(dataBucket, updateCommand)
+		errorUpdating := lib.Update(dataBucket, updateCommand, true)
 		if errorUpdating != nil {
 			return errorUpdating
 		}
@@ -502,7 +523,8 @@ func updateDoc(inputFile string, updateCommands []yqlib.UpdateCommand, writer io
 		if updateAll || currentIndex == docIndexInt {
 			log.Debugf("Updating doc %v", currentIndex)
 			for _, updateCommand := range updateCommands {
-				errorUpdating := lib.Update(dataBucket, updateCommand)
+				log.Debugf("Processing update to Path %v", updateCommand.Path)
+				errorUpdating := lib.Update(dataBucket, updateCommand, autoCreateFlag)
 				if errorUpdating != nil {
 					return errorUpdating
 				}
@@ -605,7 +627,7 @@ func readUpdateCommands(args []string, expectedArgs int, badArgsMessage string)
 		log.Debugf("Read write commands file '%v'", parsedCommands)
 		for index := range parsedCommands {
 			parsedCommand := parsedCommands[index]
-			updateCommand := yqlib.UpdateCommand{Command: parsedCommand.Command, Path: parsedCommand.Path, Value: &parsedCommand.Value}
+			updateCommand := yqlib.UpdateCommand{Command: parsedCommand.Command, Path: parsedCommand.Path, Value: &parsedCommand.Value, Overwrite: true}
 			updateCommands = append(updateCommands, updateCommand)
 		}
 
@@ -617,7 +639,7 @@ func readUpdateCommands(args []string, expectedArgs int, badArgsMessage string)
 		log.Debug("args %v", args)
 		log.Debug("path %v", args[expectedArgs-2])
 		log.Debug("Value %v", args[expectedArgs-1])
-		updateCommands[0] = yqlib.UpdateCommand{Command: "update", Path: args[expectedArgs-2], Value: valueParser.Parse(args[expectedArgs-1], customTag)}
+		updateCommands[0] = yqlib.UpdateCommand{Command: "update", Path: args[expectedArgs-2], Value: valueParser.Parse(args[expectedArgs-1], customTag), Overwrite: true}
 	}
 	return updateCommands, nil
 }