From 1aa5ec1d40d5f5cb7d5b443ed5e8aecffabcfc5e Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sun, 5 Jan 2020 17:14:14 +1300 Subject: [PATCH] Merge! wip --- commands_test.go | 31 ++++++++++----- examples/data1.yaml | 2 + examples/sample2.yaml | 10 +---- pkg/yqlib/lib.go | 14 ++++--- pkg/yqlib/update_navigation_strategy.go | 37 +++++++++--------- yq.go | 50 ++++++++++++++++++------- 6 files changed, 88 insertions(+), 56 deletions(-) diff --git a/commands_test.go b/commands_test.go index dee6722b..4f73533a 100644 --- a/commands_test.go +++ b/commands_test.go @@ -1297,32 +1297,45 @@ something: else` test.AssertResult(t, expectedOutput, strings.Trim(result.Output, "\n ")) } -func xTestMergeCmd(t *testing.T) { +func TestMergeCmd(t *testing.T) { cmd := getRootCommand() result := test.RunCmd(cmd, "merge examples/data1.yaml examples/data2.yaml") if result.Error != nil { t.Error(result.Error) } expectedOutput := `a: simple -b: -- 1 -- 2 +b: [1, 2] +c: + test: 1 + toast: leave + tell: 1 + taco: cool +` + test.AssertResult(t, expectedOutput, result.Output) +} + +func TestMergeNoAutoCreateCmd(t *testing.T) { + cmd := getRootCommand() + result := test.RunCmd(cmd, "merge -c=false examples/data1.yaml examples/data2.yaml") + if result.Error != nil { + t.Error(result.Error) + } + expectedOutput := `a: simple +b: [1, 2] c: test: 1 ` test.AssertResult(t, expectedOutput, result.Output) } -func xTestMergeOverwriteCmd(t *testing.T) { +func TestMergeOverwriteCmd(t *testing.T) { cmd := getRootCommand() - result := test.RunCmd(cmd, "merge --overwrite examples/data1.yaml examples/data2.yaml") + result := test.RunCmd(cmd, "merge -c=false --overwrite examples/data1.yaml examples/data2.yaml") if result.Error != nil { t.Error(result.Error) } expectedOutput := `a: other -b: -- 3 -- 4 +b: [3, 4] c: test: 1 ` diff --git a/examples/data1.yaml b/examples/data1.yaml index c9ad78b4..b0f017b1 100644 --- a/examples/data1.yaml +++ b/examples/data1.yaml @@ -1,2 +1,4 @@ a: simple b: [1, 2] +c: + test: 1 diff --git a/examples/sample2.yaml b/examples/sample2.yaml index e16e0b69..3df90f8b 100644 --- a/examples/sample2.yaml +++ b/examples/sample2.yaml @@ -1,10 +1,2 @@ -a: Easy! as one two three b: - c: things - d: whatever -things: - borg: snorg - thing1: - cat: 'fred' - thing2: - cat: 'sam' + c: things \ No newline at end of file diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 3f7737a6..dbf19183 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -12,10 +12,12 @@ import ( var log = logging.MustGetLogger("yq") +// TODO: enumerate type UpdateCommand struct { - Command string - Path string - Value *yaml.Node + Command string + Path string + Value *yaml.Node + Overwrite bool } func DebugNode(value *yaml.Node) { @@ -76,7 +78,7 @@ func guessKind(head string, tail []string, guess yaml.Kind) yaml.Kind { type YqLib interface { Get(rootNode *yaml.Node, path string) ([]*NodeContext, error) - Update(rootNode *yaml.Node, updateCommand UpdateCommand) error + Update(rootNode *yaml.Node, updateCommand UpdateCommand, autoCreate bool) error New(path string) yaml.Node } @@ -106,12 +108,12 @@ func (l *lib) New(path string) yaml.Node { return newNode } -func (l *lib) Update(rootNode *yaml.Node, updateCommand UpdateCommand) error { +func (l *lib) Update(rootNode *yaml.Node, updateCommand UpdateCommand, autoCreate bool) error { log.Debugf("%v to %v", updateCommand.Command, updateCommand.Path) switch updateCommand.Command { case "update": var paths = l.parser.ParsePath(updateCommand.Path) - navigator := NewDataNavigator(UpdateNavigationStrategy(updateCommand.Value)) + navigator := NewDataNavigator(UpdateNavigationStrategy(updateCommand, autoCreate)) return navigator.Traverse(rootNode, paths) case "delete": var paths = l.parser.ParsePath(updateCommand.Path) diff --git a/pkg/yqlib/update_navigation_strategy.go b/pkg/yqlib/update_navigation_strategy.go index 281a4033..8eea9069 100644 --- a/pkg/yqlib/update_navigation_strategy.go +++ b/pkg/yqlib/update_navigation_strategy.go @@ -1,32 +1,33 @@ package yqlib -import ( - yaml "gopkg.in/yaml.v3" -) - -func UpdateNavigationStrategy(changesToApply *yaml.Node) NavigationStrategy { +func UpdateNavigationStrategy(updateCommand UpdateCommand, autoCreate bool) NavigationStrategy { return &NavigationStrategyImpl{ visitedNodes: []*NodeContext{}, followAlias: func(nodeContext NodeContext) bool { return false }, autoCreateMap: func(nodeContext NodeContext) bool { - return true + return autoCreate }, visit: func(nodeContext NodeContext) error { node := nodeContext.Node - log.Debug("going to update") - DebugNode(node) - log.Debug("with") - DebugNode(changesToApply) - node.Value = changesToApply.Value - node.Tag = changesToApply.Tag - node.Kind = changesToApply.Kind - node.Style = changesToApply.Style - node.Content = changesToApply.Content - node.HeadComment = changesToApply.HeadComment - node.LineComment = changesToApply.LineComment - node.FootComment = changesToApply.FootComment + changesToApply := updateCommand.Value + if updateCommand.Overwrite == true || node.Value == "" { + log.Debug("going to update") + DebugNode(node) + log.Debug("with") + DebugNode(changesToApply) + node.Value = changesToApply.Value + node.Tag = changesToApply.Tag + node.Kind = changesToApply.Kind + node.Style = changesToApply.Style + node.Content = changesToApply.Content + node.HeadComment = changesToApply.HeadComment + node.LineComment = changesToApply.LineComment + node.FootComment = changesToApply.FootComment + } else { + log.Debug("skipping update as node already has value %v and overwriteFlag is ", node.Value, updateCommand.Overwrite) + } return nil }, } diff --git a/yq.go b/yq.go index 59bb74a2..22cace4c 100644 --- a/yq.go +++ b/yq.go @@ -22,6 +22,7 @@ var printMode = "v" var writeInplace = false var writeScript = "" var overwriteFlag = false +var autoCreateFlag = true var allowEmptyFlag = false var appendFlag = false var verbose = false @@ -235,7 +236,8 @@ Note that if you set both flags only overwrite will take effect. RunE: mergeProperties, } cmdMerge.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace") - // cmdMerge.PersistentFlags().BoolVarP(&overwriteFlag, "overwrite", "x", false, "update the yaml file by overwriting existing values") + cmdMerge.PersistentFlags().BoolVarP(&overwriteFlag, "overwrite", "x", false, "update the yaml file by overwriting existing values") + cmdMerge.PersistentFlags().BoolVarP(&autoCreateFlag, "autocreate", "c", true, "automatically create any missing entries") // cmdMerge.PersistentFlags().BoolVarP(&appendFlag, "append", "a", false, "update the yaml file by appending array values") // cmdMerge.PersistentFlags().BoolVarP(&allowEmptyFlag, "allow-empty", "e", false, "allow empty yaml files") cmdMerge.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)") @@ -256,10 +258,20 @@ func readProperty(cmd *cobra.Command, args []string) error { return errorParsingDocIndex } + matchingNodes, errorReadingStream := readYamlFile(args[0], path, updateAll, docIndexInt) + + if errorReadingStream != nil { + return errorReadingStream + } + + return printResults(matchingNodes, cmd) +} + +func readYamlFile(filename string, path string, updateAll bool, docIndexInt int) ([]*yqlib.NodeContext, error) { var matchingNodes []*yqlib.NodeContext var currentIndex = 0 - var errorReadingStream = readStream(args[0], func(decoder *yaml.Decoder) error { + var errorReadingStream = readStream(filename, func(decoder *yaml.Decoder) error { for { var dataBucket yaml.Node errorReading := decoder.Decode(&dataBucket) @@ -275,12 +287,7 @@ func readProperty(cmd *cobra.Command, args []string) error { currentIndex = currentIndex + 1 } }) - - if errorReadingStream != nil { - return errorReadingStream - } - - return printResults(matchingNodes, cmd) + return matchingNodes, errorReadingStream } func handleEOF(updateAll bool, docIndexInt int, currentIndex int) error { @@ -419,7 +426,21 @@ func writeProperty(cmd *cobra.Command, args []string) error { func mergeProperties(cmd *cobra.Command, args []string) error { // first generate update commands from the file - return nil + var filesToMerge = args[1:] + var updateCommands []yqlib.UpdateCommand = make([]yqlib.UpdateCommand, 0) + + for _, fileToMerge := range filesToMerge { + matchingNodes, errorProcessingFile := readYamlFile(fileToMerge, "**", false, 0) + if errorProcessingFile != nil { + return errorProcessingFile + } + for _, matchingNode := range matchingNodes { + mergePath := yqlib.PathStackToString(matchingNode.PathStack) + updateCommands = append(updateCommands, yqlib.UpdateCommand{Command: "update", Path: mergePath, Value: matchingNode.Node, Overwrite: overwriteFlag}) + } + } + + return updateDoc(args[0], updateCommands, cmd.OutOrStdout()) } func newProperty(cmd *cobra.Command, args []string) error { @@ -431,7 +452,7 @@ func newProperty(cmd *cobra.Command, args []string) error { for _, updateCommand := range updateCommands { - errorUpdating := lib.Update(&newNode, updateCommand) + errorUpdating := lib.Update(&newNode, updateCommand, true) if errorUpdating != nil { return errorUpdating @@ -474,7 +495,7 @@ func prefixDocument(updateAll bool, docIndexInt int, currentIndex int, dataBucke newNode := lib.New(updateCommand.Path) dataBucket.Content[0] = &newNode - errorUpdating := lib.Update(dataBucket, updateCommand) + errorUpdating := lib.Update(dataBucket, updateCommand, true) if errorUpdating != nil { return errorUpdating } @@ -502,7 +523,8 @@ func updateDoc(inputFile string, updateCommands []yqlib.UpdateCommand, writer io if updateAll || currentIndex == docIndexInt { log.Debugf("Updating doc %v", currentIndex) for _, updateCommand := range updateCommands { - errorUpdating := lib.Update(dataBucket, updateCommand) + log.Debugf("Processing update to Path %v", updateCommand.Path) + errorUpdating := lib.Update(dataBucket, updateCommand, autoCreateFlag) if errorUpdating != nil { return errorUpdating } @@ -605,7 +627,7 @@ func readUpdateCommands(args []string, expectedArgs int, badArgsMessage string) log.Debugf("Read write commands file '%v'", parsedCommands) for index := range parsedCommands { parsedCommand := parsedCommands[index] - updateCommand := yqlib.UpdateCommand{Command: parsedCommand.Command, Path: parsedCommand.Path, Value: &parsedCommand.Value} + updateCommand := yqlib.UpdateCommand{Command: parsedCommand.Command, Path: parsedCommand.Path, Value: &parsedCommand.Value, Overwrite: true} updateCommands = append(updateCommands, updateCommand) } @@ -617,7 +639,7 @@ func readUpdateCommands(args []string, expectedArgs int, badArgsMessage string) log.Debug("args %v", args) log.Debug("path %v", args[expectedArgs-2]) log.Debug("Value %v", args[expectedArgs-1]) - updateCommands[0] = yqlib.UpdateCommand{Command: "update", Path: args[expectedArgs-2], Value: valueParser.Parse(args[expectedArgs-1], customTag)} + updateCommands[0] = yqlib.UpdateCommand{Command: "update", Path: args[expectedArgs-2], Value: valueParser.Parse(args[expectedArgs-1], customTag), Overwrite: true} } return updateCommands, nil }