From 6980be3800138d397d60c3e861171afc875c42d2 Mon Sep 17 00:00:00 2001 From: kenjones Date: Fri, 22 Sep 2017 23:29:37 -0400 Subject: [PATCH] Feature: Adds merge command Adds merge command for merging multiple yaml files together. Resolves: #31 --- README.md | 14 +++++---- commands_test.go | 71 +++++++++++++++++++++++++++++++++++++++++++++ data_navigator.go | 36 +++++++++++++++++++++++ examples/data1.yaml | 2 ++ examples/data2.yaml | 3 ++ examples/data3.yaml | 5 ++++ merge.go | 12 ++++++++ merge_test.go | 30 +++++++++++++++++++ scripts/vendor.sh | 1 + vendor/vendor.json | 6 ++++ yaml.go | 55 ++++++++++++++++++++++++++++++++++- 11 files changed, 229 insertions(+), 6 deletions(-) create mode 100644 examples/data1.yaml create mode 100644 examples/data2.yaml create mode 100644 examples/data3.yaml create mode 100644 merge.go create mode 100644 merge_test.go diff --git a/README.md b/README.md index 43a7a6a8..b45c79db 100644 --- a/README.md +++ b/README.md @@ -20,6 +20,8 @@ go get github.com/mikefarah/yaml - Convert from json to yaml - Convert from yaml to json - Pipe data in by using '-' +- Merge multiple yaml files where each additional file sets values for missing or null value keys. +- Merge multiple yaml files with overwrite to support overriding previous values. ## [Usage](http://mikefarah.github.io/yaml/) @@ -30,15 +32,17 @@ Usage: yaml [command] Available Commands: + help Help about any command + merge yaml m [--inplace/-i] [--overwrite/-x] sample.yaml sample2.yaml + new yaml n [--script/-s script_file] a.b.c newValueForC read yaml r sample.yaml a.b.c write yaml w [--inplace/-i] [--script/-s script_file] sample.yaml a.b.c newValueForC - new yaml n [--script/-s script_file] a.b.c newValueForC Flags: - -h, --help[=false]: help for yaml - -j, --tojson[=false]: output as json - -t, --trim[=true]: trim yaml output - -v, --verbose[=false]: verbose mode + -h, --help help for yaml + -j, --tojson output as json + -t, --trim trim yaml output (default true) + -v, --verbose verbose mode Use "yaml [command] --help" for more information about a command. ``` diff --git a/commands_test.go b/commands_test.go index 6834a4ce..028bf1f1 100644 --- a/commands_test.go +++ b/commands_test.go @@ -303,3 +303,74 @@ func TestWriteCmd_Inplace(t *testing.T) { c: 7` assertResult(t, expectedOutput, gotOutput) } + +func TestMergeCmd(t *testing.T) { + cmd := getRootCommand() + result := runCmd(cmd, "merge examples/data1.yaml examples/data2.yaml") + if result.Error != nil { + t.Error(result.Error) + } + expectedOutput := `a: simple +b: +- 1 +- 2 +c: + test: 1 +` + assertResult(t, expectedOutput, result.Output) +} + +func TestMergeCmd_Error(t *testing.T) { + cmd := getRootCommand() + result := runCmd(cmd, "merge examples/data1.yaml") + if result.Error == nil { + t.Error("Expected command to fail due to missing arg") + } + expectedOutput := `Must provide at least 2 yaml files` + assertResult(t, expectedOutput, result.Error.Error()) +} + +func TestMergeCmd_ErrorUnreadableFile(t *testing.T) { + cmd := getRootCommand() + result := runCmd(cmd, "merge examples/data1.yaml fake-unknown") + if result.Error == nil { + t.Error("Expected command to fail due to unknown file") + } + expectedOutput := `open fake-unknown: no such file or directory` + assertResult(t, expectedOutput, result.Error.Error()) +} + +func TestMergeCmd_Verbose(t *testing.T) { + cmd := getRootCommand() + result := runCmd(cmd, "-v merge examples/data1.yaml examples/data2.yaml") + if result.Error != nil { + t.Error(result.Error) + } + expectedOutput := `a: simple +b: +- 1 +- 2 +c: + test: 1 +` + assertResult(t, expectedOutput, result.Output) +} + +func TestMergeCmd_Inplace(t *testing.T) { + filename := writeTempYamlFile(readTempYamlFile("examples/data1.yaml")) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("merge -i %s examples/data2.yaml", filename)) + if result.Error != nil { + t.Error(result.Error) + } + gotOutput := readTempYamlFile(filename) + expectedOutput := `a: simple +b: +- 1 +- 2 +c: + test: 1` + assertResult(t, expectedOutput, gotOutput) +} diff --git a/data_navigator.go b/data_navigator.go index 0fc8aa68..274071fc 100644 --- a/data_navigator.go +++ b/data_navigator.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "sort" "strconv" "gopkg.in/yaml.v2" @@ -175,3 +176,38 @@ func calculateValue(value interface{}, tail []string) (interface{}, error) { } return value, nil } + +func mapToMapSlice(data map[interface{}]interface{}) yaml.MapSlice { + var mapSlice yaml.MapSlice + + for k, v := range data { + if mv, ok := v.(map[interface{}]interface{}); ok { + v = mapToMapSlice(mv) + } + item := yaml.MapItem{Key: k, Value: v} + mapSlice = append(mapSlice, item) + } + + // because the parsing of the yaml was done via a map the order will be inconsistent + // apply order to allow a consistent output + // Only available in Go 1.8+ + // sort.SliceStable(mapSlice, func(i, j int) bool { return mapSlice[i].Key < mapSlice[j].Key }) + sort.Sort(sortMap{mapSlice}) + return mapSlice +} + +type sortMap struct { + ms yaml.MapSlice +} + +func (m sortMap) Len() int { + return len(m.ms) +} + +func (m sortMap) Less(i, j int) bool { + return m.ms[i].Key.(string) < m.ms[j].Key.(string) +} + +func (m sortMap) Swap(i, j int) { + m.ms[i], m.ms[j] = m.ms[j], m.ms[i] +} diff --git a/examples/data1.yaml b/examples/data1.yaml new file mode 100644 index 00000000..c9ad78b4 --- /dev/null +++ b/examples/data1.yaml @@ -0,0 +1,2 @@ +a: simple +b: [1, 2] diff --git a/examples/data2.yaml b/examples/data2.yaml new file mode 100644 index 00000000..a4312f10 --- /dev/null +++ b/examples/data2.yaml @@ -0,0 +1,3 @@ +a: other +c: + test: 1 diff --git a/examples/data3.yaml b/examples/data3.yaml new file mode 100644 index 00000000..655191ab --- /dev/null +++ b/examples/data3.yaml @@ -0,0 +1,5 @@ +b: [2, 3, 4] +c: + test: 2 + other: true +d: false diff --git a/merge.go b/merge.go new file mode 100644 index 00000000..b9c4c18e --- /dev/null +++ b/merge.go @@ -0,0 +1,12 @@ +package main + +import ( + "github.com/imdario/mergo" +) + +func merge(dst, src interface{}, overwrite bool) error { + if overwrite { + return mergo.MergeWithOverwrite(dst, src) + } + return mergo.Merge(dst, src) +} diff --git a/merge_test.go b/merge_test.go new file mode 100644 index 00000000..4509a8aa --- /dev/null +++ b/merge_test.go @@ -0,0 +1,30 @@ +package main + +import ( + "testing" + + yaml "gopkg.in/yaml.v2" +) + +func TestMerge(t *testing.T) { + result, _ := mergeYaml([]string{"examples/data1.yaml", "examples/data2.yaml", "examples/data3.yaml"}) + expected := yaml.MapSlice{ + yaml.MapItem{Key: "a", Value: "simple"}, + yaml.MapItem{Key: "b", Value: []interface{}{1, 2}}, + yaml.MapItem{Key: "c", Value: yaml.MapSlice{yaml.MapItem{Key: "other", Value: true}, yaml.MapItem{Key: "test", Value: 1}}}, + yaml.MapItem{Key: "d", Value: false}, + } + assertResultComplex(t, expected, result) +} + +func TestMergeWithOverwrite(t *testing.T) { + overwriteFlag = true + result, _ := mergeYaml([]string{"examples/data1.yaml", "examples/data2.yaml", "examples/data3.yaml"}) + expected := yaml.MapSlice{ + yaml.MapItem{Key: "a", Value: "other"}, + yaml.MapItem{Key: "b", Value: []interface{}{2, 3, 4}}, + yaml.MapItem{Key: "c", Value: yaml.MapSlice{yaml.MapItem{Key: "other", Value: true}, yaml.MapItem{Key: "test", Value: 2}}}, + yaml.MapItem{Key: "d", Value: false}, + } + assertResultComplex(t, expected, result) +} diff --git a/scripts/vendor.sh b/scripts/vendor.sh index dd78372d..e36e746c 100755 --- a/scripts/vendor.sh +++ b/scripts/vendor.sh @@ -5,4 +5,5 @@ set -e govendor fetch github.com/op/go-logging govendor fetch github.com/spf13/cobra govendor fetch gopkg.in/yaml.v2 +govendor fetch github.com/imdario/mergo govendor sync diff --git a/vendor/vendor.json b/vendor/vendor.json index 34cd295d..7c2a0a11 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -2,6 +2,12 @@ "comment": "", "ignore": "test", "package": [ + { + "checksumSHA1": "66lykxpWgSmQodnhkADqn6tnroQ=", + "path": "github.com/imdario/mergo", + "revision": "e3000cb3d28c72b837601cac94debd91032d19fe", + "revisionTime": "2017-06-20T10:47:01Z" + }, { "checksumSHA1": "40vJyUB4ezQSn/NSadsKEOrudMc=", "path": "github.com/inconshreveable/mousetrap", diff --git a/yaml.go b/yaml.go index 54fd9ee1..114edd6e 100644 --- a/yaml.go +++ b/yaml.go @@ -17,6 +17,7 @@ var trimOutput = true var writeInplace = false var writeScript = "" var outputToJSON = false +var overwriteFlag = false var verbose = false var log = logging.MustGetLogger("yaml") @@ -52,7 +53,7 @@ func newCommandCLI() *cobra.Command { rootCmd.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "output as json") rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "verbose mode") - rootCmd.AddCommand(createReadCmd(), createWriteCmd(), createNewCmd()) + rootCmd.AddCommand(createReadCmd(), createWriteCmd(), createNewCmd(), createMergeCmd()) rootCmd.SetOutput(os.Stdout) return rootCmd @@ -127,6 +128,30 @@ Note that you can give a create script to perform more sophisticated yaml. This return cmdNew } +func createMergeCmd() *cobra.Command { + var cmdMerge = &cobra.Command{ + Use: "merge [initial_yaml_file] [additional_yaml_file]...", + Aliases: []string{"m"}, + Short: "yaml m [--inplace/-i] [--overwrite/-x] sample.yaml sample2.yaml", + Example: ` +yaml merge things.yaml other.yaml +yaml merge --inplace things.yaml other.yaml +yaml m -i things.yaml other.yaml +yaml m --overwrite things.yaml other.yaml +yaml m -i -x things.yaml other.yaml + `, + Long: `Updates the yaml file by adding/updating the path(s) and value(s) from additional yaml file(s). +Outputs to STDOUT unless the inplace flag is used, in which case the file is updated instead. + +If overwrite flag is set then existing values will be overwritten using the values from each additional yaml file. +`, + RunE: mergeProperties, + } + cmdMerge.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace") + cmdMerge.PersistentFlags().BoolVarP(&overwriteFlag, "overwrite", "x", false, "update the yaml file by overwriting existing values") + return cmdMerge +} + func readProperty(cmd *cobra.Command, args []string) error { data, err := read(args) if err != nil { @@ -233,6 +258,34 @@ func write(cmd *cobra.Command, filename string, updatedData interface{}) error { return nil } +func mergeProperties(cmd *cobra.Command, args []string) error { + if len(args) < 2 { + return errors.New("Must provide at least 2 yaml files") + } + + updatedData, err := mergeYaml(args) + if err != nil { + return err + } + return write(cmd, args[0], updatedData) +} + +func mergeYaml(args []string) (interface{}, error) { + var updatedData map[interface{}]interface{} + + for _, f := range args { + var parsedData map[interface{}]interface{} + if err := readData(f, &parsedData); err != nil { + return nil, err + } + if err := merge(&updatedData, parsedData, overwriteFlag); err != nil { + return nil, err + } + } + + return mapToMapSlice(updatedData), nil +} + func updateParsedData(parsedData yaml.MapSlice, writeCommands yaml.MapSlice, prependCommand string) (interface{}, error) { var prefix = "" if prependCommand != "" {