From 854f5f0fc9041fa618de9800ffbd0aead1f57a88 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 10 Jan 2020 22:01:59 +1100 Subject: [PATCH] wip json encoding --- Upgrade Notes | 9 ++--- commands_test.go | 2 +- examples/instruction_sample.yaml | 5 +-- examples/sample.yaml | 2 +- pkg/yqlib/encoder.go | 44 +++++++++++++++++++++ pkg/yqlib/value_parser_test.go | 2 +- yq.go | 65 +++++++++++++++++++++----------- 7 files changed, 94 insertions(+), 35 deletions(-) create mode 100644 pkg/yqlib/encoder.go diff --git a/Upgrade Notes b/Upgrade Notes index 1b7bb513..38f97368 100644 --- a/Upgrade Notes +++ b/Upgrade Notes @@ -1,15 +1,12 @@ -# Update doco / notes - - --autocreate=false to turn off default flags - - add comments to test yaml to ensure they are kept on updating. - - update built in command notes to includes quotes around path args. # New Features - Keeps comments and formatting (e.g. inline arrays)! - Handles anchors! - Can specify yaml tags (e.g. !!int), quoting values no longer sufficient, need to specify the tag value instead. + - JSON output works for all commands! Yaml files with multiple documents are printed out as one JSON document per line. -# Update scripts file format has changed +# Update scripts file format has changed to be more powerful. Comments can be added, and delete commands have been introduced. # Merge command -- autocreates missing entries in target by default, new flag to turn that off. +- New flag 'autocreates' missing entries in target by default, new flag to turn that off. diff --git a/commands_test.go b/commands_test.go index 9f600f38..bf0b438f 100644 --- a/commands_test.go +++ b/commands_test.go @@ -494,7 +494,7 @@ func TestReadCmd_Verbose(t *testing.T) { if result.Error != nil { t.Error(result.Error) } - test.AssertResult(t, "2", result.Output) + test.AssertResult(t, "2\n", result.Output) } // func TestReadCmd_ToJson(t *testing.T) { diff --git a/examples/instruction_sample.yaml b/examples/instruction_sample.yaml index 24445bca..d531a302 100644 --- a/examples/instruction_sample.yaml +++ b/examples/instruction_sample.yaml @@ -6,6 +6,5 @@ - command: update path: b.e[+].name value: Mike Farah -- command: update - path: d.a - value: Cow \ No newline at end of file +- command: delete + path: b.d \ No newline at end of file diff --git a/examples/sample.yaml b/examples/sample.yaml index a08d1d1f..1fb93352 100644 --- a/examples/sample.yaml +++ b/examples/sample.yaml @@ -1,4 +1,4 @@ -a: Easy! as one two three +a: true b: c: 2 d: [3, 4, 5] diff --git a/pkg/yqlib/encoder.go b/pkg/yqlib/encoder.go new file mode 100644 index 00000000..affd213e --- /dev/null +++ b/pkg/yqlib/encoder.go @@ -0,0 +1,44 @@ +package yqlib + +import ( + "encoding/json" + "io" + + yaml "gopkg.in/yaml.v3" +) + +type Encoder interface { + Encode(node *yaml.Node) error +} + +type yamlEncoder struct { + encoder *yaml.Encoder +} + +func NewYamlEncoder(destination io.Writer) Encoder { + var encoder = yaml.NewEncoder(destination) + encoder.SetIndent(2) + return &yamlEncoder{encoder} +} + +func (ye *yamlEncoder) Encode(node *yaml.Node) error { + return ye.encoder.Encode(node) +} + +type jsonEncoder struct { + encoder *json.Encoder +} + +func NewJsonEncoder(destination io.Writer) Encoder { + var encoder = json.NewEncoder(destination) + return &jsonEncoder{encoder} +} + +func (je *jsonEncoder) Encode(node *yaml.Node) error { + var dataBucket interface{} + errorDecoding := node.Decode(&dataBucket) + if errorDecoding != nil { + return errorDecoding + } + return je.encoder.Encode(dataBucket) +} diff --git a/pkg/yqlib/value_parser_test.go b/pkg/yqlib/value_parser_test.go index e4930974..fb439d69 100644 --- a/pkg/yqlib/value_parser_test.go +++ b/pkg/yqlib/value_parser_test.go @@ -14,7 +14,7 @@ var parseValueTests = []struct { testDescription string }{ {"true", "", "!!bool", "boolean"}, - {"true", "!!string", "!!string", "boolean forced as string"}, + {"true", "!!str", "!!str", "boolean forced as string"}, {"3.4", "", "!!float", "float"}, {"1212121", "", "!!int", "big number"}, {"1212121.1", "", "!!float", "big float number"}, diff --git a/yq.go b/yq.go index c36ba59d..5df65ffe 100644 --- a/yq.go +++ b/yq.go @@ -22,6 +22,7 @@ var customTag = "" var printMode = "v" var writeInplace = false var writeScript = "" +var outputToJSON = false var overwriteFlag = false var autoCreateFlag = true var allowEmptyFlag = false @@ -73,6 +74,7 @@ func newCommandCLI() *cobra.Command { } rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "verbose mode") + rootCmd.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "output as json") rootCmd.Flags().BoolVarP(&version, "version", "V", false, "Print version information and quit") rootCmd.AddCommand( @@ -97,9 +99,9 @@ func createReadCmd() *cobra.Command { yq read things.yaml a.b.c yq r - a.b.c (reads from stdin) yq r things.yaml a.*.c -yq r -d1 things.yaml a.array[0].blah -yq r things.yaml a.array[*].blah -yq r -- things.yaml --key-starting-with-dashes +yq r -d1 things.yaml 'a.array[0].blah' +yq r things.yaml 'a.array[*].blah' +yq r -- things.yaml --key-starting-with-dashes.blah `, Long: "Outputs the value of the given path in the yaml file to STDOUT", RunE: readProperty, @@ -115,13 +117,15 @@ func createWriteCmd() *cobra.Command { Aliases: []string{"w"}, Short: "yq w [--inplace/-i] [--script/-s script_file] [--doc/-d index] sample.yaml a.b.c newValue", Example: ` -yq write things.yaml a.b.c cat +yq write things.yaml a.b.c true +yq write things.yaml a.b.c --tag '!!str' true +yq write things.yaml a.b.c --tag '!!float' 3 yq write --inplace -- things.yaml a.b.c --cat yq w -i things.yaml a.b.c cat yq w --script update_script.yaml things.yaml yq w -i -s update_script.yaml things.yaml -yq w --doc 2 things.yaml a.b.d[+] foo -yq w -d2 things.yaml a.b.d[+] foo +yq w --doc 2 things.yaml 'a.b.d[+]' foo +yq w -d2 things.yaml 'a.b.d[+]' foo `, Long: `Updates the yaml file w.r.t the given path and value. Outputs to STDOUT unless the inplace flag is used, in which case the file is updated instead. @@ -129,12 +133,16 @@ Outputs to STDOUT unless the inplace flag is used, in which case the file is upd Append value to array adds the value to the end of array. Update Scripts: -Note that you can give an update script to perform more sophisticated updated. Update script -format is a yaml map where the key is the path and the value is..well the value. e.g.: +Note that you can give an update script to perform more sophisticated update. Update script +format is list of update commands (update or delete) like so: --- -a.b.c: true, -a.b.e: - - name: bob +- command: update + path: b.c + value: + #great + things: frog # wow! +- command: delete + path: b.d `, RunE: writeProperty, } @@ -198,6 +206,7 @@ func createNewCmd() *cobra.Command { Example: ` yq new a.b.c cat yq n a.b.c cat +yq n a.b[+] --tag '!!str' true yq n -- --key-starting-with-dash cat yq n --script create_script.yaml `, @@ -226,6 +235,7 @@ yq m -i things.yaml other.yaml yq m --overwrite things.yaml other.yaml yq m -i -x things.yaml other.yaml yq m -i -a things.yaml other.yaml +yq m -i --autocreate=false things.yaml other.yaml `, Long: `Updates the yaml file by adding/updating the path(s) and value(s) from additional yaml file(s). Outputs to STDOUT unless the inplace flag is used, in which case the file is updated instead. @@ -313,16 +323,18 @@ func appendDocument(originalMatchingNodes []*yqlib.NodeContext, dataBucket yaml. } func printValue(node *yaml.Node, cmd *cobra.Command) error { - if node.Kind == yaml.ScalarNode { - cmd.Print(node.Value) - return nil + bufferedWriter := bufio.NewWriter(cmd.OutOrStdout()) + defer safelyFlush(bufferedWriter) + + var encoder yqlib.Encoder + if outputToJSON { + encoder = yqlib.NewJsonEncoder(bufferedWriter) + } else { + encoder = yqlib.NewYamlEncoder(bufferedWriter) } - var encoder = yaml.NewEncoder(cmd.OutOrStdout()) - encoder.SetIndent(2) if err := encoder.Encode(node); err != nil { return err } - encoder.Close() return nil } @@ -376,7 +388,7 @@ func parseDocumentIndex() (bool, int, error) { type updateDataFn func(dataBucket *yaml.Node, currentIndex int) error -func mapYamlDecoder(updateData updateDataFn, encoder *yaml.Encoder) yamlDecoderFn { +func mapYamlDecoder(updateData updateDataFn, encoder yqlib.Encoder) yamlDecoderFn { return func(decoder *yaml.Decoder) error { var dataBucket yaml.Node var errorReading error @@ -561,14 +573,21 @@ func readAndUpdate(stdOut io.Writer, inputFile string, updateData updateDataFn) safelyRenameFile(tempFile.Name(), inputFile) }() } else { - var writer = bufio.NewWriter(stdOut) - destination = writer + destination = stdOut destinationName = "Stdout" - defer safelyFlush(writer) } - var encoder = yaml.NewEncoder(destination) - encoder.SetIndent(2) + log.Debugf("Writing to %v from %v", destinationName, inputFile) + + bufferedWriter := bufio.NewWriter(destination) + defer safelyFlush(bufferedWriter) + + var encoder yqlib.Encoder + if outputToJSON { + encoder = yqlib.NewJsonEncoder(bufferedWriter) + } else { + encoder = yqlib.NewYamlEncoder(bufferedWriter) + } return readStream(inputFile, mapYamlDecoder(updateData, encoder)) }