From 86b9fe3ef91cd305194d6bd1bf45e4d5242f338c Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sun, 8 Jul 2018 21:47:01 +1000 Subject: [PATCH] Can read from all documents --- commands_test.go | 32 ++++++++++++-- examples/multiple_docs.yaml | 4 -- yq.go | 83 +++++++++++++++++++++++++------------ yq_test.go | 31 -------------- 4 files changed, 86 insertions(+), 64 deletions(-) diff --git a/commands_test.go b/commands_test.go index 45c23c5c..94e654a2 100644 --- a/commands_test.go +++ b/commands_test.go @@ -104,6 +104,19 @@ func TestReadCmd(t *testing.T) { assertResult(t, "2\n", result.Output) } +func TestReadOrderCmd(t *testing.T) { + cmd := getRootCommand() + result := runCmd(cmd, "read examples/order.yaml") + if result.Error != nil { + t.Error(result.Error) + } + assertResult(t, + `version: 3 +application: MyApp +`, + result.Output) +} + func TestReadMultiCmd(t *testing.T) { cmd := getRootCommand() result := runCmd(cmd, "read -d 1 examples/multiple_docs.yaml another.document") @@ -113,6 +126,19 @@ func TestReadMultiCmd(t *testing.T) { assertResult(t, "here\n", result.Output) } +func TestReadMultiAllCmd(t *testing.T) { + cmd := getRootCommand() + result := runCmd(cmd, "read -d* examples/multiple_docs.yaml commonKey") + if result.Error != nil { + t.Error(result.Error) + } + assertResult(t, + `- first document +- second document +- third document +`, result.Output) +} + func TestReadCmd_ArrayYaml(t *testing.T) { cmd := getRootCommand() result := runCmd(cmd, "read examples/array.yaml [0].gather_facts") @@ -192,7 +218,7 @@ func TestReadCmd_ArrayYaml_ErrorBadPath(t *testing.T) { if result.Error == nil { t.Error("Expected command to fail due to invalid path") } - expectedOutput := `Error accessing array: strconv.ParseInt: parsing "x": invalid syntax` + expectedOutput := `Error reading path in document index 0: Error accessing array: strconv.ParseInt: parsing "x": invalid syntax` assertResult(t, expectedOutput, result.Error.Error()) } @@ -202,7 +228,7 @@ func TestReadCmd_ArrayYaml_Splat_ErrorBadPath(t *testing.T) { if result.Error == nil { t.Error("Expected command to fail due to invalid path") } - expectedOutput := `Error accessing array: strconv.ParseInt: parsing "x": invalid syntax` + expectedOutput := `Error reading path in document index 0: Error accessing array: strconv.ParseInt: parsing "x": invalid syntax` assertResult(t, expectedOutput, result.Error.Error()) } @@ -254,7 +280,7 @@ func TestReadCmd_ErrorBadPath(t *testing.T) { if result.Error == nil { t.Fatal("Expected command to fail due to invalid path") } - expectedOutput := `Error accessing array: strconv.ParseInt: parsing "x": invalid syntax` + expectedOutput := `Error reading path in document index 0: Error accessing array: strconv.ParseInt: parsing "x": invalid syntax` assertResult(t, expectedOutput, result.Error.Error()) } diff --git a/examples/multiple_docs.yaml b/examples/multiple_docs.yaml index f7a0935f..cc94aadc 100644 --- a/examples/multiple_docs.yaml +++ b/examples/multiple_docs.yaml @@ -16,7 +16,3 @@ another: commonKey: third document wow: - here is another ---- -- 1 -- 2 -- 3 \ No newline at end of file diff --git a/yq.go b/yq.go index d431b9b2..cb6e96ef 100644 --- a/yq.go +++ b/yq.go @@ -97,7 +97,7 @@ yq r things.yaml a.array[*].blah Long: "Outputs the value of the given path in the yaml file to STDOUT", RunE: readProperty, } - cmdRead.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number, 0 based") + cmdRead.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)") cmdRead.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "output as json") return cmdRead } @@ -211,11 +211,57 @@ Note that if you set both flags only overwrite will take effect. } func readProperty(cmd *cobra.Command, args []string) error { - data, err := read(args) - if err != nil { - return err + var path = "" + + if len(args) < 1 { + return errors.New("Must provide filename") + } else if len(args) > 1 { + path = args[1] } - dataStr, err := toString(data) + + var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex() + if errorParsingDocIndex != nil { + return errorParsingDocIndex + } + var mappedDocs []interface{} + var dataBucket interface{} + var currentIndex = 0 + var errorReadingStream = readStream(args[0], func(decoder *yaml.Decoder) error { + for { + errorReading := decoder.Decode(&dataBucket) + if errorReading == io.EOF { + log.Debugf("done %v / %v", currentIndex, docIndexInt) + if !updateAll && currentIndex <= docIndexInt { + return fmt.Errorf("Asked to process document index %v but there are only %v document(s)", docIndex, currentIndex) + } + return nil + } + log.Debugf("processing %v / %v", currentIndex, docIndexInt) + if updateAll || currentIndex == docIndexInt { + log.Debugf("reading %v in %v", path, currentIndex) + mappedDoc, errorParsing := readPath(dataBucket, path) + log.Debugf("%v", mappedDoc) + if errorParsing != nil { + return errors.Wrapf(errorParsing, "Error reading path in document index %v", currentIndex) + } + mappedDocs = append(mappedDocs, mappedDoc) + log.Debugf("%v", mappedDocs) + } + currentIndex = currentIndex + 1 + } + }) + + if errorReadingStream != nil { + return errorReadingStream + } + + if !updateAll { + dataBucket = mappedDocs[0] + } else { + dataBucket = mappedDocs + } + + dataStr, err := toString(dataBucket) if err != nil { return err } @@ -223,28 +269,13 @@ func readProperty(cmd *cobra.Command, args []string) error { return nil } -func read(args []string) (interface{}, error) { - var path = "" - - if len(args) < 1 { - return nil, errors.New("Must provide filename") - } else if len(args) > 1 { - path = args[1] - } - var generalData interface{} - var docIndexInt, errorParsingDocumentIndex = strconv.ParseInt(docIndex, 10, 32) - if errorParsingDocumentIndex != nil { - return nil, errors.Wrapf(errorParsingDocumentIndex, "Document index %v is not a integer", docIndex) - } - if err := readData(args[0], int(docIndexInt), &generalData); err != nil { - return nil, err - } +func readPath(dataBucket interface{}, path string) (interface{}, error) { if path == "" { - return generalData, nil + log.Debug("no path") + return dataBucket, nil } - var paths = parsePath(path) - return recurse(generalData, paths[0], paths[1:]) + return recurse(dataBucket, paths[0], paths[1:]) } func newProperty(cmd *cobra.Command, args []string) error { @@ -316,8 +347,8 @@ func mapYamlDecoder(updateData updateDataFn, encoder *yaml.Encoder) yamlDecoderF errorReading = decoder.Decode(&dataBucket) if errorReading == io.EOF { - if !updateAll && currentIndex < docIndexInt { - return fmt.Errorf("Asked to process document %v but there are only %v document(s)", docIndex, currentIndex) + if !updateAll && currentIndex <= docIndexInt { + return fmt.Errorf("Asked to process document index %v but there are only %v document(s)", docIndex, currentIndex) } return nil } else if errorReading != nil { diff --git a/yq_test.go b/yq_test.go index dd9def61..72d672e1 100644 --- a/yq_test.go +++ b/yq_test.go @@ -23,37 +23,6 @@ func TestParseValue(t *testing.T) { } } -func TestRead(t *testing.T) { - result, _ := read([]string{"examples/sample.yaml", "b.c"}) - assertResult(t, 2, result) -} - -func TestReadMulti(t *testing.T) { - docIndex = "1" - result, _ := read([]string{"examples/multiple_docs.yaml", "another.document"}) - assertResult(t, "here", result) - docIndex = "0" -} - -func TestReadArray(t *testing.T) { - result, _ := read([]string{"examples/sample_array.yaml", "[1]"}) - assertResult(t, 2, result) -} - -func TestReadString(t *testing.T) { - result, _ := read([]string{"examples/sample_text.yaml"}) - assertResult(t, "hi", result) -} - -func TestOrder(t *testing.T) { - result, _ := read([]string{"examples/order.yaml"}) - formattedResult, _ := yamlToString(result) - assertResult(t, - `version: 3 -application: MyApp`, - formattedResult) -} - func TestMultilineString(t *testing.T) { testString := ` abcd