Can read from all documents

commands_test.go

@@ -104,6 +104,19 @@ func TestReadCmd(t *testing.T) {
 	assertResult(t, "2\n", result.Output)
 }
 
+func TestReadOrderCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := runCmd(cmd, "read examples/order.yaml")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	assertResult(t,
+		`version: 3
+application: MyApp
+`,
+		result.Output)
+}
+
 func TestReadMultiCmd(t *testing.T) {
 	cmd := getRootCommand()
 	result := runCmd(cmd, "read -d 1 examples/multiple_docs.yaml another.document")
@@ -113,6 +126,19 @@ func TestReadMultiCmd(t *testing.T) {
 	assertResult(t, "here\n", result.Output)
 }
 
+func TestReadMultiAllCmd(t *testing.T) {
+	cmd := getRootCommand()
+	result := runCmd(cmd, "read -d* examples/multiple_docs.yaml commonKey")
+	if result.Error != nil {
+		t.Error(result.Error)
+	}
+	assertResult(t,
+		`- first document
+- second document
+- third document
+`, result.Output)
+}
+
 func TestReadCmd_ArrayYaml(t *testing.T) {
 	cmd := getRootCommand()
 	result := runCmd(cmd, "read examples/array.yaml [0].gather_facts")
@@ -192,7 +218,7 @@ func TestReadCmd_ArrayYaml_ErrorBadPath(t *testing.T) {
 	if result.Error == nil {
 		t.Error("Expected command to fail due to invalid path")
 	}
-	expectedOutput := `Error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
+	expectedOutput := `Error reading path in document index 0: Error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
 	assertResult(t, expectedOutput, result.Error.Error())
 }
 
@@ -202,7 +228,7 @@ func TestReadCmd_ArrayYaml_Splat_ErrorBadPath(t *testing.T) {
 	if result.Error == nil {
 		t.Error("Expected command to fail due to invalid path")
 	}
-	expectedOutput := `Error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
+	expectedOutput := `Error reading path in document index 0: Error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
 	assertResult(t, expectedOutput, result.Error.Error())
 }
 
@@ -254,7 +280,7 @@ func TestReadCmd_ErrorBadPath(t *testing.T) {
 	if result.Error == nil {
 		t.Fatal("Expected command to fail due to invalid path")
 	}
-	expectedOutput := `Error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
+	expectedOutput := `Error reading path in document index 0: Error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
 	assertResult(t, expectedOutput, result.Error.Error())
 }
 

examples/multiple_docs.yaml

@@ -16,7 +16,3 @@ another:
 commonKey: third document
 wow: 
   - here is another
----
-- 1
-- 2
-- 3 

yq.go

@@ -97,7 +97,7 @@ yq r things.yaml a.array[*].blah
 		Long: "Outputs the value of the given path in the yaml file to STDOUT",
 		RunE: readProperty,
 	}
-	cmdRead.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number, 0 based")
+	cmdRead.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)")
 	cmdRead.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "output as json")
 	return cmdRead
 }
@@ -211,11 +211,57 @@ Note that if you set both flags only overwrite will take effect.
 }
 
 func readProperty(cmd *cobra.Command, args []string) error {
-	data, err := read(args)
-	if err != nil {
-		return err
+	var path = ""
+
+	if len(args) < 1 {
+		return errors.New("Must provide filename")
+	} else if len(args) > 1 {
+		path = args[1]
 	}
-	dataStr, err := toString(data)
+
+	var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex()
+	if errorParsingDocIndex != nil {
+		return errorParsingDocIndex
+	}
+	var mappedDocs []interface{}
+	var dataBucket interface{}
+	var currentIndex = 0
+	var errorReadingStream = readStream(args[0], func(decoder *yaml.Decoder) error {
+		for {
+			errorReading := decoder.Decode(&dataBucket)
+			if errorReading == io.EOF {
+				log.Debugf("done %v / %v", currentIndex, docIndexInt)
+				if !updateAll && currentIndex <= docIndexInt {
+					return fmt.Errorf("Asked to process document index %v but there are only %v document(s)", docIndex, currentIndex)
+				}
+				return nil
+			}
+			log.Debugf("processing %v / %v", currentIndex, docIndexInt)
+			if updateAll || currentIndex == docIndexInt {
+				log.Debugf("reading %v in %v", path, currentIndex)
+				mappedDoc, errorParsing := readPath(dataBucket, path)
+				log.Debugf("%v", mappedDoc)
+				if errorParsing != nil {
+					return errors.Wrapf(errorParsing, "Error reading path in document index %v", currentIndex)
+				}
+				mappedDocs = append(mappedDocs, mappedDoc)
+				log.Debugf("%v", mappedDocs)
+			}
+			currentIndex = currentIndex + 1
+		}
+	})
+
+	if errorReadingStream != nil {
+		return errorReadingStream
+	}
+
+	if !updateAll {
+		dataBucket = mappedDocs[0]
+	} else {
+		dataBucket = mappedDocs
+	}
+
+	dataStr, err := toString(dataBucket)
 	if err != nil {
 		return err
 	}
@@ -223,28 +269,13 @@ func readProperty(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
-func read(args []string) (interface{}, error) {
-	var path = ""
-
-	if len(args) < 1 {
-		return nil, errors.New("Must provide filename")
-	} else if len(args) > 1 {
-		path = args[1]
-	}
-	var generalData interface{}
-	var docIndexInt, errorParsingDocumentIndex = strconv.ParseInt(docIndex, 10, 32)
-	if errorParsingDocumentIndex != nil {
-		return nil, errors.Wrapf(errorParsingDocumentIndex, "Document index %v is not a integer", docIndex)
-	}
-	if err := readData(args[0], int(docIndexInt), &generalData); err != nil {
-		return nil, err
-	}
+func readPath(dataBucket interface{}, path string) (interface{}, error) {
 	if path == "" {
-		return generalData, nil
+		log.Debug("no path")
+		return dataBucket, nil
 	}
-
 	var paths = parsePath(path)
-	return recurse(generalData, paths[0], paths[1:])
+	return recurse(dataBucket, paths[0], paths[1:])
 }
 
 func newProperty(cmd *cobra.Command, args []string) error {
@@ -316,8 +347,8 @@ func mapYamlDecoder(updateData updateDataFn, encoder *yaml.Encoder) yamlDecoderF
 			errorReading = decoder.Decode(&dataBucket)
 
 			if errorReading == io.EOF {
-				if !updateAll && currentIndex < docIndexInt {
-					return fmt.Errorf("Asked to process document %v but there are only %v document(s)", docIndex, currentIndex)
+				if !updateAll && currentIndex <= docIndexInt {
+					return fmt.Errorf("Asked to process document index %v but there are only %v document(s)", docIndex, currentIndex)
 				}
 				return nil
 			} else if errorReading != nil {

yq_test.go

@@ -23,37 +23,6 @@ func TestParseValue(t *testing.T) {
 	}
 }
 
-func TestRead(t *testing.T) {
-	result, _ := read([]string{"examples/sample.yaml", "b.c"})
-	assertResult(t, 2, result)
-}
-
-func TestReadMulti(t *testing.T) {
-	docIndex = "1"
-	result, _ := read([]string{"examples/multiple_docs.yaml", "another.document"})
-	assertResult(t, "here", result)
-	docIndex = "0"
-}
-
-func TestReadArray(t *testing.T) {
-	result, _ := read([]string{"examples/sample_array.yaml", "[1]"})
-	assertResult(t, 2, result)
-}
-
-func TestReadString(t *testing.T) {
-	result, _ := read([]string{"examples/sample_text.yaml"})
-	assertResult(t, "hi", result)
-}
-
-func TestOrder(t *testing.T) {
-	result, _ := read([]string{"examples/order.yaml"})
-	formattedResult, _ := yamlToString(result)
-	assertResult(t,
-		`version: 3
-application: MyApp`,
-		formattedResult)
-}
-
 func TestMultilineString(t *testing.T) {
 	testString := `
 	abcd