wip split

cmd/constant.go

@@ -21,4 +21,6 @@ var prettyPrint = false
 // can be either "" (off), "extract" or "process"
 var frontMatter = ""
 
+var splitFileExp = ""
+
 var completedSuccessfully = false

cmd/evaluate_all_command.go

@@ -70,6 +70,10 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("Write inplace flag only applicable when giving an expression and at least one file")
 	}
 
+	if writeInplace && splitFileExp != "" {
+		return fmt.Errorf("Write inplace cannot be used with split file")
+	}
+
 	if writeInplace {
 		// only use colors if its forced
 		colorsEnabled = forceColor
@@ -96,7 +100,24 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+	var printerWriter yqlib.PrinterWriter
+
+	if splitFileExp == "i" {
+		colorsEnabled = forceColor
+		printerWriter = yqlib.NewMultiPrinterWriter(nil, format)
+
+	} else if splitFileExp != "" {
+		colorsEnabled = forceColor
+		splitExp, err := yqlib.NewExpressionParser().ParseExpression(splitFileExp)
+		if err != nil {
+			return nil
+		}
+		printerWriter = yqlib.NewMultiPrinterWriter(splitExp, format)
+	} else {
+		printerWriter = yqlib.NewSinglePrinterWriter(out)
+	}
+
+	printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
 
 	if frontMatter != "" {
 		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])

cmd/evalute_sequence_command.go

@@ -106,7 +106,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+	printer := yqlib.NewPrinter(yqlib.NewSinglePrinterWriter(out), format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
 
 	streamEvaluator := yqlib.NewStreamEvaluator()
 

cmd/root.go

@@ -63,6 +63,9 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
 	rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
 	rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression. This is a workaround for go-yaml to persist header content properly.")
+
+	rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). Set to 'i' if you'd like to have numbered output.")
+
 	rootCmd.AddCommand(
 		createEvaluateSequenceCommand(),
 		createEvaluateAllCommand(),

go.mod

@@ -13,13 +13,8 @@ require (
 )
 
 require (
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/mattn/go-colorable v0.1.8 // indirect
-	github.com/mattn/go-isatty v0.0.12 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/timtadh/data-structures v0.5.3 // indirect
 	golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 )
 
 go 1.17

pkg/yqlib/operator_encoder_decoder.go

@@ -13,7 +13,8 @@ import (
 func yamlToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) {
 	var output bytes.Buffer
 	log.Debug("printing with indent: %v", prefs.indent)
-	printer := NewPrinter(bufio.NewWriter(&output), prefs.format, true, false, prefs.indent, true)
+
+	printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), prefs.format, true, false, prefs.indent, true)
 	err := printer.PrintResults(candidate.AsList())
 	return output.String(), err
 }

pkg/yqlib/printer.go

@@ -44,7 +44,7 @@ type resultsPrinter struct {
 	colorsEnabled      bool
 	indent             int
 	printDocSeparators bool
-	printerWriter      printerWriter
+	printerWriter      PrinterWriter
 	firstTimePrinting  bool
 	previousDocIndex   uint
 	previousFileIndex  int
@@ -53,7 +53,11 @@ type resultsPrinter struct {
 	appendixReader     io.Reader
 }
 
-func NewPrinter(printerWriter printerWriter, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
+func NewPrinterWithSingleWriter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
+	return NewPrinter(NewSinglePrinterWriter(writer), outputFormat, unwrapScalar, colorsEnabled, indent, printDocSeparators)
+}
+
+func NewPrinter(printerWriter PrinterWriter, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
 	return &resultsPrinter{
 		printerWriter:      printerWriter,
 		outputFormat:       outputFormat,
@@ -212,10 +216,14 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 	}
 
 	if p.appendixReader != nil && p.outputFormat == YamlOutputFormat {
-		writer := p.printerWriter.GetWriter(nil, index)
+		writer, err := p.printerWriter.GetWriter(nil, index)
+		if err != nil {
+			return err
+		}
+
 		log.Debug("Piping appendix reader...")
 		betterReader := bufio.NewReader(p.appendixReader)
-		_, err := io.Copy(writer, betterReader)
+		_, err = io.Copy(writer, betterReader)
 		if err != nil {
 			return err
 		}

pkg/yqlib/printer_test.go

@@ -36,7 +36,7 @@ func nodeToList(candidate *CandidateNode) *list.List {
 func TestPrinterMultipleDocsInSequence(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -74,7 +74,7 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) {
 func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -116,7 +116,7 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 func TestPrinterMultipleFilesInSequence(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -163,7 +163,7 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) {
 func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -213,7 +213,7 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -232,7 +232,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -261,7 +261,7 @@ a: coconut
 func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -287,7 +287,7 @@ a: coconut
 func TestPrinterScalarWithLeadingCont(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+	printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(".a")
 	if err != nil {
@@ -314,7 +314,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	// note printDocSeparators is true, it should still not print document separators
 	// when outputing JSON.
-	printer := NewPrinter(writer, JsonOutputFormat, true, false, 0, true)
+	printer := NewPrinterWithSingleWriter(writer, JsonOutputFormat, true, false, 0, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {

pkg/yqlib/printer_writer.go

@@ -4,9 +4,10 @@ import (
 	"bufio"
 	"fmt"
 	"io"
+	"os"
 )
 
-type printerWriter interface {
+type PrinterWriter interface {
 	GetWriter(node *CandidateNode, index int) (*bufio.Writer, error)
 }
 
@@ -14,7 +15,7 @@ type singlePrinterWriter struct {
 	bufferedWriter *bufio.Writer
 }
 
-func NewSinglePrinterWriter(writer io.Writer) printerWriter {
+func NewSinglePrinterWriter(writer io.Writer) PrinterWriter {
 	return &singlePrinterWriter{
 		bufferedWriter: bufio.NewWriter(writer),
 	}
@@ -30,6 +31,23 @@ type multiPrintWriter struct {
 	extension      string
 }
 
+func NewMultiPrinterWriter(expression *ExpressionNode, format PrinterOutputFormat) PrinterWriter {
+	extension := "yml"
+
+	switch format {
+	case JsonOutputFormat:
+		extension = "json"
+	case PropsOutputFormat:
+		extension = "properties"
+	}
+
+	return &multiPrintWriter{
+		nameExpression: expression,
+		extension:      extension,
+		treeNavigator:  NewDataTreeNavigator(),
+	}
+}
+
 func (sp *multiPrintWriter) GetWriter(node *CandidateNode, index int) (*bufio.Writer, error) {
 	name := ""
 
@@ -49,4 +67,12 @@ func (sp *multiPrintWriter) GetWriter(node *CandidateNode, index int) (*bufio.Wr
 		name = fmt.Sprintf("%v.%v", name, sp.extension)
 	}
 
+	f, err := os.Create(name)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return bufio.NewWriter(f), nil
+
 }