From c2d9b98aeb6f5162db56e6f9d63f0c4cac228821 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 29 Oct 2021 20:49:06 +1100 Subject: [PATCH] wip split --- cmd/constant.go | 2 ++ cmd/evaluate_all_command.go | 23 +++++++++++++++++++- cmd/evalute_sequence_command.go | 2 +- cmd/root.go | 3 +++ go.mod | 5 ----- pkg/yqlib/operator_encoder_decoder.go | 3 ++- pkg/yqlib/printer.go | 16 ++++++++++---- pkg/yqlib/printer_test.go | 18 ++++++++-------- pkg/yqlib/printer_writer.go | 30 +++++++++++++++++++++++++-- 9 files changed, 79 insertions(+), 23 deletions(-) diff --git a/cmd/constant.go b/cmd/constant.go index 9b6369c9..0b921410 100644 --- a/cmd/constant.go +++ b/cmd/constant.go @@ -21,4 +21,6 @@ var prettyPrint = false // can be either "" (off), "extract" or "process" var frontMatter = "" +var splitFileExp = "" + var completedSuccessfully = false diff --git a/cmd/evaluate_all_command.go b/cmd/evaluate_all_command.go index bb3f04bf..92c46842 100644 --- a/cmd/evaluate_all_command.go +++ b/cmd/evaluate_all_command.go @@ -70,6 +70,10 @@ func evaluateAll(cmd *cobra.Command, args []string) error { return fmt.Errorf("Write inplace flag only applicable when giving an expression and at least one file") } + if writeInplace && splitFileExp != "" { + return fmt.Errorf("Write inplace cannot be used with split file") + } + if writeInplace { // only use colors if its forced colorsEnabled = forceColor @@ -96,7 +100,24 @@ func evaluateAll(cmd *cobra.Command, args []string) error { return err } - printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators) + var printerWriter yqlib.PrinterWriter + + if splitFileExp == "i" { + colorsEnabled = forceColor + printerWriter = yqlib.NewMultiPrinterWriter(nil, format) + + } else if splitFileExp != "" { + colorsEnabled = forceColor + splitExp, err := yqlib.NewExpressionParser().ParseExpression(splitFileExp) + if err != nil { + return nil + } + printerWriter = yqlib.NewMultiPrinterWriter(splitExp, format) + } else { + printerWriter = yqlib.NewSinglePrinterWriter(out) + } + + printer := yqlib.NewPrinter(printerWriter, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators) if frontMatter != "" { frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex]) diff --git a/cmd/evalute_sequence_command.go b/cmd/evalute_sequence_command.go index c5daea88..450337d2 100644 --- a/cmd/evalute_sequence_command.go +++ b/cmd/evalute_sequence_command.go @@ -106,7 +106,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) error { return err } - printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators) + printer := yqlib.NewPrinter(yqlib.NewSinglePrinterWriter(out), format, unwrapScalar, colorsEnabled, indent, !noDocSeparators) streamEvaluator := yqlib.NewStreamEvaluator() diff --git a/cmd/root.go b/cmd/root.go index e7b4a116..1a8fb613 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -63,6 +63,9 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`, rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors") rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact") rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression. This is a workaround for go-yaml to persist header content properly.") + + rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). Set to 'i' if you'd like to have numbered output.") + rootCmd.AddCommand( createEvaluateSequenceCommand(), createEvaluateAllCommand(), diff --git a/go.mod b/go.mod index 6f7e79f4..bc4fa628 100644 --- a/go.mod +++ b/go.mod @@ -13,13 +13,8 @@ require ( ) require ( - github.com/inconshreveable/mousetrap v1.0.0 // indirect - github.com/mattn/go-colorable v0.1.8 // indirect - github.com/mattn/go-isatty v0.0.12 // indirect - github.com/spf13/pflag v1.0.5 // indirect github.com/timtadh/data-structures v0.5.3 // indirect golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e // indirect - golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect ) go 1.17 diff --git a/pkg/yqlib/operator_encoder_decoder.go b/pkg/yqlib/operator_encoder_decoder.go index e20a5166..33887797 100644 --- a/pkg/yqlib/operator_encoder_decoder.go +++ b/pkg/yqlib/operator_encoder_decoder.go @@ -13,7 +13,8 @@ import ( func yamlToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) { var output bytes.Buffer log.Debug("printing with indent: %v", prefs.indent) - printer := NewPrinter(bufio.NewWriter(&output), prefs.format, true, false, prefs.indent, true) + + printer := NewPrinterWithSingleWriter(bufio.NewWriter(&output), prefs.format, true, false, prefs.indent, true) err := printer.PrintResults(candidate.AsList()) return output.String(), err } diff --git a/pkg/yqlib/printer.go b/pkg/yqlib/printer.go index 6685e68a..ac95e943 100644 --- a/pkg/yqlib/printer.go +++ b/pkg/yqlib/printer.go @@ -44,7 +44,7 @@ type resultsPrinter struct { colorsEnabled bool indent int printDocSeparators bool - printerWriter printerWriter + printerWriter PrinterWriter firstTimePrinting bool previousDocIndex uint previousFileIndex int @@ -53,7 +53,11 @@ type resultsPrinter struct { appendixReader io.Reader } -func NewPrinter(printerWriter printerWriter, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer { +func NewPrinterWithSingleWriter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer { + return NewPrinter(NewSinglePrinterWriter(writer), outputFormat, unwrapScalar, colorsEnabled, indent, printDocSeparators) +} + +func NewPrinter(printerWriter PrinterWriter, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer { return &resultsPrinter{ printerWriter: printerWriter, outputFormat: outputFormat, @@ -212,10 +216,14 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error { } if p.appendixReader != nil && p.outputFormat == YamlOutputFormat { - writer := p.printerWriter.GetWriter(nil, index) + writer, err := p.printerWriter.GetWriter(nil, index) + if err != nil { + return err + } + log.Debug("Piping appendix reader...") betterReader := bufio.NewReader(p.appendixReader) - _, err := io.Copy(writer, betterReader) + _, err = io.Copy(writer, betterReader) if err != nil { return err } diff --git a/pkg/yqlib/printer_test.go b/pkg/yqlib/printer_test.go index cda5270f..56acd5a8 100644 --- a/pkg/yqlib/printer_test.go +++ b/pkg/yqlib/printer_test.go @@ -36,7 +36,7 @@ func nodeToList(candidate *CandidateNode) *list.List { func TestPrinterMultipleDocsInSequence(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) - printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true) + printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0) if err != nil { @@ -74,7 +74,7 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) { func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) - printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true) + printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0) if err != nil { @@ -116,7 +116,7 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) { func TestPrinterMultipleFilesInSequence(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) - printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true) + printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0) if err != nil { @@ -163,7 +163,7 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) { func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) - printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true) + printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0) if err != nil { @@ -213,7 +213,7 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) { func TestPrinterMultipleDocsInSinglePrint(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) - printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true) + printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0) if err != nil { @@ -232,7 +232,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) { func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) - printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true) + printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0) if err != nil { @@ -261,7 +261,7 @@ a: coconut func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) - printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true) + printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0) if err != nil { @@ -287,7 +287,7 @@ a: coconut func TestPrinterScalarWithLeadingCont(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) - printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true) + printer := NewPrinterWithSingleWriter(writer, YamlOutputFormat, true, false, 2, true) node, err := NewExpressionParser().ParseExpression(".a") if err != nil { @@ -314,7 +314,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) { var writer = bufio.NewWriter(&output) // note printDocSeparators is true, it should still not print document separators // when outputing JSON. - printer := NewPrinter(writer, JsonOutputFormat, true, false, 0, true) + printer := NewPrinterWithSingleWriter(writer, JsonOutputFormat, true, false, 0, true) inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0) if err != nil { diff --git a/pkg/yqlib/printer_writer.go b/pkg/yqlib/printer_writer.go index 30d05f5b..d74baf52 100644 --- a/pkg/yqlib/printer_writer.go +++ b/pkg/yqlib/printer_writer.go @@ -4,9 +4,10 @@ import ( "bufio" "fmt" "io" + "os" ) -type printerWriter interface { +type PrinterWriter interface { GetWriter(node *CandidateNode, index int) (*bufio.Writer, error) } @@ -14,7 +15,7 @@ type singlePrinterWriter struct { bufferedWriter *bufio.Writer } -func NewSinglePrinterWriter(writer io.Writer) printerWriter { +func NewSinglePrinterWriter(writer io.Writer) PrinterWriter { return &singlePrinterWriter{ bufferedWriter: bufio.NewWriter(writer), } @@ -30,6 +31,23 @@ type multiPrintWriter struct { extension string } +func NewMultiPrinterWriter(expression *ExpressionNode, format PrinterOutputFormat) PrinterWriter { + extension := "yml" + + switch format { + case JsonOutputFormat: + extension = "json" + case PropsOutputFormat: + extension = "properties" + } + + return &multiPrintWriter{ + nameExpression: expression, + extension: extension, + treeNavigator: NewDataTreeNavigator(), + } +} + func (sp *multiPrintWriter) GetWriter(node *CandidateNode, index int) (*bufio.Writer, error) { name := "" @@ -49,4 +67,12 @@ func (sp *multiPrintWriter) GetWriter(node *CandidateNode, index int) (*bufio.Wr name = fmt.Sprintf("%v.%v", name, sp.extension) } + f, err := os.Create(name) + + if err != nil { + return nil, err + } + + return bufio.NewWriter(f), nil + }