From 4e628327c4f79237d68187ee454491bc3febfadc Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Tue, 20 Jul 2021 10:19:55 +1000 Subject: [PATCH] Better way of processing leading content --- acceptance_tests/leading-seperator.sh | 5 +-- cmd/root.go | 2 +- examples/data1.yaml | 6 +-- pkg/yqlib/all_at_once_evaluator.go | 6 ++- pkg/yqlib/operators_test.go | 2 +- pkg/yqlib/printer.go | 55 ++++++++++++++++++--------- pkg/yqlib/printer_test.go | 44 ++++++++++++--------- pkg/yqlib/stream_evaluator.go | 14 +++---- pkg/yqlib/utils.go | 20 +++++++--- 9 files changed, 92 insertions(+), 62 deletions(-) diff --git a/acceptance_tests/leading-seperator.sh b/acceptance_tests/leading-seperator.sh index 7a4a0156..3fc97c58 100755 --- a/acceptance_tests/leading-seperator.sh +++ b/acceptance_tests/leading-seperator.sh @@ -248,8 +248,7 @@ a: test b: sane EOM - - X=$(./yq e --header-preprocess=false '... comments=""' test.yml) + X=$(./yq e '... comments=""' test.yml) assertEquals "$expected" "$X" } @@ -265,9 +264,7 @@ a: test b: sane EOL - # it will be hard to remove that top level separator read -r -d '' expected << EOM ---- # hi peeps # cool a: test diff --git a/cmd/root.go b/cmd/root.go index d34f2869..c080745f 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -55,7 +55,7 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`, rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors") rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors") rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact") - rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and seperators before processing expression. This is a workaround for go-yaml to persist header content. You will want this off if you want to remove leading comments.") + rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and seperators before processing expression. This is a workaround for go-yaml to persist header content. This flag will be removed once this feature has been out in the wild for a while.") rootCmd.AddCommand( createEvaluateSequenceCommand(), createEvaluateAllCommand(), diff --git a/examples/data1.yaml b/examples/data1.yaml index 62b7544c..0d0503c0 100644 --- a/examples/data1.yaml +++ b/examples/data1.yaml @@ -1,8 +1,4 @@ --- # hi peeps # cool -a: test ---- -# this is another doc -# great -b: sane \ No newline at end of file +a: test \ No newline at end of file diff --git a/pkg/yqlib/all_at_once_evaluator.go b/pkg/yqlib/all_at_once_evaluator.go index 888d8614..2db29aae 100644 --- a/pkg/yqlib/all_at_once_evaluator.go +++ b/pkg/yqlib/all_at_once_evaluator.go @@ -73,15 +73,17 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string candidateNode := &CandidateNode{ Document: 0, Filename: "", - Node: &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode}, + Node: &yaml.Node{Kind: yaml.DocumentNode, HeadComment: firstFileLeadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}}, FileIndex: 0, } allDocuments.PushBack(candidateNode) + } else { + allDocuments.Front().Value.(*CandidateNode).Node.HeadComment = firstFileLeadingContent } matches, err := e.EvaluateCandidateNodes(expression, allDocuments) if err != nil { return err } - return printer.PrintResults(matches, firstFileLeadingContent) + return printer.PrintResults(matches) } diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index 4299fba9..c9201b26 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -256,7 +256,7 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt t.Error(err, s.expression) } - err = printer.PrintResults(context.MatchingNodes, "") + err = printer.PrintResults(context.MatchingNodes) if err != nil { t.Error(err, s.expression) } diff --git a/pkg/yqlib/printer.go b/pkg/yqlib/printer.go index bfc2da7c..153448d0 100644 --- a/pkg/yqlib/printer.go +++ b/pkg/yqlib/printer.go @@ -10,7 +10,7 @@ import ( ) type Printer interface { - PrintResults(matchingNodes *list.List, leadingContent string) error + PrintResults(matchingNodes *list.List) error PrintedAnything() bool //e.g. when given a front-matter doc, like jekyll SetAppendix(reader io.Reader) @@ -85,7 +85,7 @@ func (p *resultsPrinter) safelyFlush(writer *bufio.Writer) { } } -func (p *resultsPrinter) PrintResults(matchingNodes *list.List, leadingContent string) error { +func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error { log.Debug("PrintResults for %v matches", matchingNodes.Len()) if p.outputToJSON { explodeOp := Operation{OperationType: explodeOpType} @@ -101,9 +101,6 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List, leadingContent s defer p.safelyFlush(bufferedWriter) if matchingNodes.Len() == 0 { - if err := p.writeString(bufferedWriter, leadingContent); err != nil { - return err - } log.Debug("no matching results, nothing to print") return nil } @@ -114,32 +111,52 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List, leadingContent s p.firstTimePrinting = false } - printedLead := false - for el := matchingNodes.Front(); el != nil; el = el.Next() { mappedDoc := el.Value.(*CandidateNode) log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v, printDocSeparators: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document, p.printDocSeparators) - if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators && - (printedLead || !strings.HasPrefix(leadingContent, "---")) { + + commentStartsWithSeparator := strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$\n$yqDocSeperator$") + + if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators && !commentStartsWithSeparator { log.Debug("-- writing doc sep") if err := p.writeString(bufferedWriter, "---\n"); err != nil { return err } } - if !printedLead { - // dont print leading comments and seperator if: - // - we are print json; or - // - we are printing an unwrapped scalar node - if !p.outputToJSON && (mappedDoc.Node.Kind != yaml.ScalarNode || !p.unwrapScalar) { - // we want to print this after the seperator logic - if err := p.writeString(bufferedWriter, leadingContent); err != nil { - return err + if strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$") { + log.Debug("headcommentwas %v", mappedDoc.Node.HeadComment) + log.Debug("finished headcomment") + reader := bufio.NewReader(strings.NewReader(mappedDoc.Node.HeadComment)) + mappedDoc.Node.HeadComment = "" + + for { + + readline, errReading := reader.ReadString('\n') + if errReading != nil && errReading != io.EOF { + return errReading + } + if strings.Contains(readline, "$yqLeadingContent$") { + // skip this + + } else if strings.Contains(readline, "$yqDocSeperator$") { + if p.printDocSeparators { + if err := p.writeString(bufferedWriter, "---\n"); err != nil { + return err + } + } + } else if !p.outputToJSON { + if err := p.writeString(bufferedWriter, readline); err != nil { + return err + } + } + + if errReading == io.EOF { + break } } - printedLead = true - } + } if err := p.printNode(mappedDoc.Node, bufferedWriter); err != nil { return err } diff --git a/pkg/yqlib/printer_test.go b/pkg/yqlib/printer_test.go index 7a30787f..aa729b0f 100644 --- a/pkg/yqlib/printer_test.go +++ b/pkg/yqlib/printer_test.go @@ -52,17 +52,17 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) { el = el.Next() sample3 := nodeToList(el.Value.(*CandidateNode)) - err = printer.PrintResults(sample1, "") + err = printer.PrintResults(sample1) if err != nil { panic(err) } - err = printer.PrintResults(sample2, "") + err = printer.PrintResults(sample2) if err != nil { panic(err) } - err = printer.PrintResults(sample3, "") + err = printer.PrintResults(sample3) if err != nil { panic(err) } @@ -82,25 +82,28 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) { } el := inputs.Front() + el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n" sample1 := nodeToList(el.Value.(*CandidateNode)) el = el.Next() + el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n" sample2 := nodeToList(el.Value.(*CandidateNode)) el = el.Next() + el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n# cool\n" sample3 := nodeToList(el.Value.(*CandidateNode)) - err = printer.PrintResults(sample1, "# go cats\n---\n") + err = printer.PrintResults(sample1) if err != nil { panic(err) } - err = printer.PrintResults(sample2, "---\n") + err = printer.PrintResults(sample2) if err != nil { panic(err) } - err = printer.PrintResults(sample3, "---\n# cool\n") + err = printer.PrintResults(sample3) if err != nil { panic(err) } @@ -138,17 +141,17 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) { elNode.FileIndex = 2 sample3 := nodeToList(elNode) - err = printer.PrintResults(sample1, "") + err = printer.PrintResults(sample1) if err != nil { panic(err) } - err = printer.PrintResults(sample2, "") + err = printer.PrintResults(sample2) if err != nil { panic(err) } - err = printer.PrintResults(sample3, "") + err = printer.PrintResults(sample3) if err != nil { panic(err) } @@ -171,31 +174,34 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) { elNode := el.Value.(*CandidateNode) elNode.Document = 0 elNode.FileIndex = 0 + elNode.Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n" sample1 := nodeToList(elNode) el = el.Next() elNode = el.Value.(*CandidateNode) elNode.Document = 0 elNode.FileIndex = 1 + elNode.Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n" sample2 := nodeToList(elNode) el = el.Next() elNode = el.Value.(*CandidateNode) elNode.Document = 0 elNode.FileIndex = 2 + elNode.Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n# cool\n" sample3 := nodeToList(elNode) - err = printer.PrintResults(sample1, "# go cats\n---\n") + err = printer.PrintResults(sample1) if err != nil { panic(err) } - err = printer.PrintResults(sample2, "---\n") + err = printer.PrintResults(sample2) if err != nil { panic(err) } - err = printer.PrintResults(sample3, "---\n# cool\n") + err = printer.PrintResults(sample3) if err != nil { panic(err) } @@ -214,7 +220,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) { panic(err) } - err = printer.PrintResults(inputs, "") + err = printer.PrintResults(inputs) if err != nil { panic(err) } @@ -233,7 +239,9 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) { panic(err) } - err = printer.PrintResults(inputs, "# go cats\n---\n") + inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n" + + err = printer.PrintResults(inputs) if err != nil { panic(err) } @@ -259,8 +267,8 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) { if err != nil { panic(err) } - - err = printer.PrintResults(inputs, "---\n") + inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n" + err = printer.PrintResults(inputs) if err != nil { panic(err) } @@ -313,7 +321,9 @@ func TestPrinterMultipleDocsJson(t *testing.T) { panic(err) } - err = printer.PrintResults(inputs, "# ignore this") + inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# ignore this\n" + + err = printer.PrintResults(inputs) if err != nil { panic(err) } diff --git a/pkg/yqlib/stream_evaluator.go b/pkg/yqlib/stream_evaluator.go index 6da4a3f2..84cebc2c 100644 --- a/pkg/yqlib/stream_evaluator.go +++ b/pkg/yqlib/stream_evaluator.go @@ -35,7 +35,7 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadin candidateNode := &CandidateNode{ Document: 0, Filename: "", - Node: &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode}, + Node: &yaml.Node{Kind: yaml.DocumentNode, HeadComment: leadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}}, FileIndex: 0, } inputList := list.New() @@ -45,7 +45,7 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadin if errorParsing != nil { return errorParsing } - return printer.PrintResults(result.MatchingNodes, leadingContent) + return printer.PrintResults(result.MatchingNodes) } func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error { @@ -100,6 +100,9 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr } else if errorReading != nil { return currentIndex, errorReading } + if currentIndex == 0 { + dataBucket.HeadComment = leadingContent + } candidateNode := &CandidateNode{ Document: currentIndex, Filename: filename, @@ -113,12 +116,7 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr if errorParsing != nil { return currentIndex, errorParsing } - var err error - if currentIndex == 0 { - err = printer.PrintResults(result.MatchingNodes, leadingContent) - } else { - err = printer.PrintResults(result.MatchingNodes, "") - } + err := printer.PrintResults(result.MatchingNodes) if err != nil { return currentIndex, err diff --git a/pkg/yqlib/utils.go b/pkg/yqlib/utils.go index ff4950c4..4927e1f6 100644 --- a/pkg/yqlib/utils.go +++ b/pkg/yqlib/utils.go @@ -12,7 +12,6 @@ import ( ) func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, string, error) { - var commentLineRegEx = regexp.MustCompile(`^\s*#`) var reader *bufio.Reader if filename == "-" { reader = bufio.NewReader(os.Stdin) @@ -25,21 +24,33 @@ func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, s } reader = bufio.NewReader(file) } - var sb strings.Builder if !leadingContentPreProcessing { return reader, "", nil } + return processReadStream(reader) +} +func processReadStream(reader *bufio.Reader) (io.Reader, string, error) { + var commentLineRegEx = regexp.MustCompile(`^\s*#`) + var sb strings.Builder + sb.WriteString("$yqLeadingContent$\n") for { peekBytes, err := reader.Peek(3) - if err == io.EOF { // EOF are handled else where.. return reader, sb.String(), nil } else if err != nil { return reader, sb.String(), err - } else if string(peekBytes) == "---" || commentLineRegEx.MatchString(string(peekBytes)) { + } else if string(peekBytes) == "---" { + _, err := reader.ReadString('\n') + sb.WriteString("$yqDocSeperator$\n") + if err == io.EOF { + return reader, sb.String(), nil + } else if err != nil { + return reader, sb.String(), err + } + } else if commentLineRegEx.MatchString(string(peekBytes)) { line, err := reader.ReadString('\n') sb.WriteString(line) if err == io.EOF { @@ -51,7 +62,6 @@ func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, s return reader, sb.String(), nil } } - } func readDocuments(reader io.Reader, filename string, fileIndex int) (*list.List, error) {