From 3a6f73e836445f25d77a89f920459230626060de Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Mon, 19 Jul 2021 19:52:51 +1000 Subject: [PATCH] wip --- .gitignore | 1 + acceptance_tests/leading-seperator.sh | 126 +++++++++++++++++++++++++- cmd/evaluate_all_command.go | 2 +- cmd/evalute_sequence_command.go | 2 +- pkg/yqlib/all_at_once_evaluator.go | 24 +---- pkg/yqlib/operators_test.go | 2 +- pkg/yqlib/printer.go | 36 +++----- pkg/yqlib/stream_evaluator.go | 45 +++++---- pkg/yqlib/utils.go | 49 +++++----- 9 files changed, 197 insertions(+), 90 deletions(-) diff --git a/.gitignore b/.gitignore index 05f48c6e..f76be8ee 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ prime/ .snapcraft/ yq*.snap test.yml +test2.yml diff --git a/acceptance_tests/leading-seperator.sh b/acceptance_tests/leading-seperator.sh index 08c4582c..8b5d9825 100755 --- a/acceptance_tests/leading-seperator.sh +++ b/acceptance_tests/leading-seperator.sh @@ -7,6 +7,25 @@ a: test EOL } +testLeadingSeperatorWithDoc() { + cat >test.yml <test.yml <test.yml <test2.yml <test.yml <test2.yml <test.yml <test2.yml < 0 { - reader, _, err := readStream(filenames[0]) - if err != nil { - return err - } - switch reader := reader.(type) { - case *os.File: - defer safelyCloseFile(reader) - } - printer.SetPreamble(reader) - } } matches, err := e.EvaluateCandidateNodes(expression, allDocuments) if err != nil { return err } - printer.SetPrintLeadingSeperator(firstFileLeadingSeperator) - return printer.PrintResults(matches) + return printer.PrintResults(matches, firstFileLeadingContent) } diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index 55083227..c5fe1fe4 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -256,7 +256,7 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt t.Error(err, s.expression) } - err = printer.PrintResults(context.MatchingNodes) + err = printer.PrintResults(context.MatchingNodes, "") if err != nil { t.Error(err, s.expression) } diff --git a/pkg/yqlib/printer.go b/pkg/yqlib/printer.go index 693b99af..7f3a1a05 100644 --- a/pkg/yqlib/printer.go +++ b/pkg/yqlib/printer.go @@ -4,17 +4,14 @@ import ( "bufio" "container/list" "io" + "strings" yaml "gopkg.in/yaml.v3" ) type Printer interface { - PrintResults(matchingNodes *list.List) error + PrintResults(matchingNodes *list.List, leadingContent string) error PrintedAnything() bool - SetPrintLeadingSeperator(bool) - - SetPreamble(reader io.Reader) - //e.g. when given a front-matter doc, like jekyll SetAppendix(reader io.Reader) } @@ -48,13 +45,6 @@ func NewPrinter(writer io.Writer, outputToJSON bool, unwrapScalar bool, colorsEn } } -func (p *resultsPrinter) SetPrintLeadingSeperator(printLeadingSeperator bool) { - if printLeadingSeperator { - p.firstTimePrinting = false - p.previousFileIndex = -1 - } -} - func (p *resultsPrinter) SetPreamble(reader io.Reader) { p.preambleReader = reader } @@ -95,7 +85,7 @@ func (p *resultsPrinter) safelyFlush(writer *bufio.Writer) { } } -func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error { +func (p *resultsPrinter) PrintResults(matchingNodes *list.List, leadingContent string) error { log.Debug("PrintResults for %v matches", matchingNodes.Len()) if p.outputToJSON { explodeOp := Operation{OperationType: explodeOpType} @@ -110,15 +100,8 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error { bufferedWriter := bufio.NewWriter(p.writer) defer p.safelyFlush(bufferedWriter) - if p.preambleReader != nil && !p.outputToJSON { - log.Debug("Piping preamble reader...") - _, err := io.Copy(bufferedWriter, p.preambleReader) - if err != nil { - return err - } - } - if matchingNodes.Len() == 0 { + p.writeString(bufferedWriter, leadingContent) log.Debug("no matching results, nothing to print") return nil } @@ -129,16 +112,25 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error { p.firstTimePrinting = false } + printedLead := false + for el := matchingNodes.Front(); el != nil; el = el.Next() { mappedDoc := el.Value.(*CandidateNode) log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v, printDocSeparators: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document, p.printDocSeparators) - if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators { + if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators && + (printedLead || !strings.HasPrefix(leadingContent, "---")) { log.Debug("-- writing doc sep") if err := p.writeString(bufferedWriter, "---\n"); err != nil { return err } } + if !printedLead { + // we want to print this after the seperator logic + p.writeString(bufferedWriter, leadingContent) + printedLead = true + } + if err := p.printNode(mappedDoc.Node, bufferedWriter); err != nil { return err } diff --git a/pkg/yqlib/stream_evaluator.go b/pkg/yqlib/stream_evaluator.go index 9c0521df..39e63c1e 100644 --- a/pkg/yqlib/stream_evaluator.go +++ b/pkg/yqlib/stream_evaluator.go @@ -12,9 +12,9 @@ import ( // Uses less memory than loading all documents and running the expression once, but this cannot process // cross document expressions. type StreamEvaluator interface { - Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer) (uint, error) + Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, leadingContent string) (uint, error) EvaluateFiles(expression string, filenames []string, printer Printer) error - EvaluateNew(expression string, printer Printer) error + EvaluateNew(expression string, printer Printer, leadingContent string) error } type streamEvaluator struct { @@ -27,7 +27,7 @@ func NewStreamEvaluator() StreamEvaluator { return &streamEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: NewExpressionParser()} } -func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error { +func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadingContent string) error { node, err := s.treeCreator.ParseExpression(expression) if err != nil { return err @@ -45,7 +45,7 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error if errorParsing != nil { return errorParsing } - return printer.PrintResults(result.MatchingNodes) + return printer.PrintResults(result.MatchingNodes, leadingContent) } func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error { @@ -55,15 +55,19 @@ func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, p return err } + var firstFileLeadingContent string + for index, filename := range filenames { - reader, leadingSeperator, err := readStream(filename) - if index == 0 && leadingSeperator { - printer.SetPrintLeadingSeperator(leadingSeperator) + reader, leadingContent, err := readStream(filename) + + if index == 0 { + firstFileLeadingContent = leadingContent } + if err != nil { return err } - processedDocs, err := s.Evaluate(filename, reader, node, printer) + processedDocs, err := s.Evaluate(filename, reader, node, printer, leadingContent) if err != nil { return err } @@ -76,26 +80,13 @@ func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, p } if totalProcessDocs == 0 { - - if len(filenames) > 0 { - reader, _, err := readStream(filenames[0]) - if err != nil { - return err - } - switch reader := reader.(type) { - case *os.File: - defer safelyCloseFile(reader) - } - printer.SetPreamble(reader) - } - - return s.EvaluateNew(expression, printer) + return s.EvaluateNew(expression, printer, firstFileLeadingContent) } return nil } -func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer) (uint, error) { +func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, leadingContent string) (uint, error) { var currentIndex uint decoder := yaml.NewDecoder(reader) @@ -122,7 +113,13 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr if errorParsing != nil { return currentIndex, errorParsing } - err := printer.PrintResults(result.MatchingNodes) + var err error + if currentIndex == 0 { + err = printer.PrintResults(result.MatchingNodes, leadingContent) + } else { + err = printer.PrintResults(result.MatchingNodes, "") + } + if err != nil { return currentIndex, err } diff --git a/pkg/yqlib/utils.go b/pkg/yqlib/utils.go index 718e3a02..a349d05e 100644 --- a/pkg/yqlib/utils.go +++ b/pkg/yqlib/utils.go @@ -5,42 +5,49 @@ import ( "container/list" "io" "os" + "regexp" + "strings" yaml "gopkg.in/yaml.v3" ) -func readStream(filename string) (io.Reader, bool, error) { - +func readStream(filename string) (io.Reader, string, error) { + var commentLineRegEx = regexp.MustCompile(`^\s*#`) + var reader *bufio.Reader if filename == "-" { - reader := bufio.NewReader(os.Stdin) - - seperatorBytes, err := reader.Peek(3) - - if err == io.EOF { - // EOF are handled else where.. - return reader, false, nil - } - - return reader, string(seperatorBytes) == "---", err + reader = bufio.NewReader(os.Stdin) } else { // ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory, // and ensuring that it's not possible to give a path to a file outside thar directory. - reader, err := os.Open(filename) // #nosec + file, err := os.Open(filename) // #nosec if err != nil { - return nil, false, err + return nil, "", err } - seperatorBytes := make([]byte, 3) - _, err = reader.Read(seperatorBytes) + reader = bufio.NewReader(file) + } + var sb strings.Builder + + for { + peekBytes, err := reader.Peek(3) + if err == io.EOF { // EOF are handled else where.. - return reader, false, nil + return reader, sb.String(), nil } else if err != nil { - return nil, false, err + return reader, sb.String(), err + } else if string(peekBytes) == "---" || commentLineRegEx.MatchString(string(peekBytes)) { + line, err := reader.ReadString('\n') + sb.WriteString(line) + if err == io.EOF { + return reader, sb.String(), nil + } else if err != nil { + return reader, sb.String(), err + } + } else { + return reader, sb.String(), nil } - _, err = reader.Seek(0, 0) - - return reader, string(seperatorBytes) == "---", err } + } func readDocuments(reader io.Reader, filename string, fileIndex int) (*list.List, error) {