From d74bb8e28da694c0f90c74848043481af9b38060 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Wed, 14 Jul 2021 20:48:16 +1000 Subject: [PATCH] Adding magic to detect leading seperators --- examples/data1.yaml | 1 + examples/leading-seperator.yaml | 2 ++ pkg/yqlib/all_at_once_evaluator.go | 9 ++++- pkg/yqlib/printer.go | 8 +++++ pkg/yqlib/printer_test.go | 24 +++++++++++++ pkg/yqlib/stream_evaluator.go | 7 ++-- pkg/yqlib/utils.go | 21 +++++++++-- scripts/acceptance.sh | 58 ++++++++++++++++++++++++++++++ 8 files changed, 124 insertions(+), 6 deletions(-) create mode 100644 examples/leading-seperator.yaml diff --git a/examples/data1.yaml b/examples/data1.yaml index 18e5c1a3..119349b0 100644 --- a/examples/data1.yaml +++ b/examples/data1.yaml @@ -1 +1,2 @@ +--- {"a":{"b":1}} \ No newline at end of file diff --git a/examples/leading-seperator.yaml b/examples/leading-seperator.yaml new file mode 100644 index 00000000..cf11014d --- /dev/null +++ b/examples/leading-seperator.yaml @@ -0,0 +1,2 @@ +--- +a: test \ No newline at end of file diff --git a/pkg/yqlib/all_at_once_evaluator.go b/pkg/yqlib/all_at_once_evaluator.go index cfb936b7..be6ccead 100644 --- a/pkg/yqlib/all_at_once_evaluator.go +++ b/pkg/yqlib/all_at_once_evaluator.go @@ -48,13 +48,19 @@ func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCand func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error { fileIndex := 0 + firstFileLeadingSeperator := false var allDocuments *list.List = list.New() for _, filename := range filenames { - reader, err := readStream(filename) + reader, leadingSeperator, err := readStream(filename) if err != nil { return err } + + if fileIndex == 0 && leadingSeperator { + firstFileLeadingSeperator = leadingSeperator + } + fileDocuments, err := readDocuments(reader, filename, fileIndex) if err != nil { return err @@ -66,5 +72,6 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string if err != nil { return err } + printer.SetPrintLeadingSeperator(firstFileLeadingSeperator) return printer.PrintResults(matches) } diff --git a/pkg/yqlib/printer.go b/pkg/yqlib/printer.go index b4d9b9f0..fc0d549a 100644 --- a/pkg/yqlib/printer.go +++ b/pkg/yqlib/printer.go @@ -11,6 +11,7 @@ import ( type Printer interface { PrintResults(matchingNodes *list.List) error PrintedAnything() bool + SetPrintLeadingSeperator(bool) } type resultsPrinter struct { @@ -40,6 +41,13 @@ func NewPrinter(writer io.Writer, outputToJSON bool, unwrapScalar bool, colorsEn } } +func (p *resultsPrinter) SetPrintLeadingSeperator(printLeadingSeperator bool) { + if printLeadingSeperator { + p.firstTimePrinting = false + p.previousFileIndex = -1 + } +} + func (p *resultsPrinter) PrintedAnything() bool { return p.printedMatches } diff --git a/pkg/yqlib/printer_test.go b/pkg/yqlib/printer_test.go index 5bad9ff5..fddf6485 100644 --- a/pkg/yqlib/printer_test.go +++ b/pkg/yqlib/printer_test.go @@ -17,12 +17,36 @@ a: apple a: coconut ` +var leadingSeperatorSample = `--- +a: good doc +` + func nodeToList(candidate *CandidateNode) *list.List { elMap := list.New() elMap.PushBack(candidate) return elMap } +func TestPrinterWithLeadingSeperator(t *testing.T) { + var output bytes.Buffer + var writer = bufio.NewWriter(&output) + printer := NewPrinter(writer, false, true, false, 2, true) + + inputs, err := readDocuments(strings.NewReader(leadingSeperatorSample), "sample.yml", 0) + if err != nil { + panic(err) + } + printer.SetPrintLeadingSeperator(true) + + err = printer.PrintResults(inputs) + if err != nil { + panic(err) + } + + writer.Flush() + test.AssertResult(t, leadingSeperatorSample, output.String()) +} + func TestPrinterMultipleDocsInSequence(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) diff --git a/pkg/yqlib/stream_evaluator.go b/pkg/yqlib/stream_evaluator.go index 093c2777..17517720 100644 --- a/pkg/yqlib/stream_evaluator.go +++ b/pkg/yqlib/stream_evaluator.go @@ -55,8 +55,11 @@ func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, p return err } - for _, filename := range filenames { - reader, err := readStream(filename) + for index, filename := range filenames { + reader, leadingSeperator, err := readStream(filename) + if index == 0 && leadingSeperator { + printer.SetPrintLeadingSeperator(leadingSeperator) + } if err != nil { return err } diff --git a/pkg/yqlib/utils.go b/pkg/yqlib/utils.go index aee9f5e1..4f32d64d 100644 --- a/pkg/yqlib/utils.go +++ b/pkg/yqlib/utils.go @@ -9,13 +9,28 @@ import ( yaml "gopkg.in/yaml.v3" ) -func readStream(filename string) (io.Reader, error) { +func readStream(filename string) (io.Reader, bool, error) { + if filename == "-" { - return bufio.NewReader(os.Stdin), nil + reader := bufio.NewReader(os.Stdin) + + seperatorBytes, err := reader.Peek(3) + return reader, string(seperatorBytes) == "---", err } else { // ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory, // and ensuring that it's not possible to give a path to a file outside thar directory. - return os.Open(filename) // #nosec + reader, err := os.Open(filename) + if err != nil { + return nil, false, err + } + seperatorBytes := make([]byte, 3) + _, err = reader.Read(seperatorBytes) + if err != nil { + return nil, false, err + } + _, err = reader.Seek(0, 0) + + return reader, string(seperatorBytes) == "---", err } } diff --git a/scripts/acceptance.sh b/scripts/acceptance.sh index b543a8b3..b4935c1f 100755 --- a/scripts/acceptance.sh +++ b/scripts/acceptance.sh @@ -55,6 +55,64 @@ if [[ $? != 1 ]]; then exit 1 fi +# Test leading seperator logic +expected=$(cat examples/leading-seperator.yaml) + +X=$(cat examples/leading-seperator.yaml | ./yq e '.' -) +if [[ $X != $expected ]]; then + echo "Pipe into e" + echo "Expected $expected but was $X" + exit 1 +fi + +X=$(./yq e '.' examples/leading-seperator.yaml) +expected=$(cat examples/leading-seperator.yaml) +if [[ $X != $expected ]]; then + echo "read given file e" + echo "Expected $expected but was $X" + exit 1 +fi + +X=$(cat examples/leading-seperator.yaml | ./yq ea '.' -) +if [[ $X != $expected ]]; then + echo "Pipe into e" + echo "Expected $expected but was $X" + exit 1 +fi + +X=$(./yq ea '.' examples/leading-seperator.yaml) +expected=$(cat examples/leading-seperator.yaml) +if [[ $X != $expected ]]; then + echo "read given file e" + echo "Expected $expected but was $X" + exit 1 +fi + +# multidoc +read -r -d '' expected << EOM +--- +a: test +--- +version: 3 +application: MyApp +EOM + +X=$(./yq e '.' examples/leading-seperator.yaml examples/order.yaml) + +if [[ $X != $expected ]]; then + echo "Multidoc with leading seperator" + echo "Expected $expected but was $X" + exit 1 +fi + +X=$(./yq ea '.' examples/leading-seperator.yaml examples/order.yaml) + +if [[ $X != $expected ]]; then + echo "Multidoc with leading seperator" + echo "Expected $expected but was $X" + exit 1 +fi + echo "--success" set -e