Better way of processing leading content

acceptance_tests/leading-seperator.sh

@@ -248,8 +248,7 @@ a: test
 b: sane
 EOM
 
-
-  X=$(./yq e --header-preprocess=false '... comments=""'  test.yml)
+  X=$(./yq e '... comments=""'  test.yml)
   assertEquals "$expected" "$X"
 }
 
@@ -265,9 +264,7 @@ a: test
 b: sane
 EOL
 
-  # it will be hard to remove that top level separator
   read -r -d '' expected << EOM
----
 # hi peeps
 # cool
 a: test

cmd/root.go

@@ -55,7 +55,7 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 	rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")
 	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
 	rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
-	rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and seperators before processing expression. This is a workaround for go-yaml to persist header content. You will want this off if you want to remove leading comments.")
+	rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and seperators before processing expression. This is a workaround for go-yaml to persist header content. This flag will be removed once this feature has been out in the wild for a while.")
 	rootCmd.AddCommand(
 		createEvaluateSequenceCommand(),
 		createEvaluateAllCommand(),

examples/data1.yaml

@@ -2,7 +2,3 @@
 # hi peeps
 # cool
 a: test
----
-# this is another doc
-# great
-b: sane

pkg/yqlib/all_at_once_evaluator.go

@@ -73,15 +73,17 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
 		candidateNode := &CandidateNode{
 			Document:  0,
 			Filename:  "",
-			Node:      &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode},
+			Node:      &yaml.Node{Kind: yaml.DocumentNode, HeadComment: firstFileLeadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
 			FileIndex: 0,
 		}
 		allDocuments.PushBack(candidateNode)
+	} else {
+		allDocuments.Front().Value.(*CandidateNode).Node.HeadComment = firstFileLeadingContent
 	}
 
 	matches, err := e.EvaluateCandidateNodes(expression, allDocuments)
 	if err != nil {
 		return err
 	}
-	return printer.PrintResults(matches, firstFileLeadingContent)
+	return printer.PrintResults(matches)
 }

pkg/yqlib/operators_test.go

@@ -256,7 +256,7 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt
 		t.Error(err, s.expression)
 	}
 
-	err = printer.PrintResults(context.MatchingNodes, "")
+	err = printer.PrintResults(context.MatchingNodes)
 	if err != nil {
 		t.Error(err, s.expression)
 	}

pkg/yqlib/printer.go

@@ -10,7 +10,7 @@ import (
 )
 
 type Printer interface {
-	PrintResults(matchingNodes *list.List, leadingContent string) error
+	PrintResults(matchingNodes *list.List) error
 	PrintedAnything() bool
 	//e.g. when given a front-matter doc, like jekyll
 	SetAppendix(reader io.Reader)
@@ -85,7 +85,7 @@ func (p *resultsPrinter) safelyFlush(writer *bufio.Writer) {
 	}
 }
 
-func (p *resultsPrinter) PrintResults(matchingNodes *list.List, leadingContent string) error {
+func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 	log.Debug("PrintResults for %v matches", matchingNodes.Len())
 	if p.outputToJSON {
 		explodeOp := Operation{OperationType: explodeOpType}
@@ -101,9 +101,6 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List, leadingContent s
 	defer p.safelyFlush(bufferedWriter)
 
 	if matchingNodes.Len() == 0 {
-		if err := p.writeString(bufferedWriter, leadingContent); err != nil {
-			return err
-		}
 		log.Debug("no matching results, nothing to print")
 		return nil
 	}
@@ -114,32 +111,52 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List, leadingContent s
 		p.firstTimePrinting = false
 	}
 
-	printedLead := false
-
 	for el := matchingNodes.Front(); el != nil; el = el.Next() {
 		mappedDoc := el.Value.(*CandidateNode)
 		log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v, printDocSeparators: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document, p.printDocSeparators)
-		if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators &&
-			(printedLead || !strings.HasPrefix(leadingContent, "---")) {
+
+		commentStartsWithSeparator := strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$\n$yqDocSeperator$")
+
+		if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators && !commentStartsWithSeparator {
 			log.Debug("-- writing doc sep")
 			if err := p.writeString(bufferedWriter, "---\n"); err != nil {
 				return err
 			}
 		}
 
-		if !printedLead {
-			// dont print leading comments and seperator if:
-			// - we are print json; or
-			// - we are printing an unwrapped scalar node
-			if !p.outputToJSON && (mappedDoc.Node.Kind != yaml.ScalarNode || !p.unwrapScalar) {
-				// we want to print this after the seperator logic
-				if err := p.writeString(bufferedWriter, leadingContent); err != nil {
+		if strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$") {
+			log.Debug("headcommentwas %v", mappedDoc.Node.HeadComment)
+			log.Debug("finished headcomment")
+			reader := bufio.NewReader(strings.NewReader(mappedDoc.Node.HeadComment))
+			mappedDoc.Node.HeadComment = ""
+
+			for {
+
+				readline, errReading := reader.ReadString('\n')
+				if errReading != nil && errReading != io.EOF {
+					return errReading
+				}
+				if strings.Contains(readline, "$yqLeadingContent$") {
+					// skip this
+
+				} else if strings.Contains(readline, "$yqDocSeperator$") {
+					if p.printDocSeparators {
+						if err := p.writeString(bufferedWriter, "---\n"); err != nil {
 							return err
 						}
 					}
-			printedLead = true
+				} else if !p.outputToJSON {
+					if err := p.writeString(bufferedWriter, readline); err != nil {
+						return err
+					}
 				}
 
+				if errReading == io.EOF {
+					break
+				}
+			}
+
+		}
 		if err := p.printNode(mappedDoc.Node, bufferedWriter); err != nil {
 			return err
 		}

pkg/yqlib/printer_test.go

@@ -52,17 +52,17 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) {
 	el = el.Next()
 	sample3 := nodeToList(el.Value.(*CandidateNode))
 
-	err = printer.PrintResults(sample1, "")
+	err = printer.PrintResults(sample1)
 	if err != nil {
 		panic(err)
 	}
 
-	err = printer.PrintResults(sample2, "")
+	err = printer.PrintResults(sample2)
 	if err != nil {
 		panic(err)
 	}
 
-	err = printer.PrintResults(sample3, "")
+	err = printer.PrintResults(sample3)
 	if err != nil {
 		panic(err)
 	}
@@ -82,25 +82,28 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 	}
 
 	el := inputs.Front()
+	el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
 	sample1 := nodeToList(el.Value.(*CandidateNode))
 
 	el = el.Next()
+	el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
 	sample2 := nodeToList(el.Value.(*CandidateNode))
 
 	el = el.Next()
+	el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n# cool\n"
 	sample3 := nodeToList(el.Value.(*CandidateNode))
 
-	err = printer.PrintResults(sample1, "# go cats\n---\n")
+	err = printer.PrintResults(sample1)
 	if err != nil {
 		panic(err)
 	}
 
-	err = printer.PrintResults(sample2, "---\n")
+	err = printer.PrintResults(sample2)
 	if err != nil {
 		panic(err)
 	}
 
-	err = printer.PrintResults(sample3, "---\n# cool\n")
+	err = printer.PrintResults(sample3)
 	if err != nil {
 		panic(err)
 	}
@@ -138,17 +141,17 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) {
 	elNode.FileIndex = 2
 	sample3 := nodeToList(elNode)
 
-	err = printer.PrintResults(sample1, "")
+	err = printer.PrintResults(sample1)
 	if err != nil {
 		panic(err)
 	}
 
-	err = printer.PrintResults(sample2, "")
+	err = printer.PrintResults(sample2)
 	if err != nil {
 		panic(err)
 	}
 
-	err = printer.PrintResults(sample3, "")
+	err = printer.PrintResults(sample3)
 	if err != nil {
 		panic(err)
 	}
@@ -171,31 +174,34 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 	elNode := el.Value.(*CandidateNode)
 	elNode.Document = 0
 	elNode.FileIndex = 0
+	elNode.Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
 	sample1 := nodeToList(elNode)
 
 	el = el.Next()
 	elNode = el.Value.(*CandidateNode)
 	elNode.Document = 0
 	elNode.FileIndex = 1
+	elNode.Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
 	sample2 := nodeToList(elNode)
 
 	el = el.Next()
 	elNode = el.Value.(*CandidateNode)
 	elNode.Document = 0
 	elNode.FileIndex = 2
+	elNode.Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n# cool\n"
 	sample3 := nodeToList(elNode)
 
-	err = printer.PrintResults(sample1, "# go cats\n---\n")
+	err = printer.PrintResults(sample1)
 	if err != nil {
 		panic(err)
 	}
 
-	err = printer.PrintResults(sample2, "---\n")
+	err = printer.PrintResults(sample2)
 	if err != nil {
 		panic(err)
 	}
 
-	err = printer.PrintResults(sample3, "---\n# cool\n")
+	err = printer.PrintResults(sample3)
 	if err != nil {
 		panic(err)
 	}
@@ -214,7 +220,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 		panic(err)
 	}
 
-	err = printer.PrintResults(inputs, "")
+	err = printer.PrintResults(inputs)
 	if err != nil {
 		panic(err)
 	}
@@ -233,7 +239,9 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
 		panic(err)
 	}
 
-	err = printer.PrintResults(inputs, "# go cats\n---\n")
+	inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
+
+	err = printer.PrintResults(inputs)
 	if err != nil {
 		panic(err)
 	}
@@ -259,8 +267,8 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
 	if err != nil {
 		panic(err)
 	}
-
-	err = printer.PrintResults(inputs, "---\n")
+	inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
+	err = printer.PrintResults(inputs)
 	if err != nil {
 		panic(err)
 	}
@@ -313,7 +321,9 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 		panic(err)
 	}
 
-	err = printer.PrintResults(inputs, "# ignore this")
+	inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# ignore this\n"
+
+	err = printer.PrintResults(inputs)
 	if err != nil {
 		panic(err)
 	}

pkg/yqlib/stream_evaluator.go

@@ -35,7 +35,7 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadin
 	candidateNode := &CandidateNode{
 		Document:  0,
 		Filename:  "",
-		Node:      &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode},
+		Node:      &yaml.Node{Kind: yaml.DocumentNode, HeadComment: leadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
 		FileIndex: 0,
 	}
 	inputList := list.New()
@@ -45,7 +45,7 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadin
 	if errorParsing != nil {
 		return errorParsing
 	}
-	return printer.PrintResults(result.MatchingNodes, leadingContent)
+	return printer.PrintResults(result.MatchingNodes)
 }
 
 func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error {
@@ -100,6 +100,9 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr
 		} else if errorReading != nil {
 			return currentIndex, errorReading
 		}
+		if currentIndex == 0 {
+			dataBucket.HeadComment = leadingContent
+		}
 		candidateNode := &CandidateNode{
 			Document:  currentIndex,
 			Filename:  filename,
@@ -113,12 +116,7 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr
 		if errorParsing != nil {
 			return currentIndex, errorParsing
 		}
-		var err error
-		if currentIndex == 0 {
-			err = printer.PrintResults(result.MatchingNodes, leadingContent)
-		} else {
-			err = printer.PrintResults(result.MatchingNodes, "")
-		}
+		err := printer.PrintResults(result.MatchingNodes)
 
 		if err != nil {
 			return currentIndex, err

pkg/yqlib/utils.go

@@ -12,7 +12,6 @@ import (
 )
 
 func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, string, error) {
-	var commentLineRegEx = regexp.MustCompile(`^\s*#`)
 	var reader *bufio.Reader
 	if filename == "-" {
 		reader = bufio.NewReader(os.Stdin)
@@ -25,21 +24,33 @@ func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, s
 		}
 		reader = bufio.NewReader(file)
 	}
-	var sb strings.Builder
 
 	if !leadingContentPreProcessing {
 		return reader, "", nil
 	}
+	return processReadStream(reader)
+}
 
+func processReadStream(reader *bufio.Reader) (io.Reader, string, error) {
+	var commentLineRegEx = regexp.MustCompile(`^\s*#`)
+	var sb strings.Builder
+	sb.WriteString("$yqLeadingContent$\n")
 	for {
 		peekBytes, err := reader.Peek(3)
-
 		if err == io.EOF {
 			// EOF are handled else where..
 			return reader, sb.String(), nil
 		} else if err != nil {
 			return reader, sb.String(), err
-		} else if string(peekBytes) == "---" || commentLineRegEx.MatchString(string(peekBytes)) {
+		} else if string(peekBytes) == "---" {
+			_, err := reader.ReadString('\n')
+			sb.WriteString("$yqDocSeperator$\n")
+			if err == io.EOF {
+				return reader, sb.String(), nil
+			} else if err != nil {
+				return reader, sb.String(), err
+			}
+		} else if commentLineRegEx.MatchString(string(peekBytes)) {
 			line, err := reader.ReadString('\n')
 			sb.WriteString(line)
 			if err == io.EOF {
@@ -51,7 +62,6 @@ func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, s
 			return reader, sb.String(), nil
 		}
 	}
-
 }
 
 func readDocuments(reader io.Reader, filename string, fileIndex int) (*list.List, error) {