Better way of processing leading content

acceptance_tests/basic.sh

@@ -40,4 +40,33 @@ testBasicExitStatus() {
   assertEquals 1 "$?"
 }
 
+testBasicExtractFieldWithSeperator() {
+    cat >test.yml <<EOL
+---
+name: chart-name
+version: 1.2.3
+EOL
+  X=$(./yq e '.name' test.yml)
+  assertEquals "chart-name" "$X"
+}
+
+testBasicExtractMultipleFieldWithSeperator() {
+    cat >test.yml <<EOL
+---
+name: chart-name
+version: 1.2.3
+---
+name: thing
+version: 1.2.3
+EOL
+
+read -r -d '' expected << EOM
+chart-name
+---
+thing
+EOM
+  X=$(./yq e '.name' test.yml)
+  assertEquals "$expected" "$X"
+}
+
 source ./scripts/shunit2

acceptance_tests/leading-seperator.sh

@@ -229,6 +229,113 @@ EOM
   assertEquals "$expected" "$X"
 }
 
+testLeadingSeperatorMultiDocEvalCommentsStripComments() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  # it will be hard to remove that top level separator
+  read -r -d '' expected << EOM
+a: test
+---
+b: sane
+EOM
+
+  X=$(./yq e '... comments=""'  test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalCommentsLeadingSepNoDocFlag() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+a: test
+# this is another doc
+# great
+b: sane
+EOM
+
+
+  X=$(./yq e '.' --no-doc test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalJsonFlag() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+EOL
+
+cat >test2.yml <<EOL
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+{
+  "a": "test"
+}
+{
+  "b": "sane"
+}
+EOM
+
+
+  X=$(./yq e '.' -j test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalAllJsonFlag() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+EOL
+
+cat >test2.yml <<EOL
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+{
+  "a": "test"
+}
+{
+  "b": "sane"
+}
+EOM
+
+
+  X=$(./yq ea '.' -j test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
 testLeadingSeperatorMultiDocEvalAll() {
   read -r -d '' expected << EOM
 ---

cmd/constant.go

@@ -1,5 +1,6 @@
 package cmd
 
+var leadingContentPreProcessing = true
 var unwrapScalar = true
 
 var writeInplace = false

cmd/evaluate_all_command.go

@@ -112,7 +112,7 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 	switch len(args) {
 	case 0:
 		if pipingStdIn {
-			err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer)
+			err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer, leadingContentPreProcessing)
 		} else {
 			cmd.Println(cmd.UsageString())
 			return nil
@@ -121,10 +121,10 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 		if nullInput {
 			err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(args[0]), printer, "")
 		} else {
-			err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer)
+			err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer, leadingContentPreProcessing)
 		}
 	default:
-		err = allAtOnceEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer)
+		err = allAtOnceEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing)
 	}
 
 	completedSuccessfully = err == nil

cmd/evalute_sequence_command.go

@@ -125,7 +125,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 	switch len(args) {
 	case 0:
 		if pipingStdIn {
-			err = streamEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer)
+			err = streamEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer, leadingContentPreProcessing)
 		} else {
 			cmd.Println(cmd.UsageString())
 			return nil
@@ -134,10 +134,10 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 		if nullInput {
 			err = streamEvaluator.EvaluateNew(processExpression(args[0]), printer, "")
 		} else {
-			err = streamEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer)
+			err = streamEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer, leadingContentPreProcessing)
 		}
 	default:
-		err = streamEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer)
+		err = streamEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing)
 	}
 	completedSuccessfully = err == nil
 

cmd/root.go

@@ -55,6 +55,7 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 	rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")
 	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
 	rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
+	rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and seperators before processing expression. This is a workaround for go-yaml to persist header content. This flag will be removed once this feature has been out in the wild for a while.")
 	rootCmd.AddCommand(
 		createEvaluateSequenceCommand(),
 		createEvaluateAllCommand(),

examples/data1.yaml

@@ -1,2 +1,4 @@
 ---
-{"a":{"b":1}}
+# hi peeps
+# cool
+a: test

pkg/yqlib/all_at_once_evaluator.go

@@ -8,7 +8,7 @@ import (
 
 // A yaml expression evaluator that runs the expression once against all files/nodes in memory.
 type Evaluator interface {
-	EvaluateFiles(expression string, filenames []string, printer Printer) error
+	EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error
 
 	// EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes
 	EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error)
@@ -46,13 +46,13 @@ func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCand
 	return context.MatchingNodes, nil
 }
 
-func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error {
+func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error {
 	fileIndex := 0
 	firstFileLeadingContent := ""
 
 	var allDocuments *list.List = list.New()
 	for _, filename := range filenames {
-		reader, leadingContent, err := readStream(filename)
+		reader, leadingContent, err := readStream(filename, leadingContentPreProcessing)
 		if err != nil {
 			return err
 		}
@@ -73,15 +73,17 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
 		candidateNode := &CandidateNode{
 			Document:  0,
 			Filename:  "",
-			Node:      &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode},
+			Node:      &yaml.Node{Kind: yaml.DocumentNode, HeadComment: firstFileLeadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
 			FileIndex: 0,
 		}
 		allDocuments.PushBack(candidateNode)
+	} else {
+		allDocuments.Front().Value.(*CandidateNode).Node.HeadComment = firstFileLeadingContent
 	}
 
 	matches, err := e.EvaluateCandidateNodes(expression, allDocuments)
 	if err != nil {
 		return err
 	}
-	return printer.PrintResults(matches, firstFileLeadingContent)
+	return printer.PrintResults(matches)
 }

pkg/yqlib/operators_test.go

@@ -256,7 +256,7 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt
 		t.Error(err, s.expression)
 	}
 
-	err = printer.PrintResults(context.MatchingNodes, "")
+	err = printer.PrintResults(context.MatchingNodes)
 	if err != nil {
 		t.Error(err, s.expression)
 	}

pkg/yqlib/printer.go

@@ -10,7 +10,7 @@ import (
 )
 
 type Printer interface {
-	PrintResults(matchingNodes *list.List, leadingContent string) error
+	PrintResults(matchingNodes *list.List) error
 	PrintedAnything() bool
 	//e.g. when given a front-matter doc, like jekyll
 	SetAppendix(reader io.Reader)
@@ -85,7 +85,7 @@ func (p *resultsPrinter) safelyFlush(writer *bufio.Writer) {
 	}
 }
 
-func (p *resultsPrinter) PrintResults(matchingNodes *list.List, leadingContent string) error {
+func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 	log.Debug("PrintResults for %v matches", matchingNodes.Len())
 	if p.outputToJSON {
 		explodeOp := Operation{OperationType: explodeOpType}
@@ -101,9 +101,6 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List, leadingContent s
 	defer p.safelyFlush(bufferedWriter)
 
 	if matchingNodes.Len() == 0 {
-		if err := p.writeString(bufferedWriter, leadingContent); err != nil {
-			return err
-		}
 		log.Debug("no matching results, nothing to print")
 		return nil
 	}
@@ -114,27 +111,52 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List, leadingContent s
 		p.firstTimePrinting = false
 	}
 
-	printedLead := false
-
 	for el := matchingNodes.Front(); el != nil; el = el.Next() {
 		mappedDoc := el.Value.(*CandidateNode)
 		log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v, printDocSeparators: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document, p.printDocSeparators)
-		if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators &&
-			(printedLead || !strings.HasPrefix(leadingContent, "---")) {
+
+		commentStartsWithSeparator := strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$\n$yqDocSeperator$")
+
+		if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators && !commentStartsWithSeparator {
 			log.Debug("-- writing doc sep")
 			if err := p.writeString(bufferedWriter, "---\n"); err != nil {
 				return err
 			}
 		}
 
-		if !printedLead {
-			// we want to print this after the seperator logic
-			if err := p.writeString(bufferedWriter, leadingContent); err != nil {
-				return err
-			}
-			printedLead = true
-		}
+		if strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$") {
+			log.Debug("headcommentwas %v", mappedDoc.Node.HeadComment)
+			log.Debug("finished headcomment")
+			reader := bufio.NewReader(strings.NewReader(mappedDoc.Node.HeadComment))
+			mappedDoc.Node.HeadComment = ""
 
+			for {
+
+				readline, errReading := reader.ReadString('\n')
+				if errReading != nil && errReading != io.EOF {
+					return errReading
+				}
+				if strings.Contains(readline, "$yqLeadingContent$") {
+					// skip this
+
+				} else if strings.Contains(readline, "$yqDocSeperator$") {
+					if p.printDocSeparators {
+						if err := p.writeString(bufferedWriter, "---\n"); err != nil {
+							return err
+						}
+					}
+				} else if !p.outputToJSON {
+					if err := p.writeString(bufferedWriter, readline); err != nil {
+						return err
+					}
+				}
+
+				if errReading == io.EOF {
+					break
+				}
+			}
+
+		}
 		if err := p.printNode(mappedDoc.Node, bufferedWriter); err != nil {
 			return err
 		}

pkg/yqlib/printer_test.go

@@ -52,17 +52,17 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) {
 	el = el.Next()
 	sample3 := nodeToList(el.Value.(*CandidateNode))
 
-	err = printer.PrintResults(sample1, "")
+	err = printer.PrintResults(sample1)
 	if err != nil {
 		panic(err)
 	}
 
-	err = printer.PrintResults(sample2, "")
+	err = printer.PrintResults(sample2)
 	if err != nil {
 		panic(err)
 	}
 
-	err = printer.PrintResults(sample3, "")
+	err = printer.PrintResults(sample3)
 	if err != nil {
 		panic(err)
 	}
@@ -82,25 +82,28 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 	}
 
 	el := inputs.Front()
+	el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
 	sample1 := nodeToList(el.Value.(*CandidateNode))
 
 	el = el.Next()
+	el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
 	sample2 := nodeToList(el.Value.(*CandidateNode))
 
 	el = el.Next()
+	el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n# cool\n"
 	sample3 := nodeToList(el.Value.(*CandidateNode))
 
-	err = printer.PrintResults(sample1, "# go cats\n---\n")
+	err = printer.PrintResults(sample1)
 	if err != nil {
 		panic(err)
 	}
 
-	err = printer.PrintResults(sample2, "---\n")
+	err = printer.PrintResults(sample2)
 	if err != nil {
 		panic(err)
 	}
 
-	err = printer.PrintResults(sample3, "---\n# cool\n")
+	err = printer.PrintResults(sample3)
 	if err != nil {
 		panic(err)
 	}
@@ -138,17 +141,17 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) {
 	elNode.FileIndex = 2
 	sample3 := nodeToList(elNode)
 
-	err = printer.PrintResults(sample1, "")
+	err = printer.PrintResults(sample1)
 	if err != nil {
 		panic(err)
 	}
 
-	err = printer.PrintResults(sample2, "")
+	err = printer.PrintResults(sample2)
 	if err != nil {
 		panic(err)
 	}
 
-	err = printer.PrintResults(sample3, "")
+	err = printer.PrintResults(sample3)
 	if err != nil {
 		panic(err)
 	}
@@ -171,31 +174,34 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 	elNode := el.Value.(*CandidateNode)
 	elNode.Document = 0
 	elNode.FileIndex = 0
+	elNode.Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
 	sample1 := nodeToList(elNode)
 
 	el = el.Next()
 	elNode = el.Value.(*CandidateNode)
 	elNode.Document = 0
 	elNode.FileIndex = 1
+	elNode.Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
 	sample2 := nodeToList(elNode)
 
 	el = el.Next()
 	elNode = el.Value.(*CandidateNode)
 	elNode.Document = 0
 	elNode.FileIndex = 2
+	elNode.Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n# cool\n"
 	sample3 := nodeToList(elNode)
 
-	err = printer.PrintResults(sample1, "# go cats\n---\n")
+	err = printer.PrintResults(sample1)
 	if err != nil {
 		panic(err)
 	}
 
-	err = printer.PrintResults(sample2, "---\n")
+	err = printer.PrintResults(sample2)
 	if err != nil {
 		panic(err)
 	}
 
-	err = printer.PrintResults(sample3, "---\n# cool\n")
+	err = printer.PrintResults(sample3)
 	if err != nil {
 		panic(err)
 	}
@@ -214,7 +220,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 		panic(err)
 	}
 
-	err = printer.PrintResults(inputs, "")
+	err = printer.PrintResults(inputs)
 	if err != nil {
 		panic(err)
 	}
@@ -233,7 +239,9 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
 		panic(err)
 	}
 
-	err = printer.PrintResults(inputs, "# go cats\n---\n")
+	inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
+
+	err = printer.PrintResults(inputs)
 	if err != nil {
 		panic(err)
 	}
@@ -259,8 +267,8 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
 	if err != nil {
 		panic(err)
 	}
-
-	err = printer.PrintResults(inputs, "---\n")
+	inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
+	err = printer.PrintResults(inputs)
 	if err != nil {
 		panic(err)
 	}
@@ -276,6 +284,31 @@ a: coconut
 	test.AssertResult(t, expected, output.String())
 }
 
+func TestPrinterScalarWithLeadingCont(t *testing.T) {
+	var output bytes.Buffer
+	var writer = bufio.NewWriter(&output)
+	printer := NewPrinter(writer, false, true, false, 2, true)
+
+	node, err := NewExpressionParser().ParseExpression(".a")
+	if err != nil {
+		panic(err)
+	}
+	streamEvaluator := NewStreamEvaluator()
+	_, err = streamEvaluator.Evaluate("sample", strings.NewReader(multiDocSample), node, printer, "# blah\n")
+	if err != nil {
+		panic(err)
+	}
+
+	writer.Flush()
+	expected := `banana
+---
+apple
+---
+coconut
+`
+	test.AssertResult(t, expected, output.String())
+}
+
 func TestPrinterMultipleDocsJson(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
@@ -288,7 +321,9 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 		panic(err)
 	}
 
-	err = printer.PrintResults(inputs, "")
+	inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# ignore this\n"
+
+	err = printer.PrintResults(inputs)
 	if err != nil {
 		panic(err)
 	}

pkg/yqlib/stream_evaluator.go

@@ -13,7 +13,7 @@ import (
 // cross document expressions.
 type StreamEvaluator interface {
 	Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, leadingContent string) (uint, error)
-	EvaluateFiles(expression string, filenames []string, printer Printer) error
+	EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error
 	EvaluateNew(expression string, printer Printer, leadingContent string) error
 }
 
@@ -35,7 +35,7 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadin
 	candidateNode := &CandidateNode{
 		Document:  0,
 		Filename:  "",
-		Node:      &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode},
+		Node:      &yaml.Node{Kind: yaml.DocumentNode, HeadComment: leadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
 		FileIndex: 0,
 	}
 	inputList := list.New()
@@ -45,10 +45,10 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadin
 	if errorParsing != nil {
 		return errorParsing
 	}
-	return printer.PrintResults(result.MatchingNodes, leadingContent)
+	return printer.PrintResults(result.MatchingNodes)
 }
 
-func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error {
+func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error {
 	var totalProcessDocs uint = 0
 	node, err := s.treeCreator.ParseExpression(expression)
 	if err != nil {
@@ -58,7 +58,7 @@ func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, p
 	var firstFileLeadingContent string
 
 	for index, filename := range filenames {
-		reader, leadingContent, err := readStream(filename)
+		reader, leadingContent, err := readStream(filename, leadingContentPreProcessing)
 
 		if index == 0 {
 			firstFileLeadingContent = leadingContent
@@ -100,6 +100,9 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr
 		} else if errorReading != nil {
 			return currentIndex, errorReading
 		}
+		if currentIndex == 0 {
+			dataBucket.HeadComment = leadingContent
+		}
 		candidateNode := &CandidateNode{
 			Document:  currentIndex,
 			Filename:  filename,
@@ -113,12 +116,7 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr
 		if errorParsing != nil {
 			return currentIndex, errorParsing
 		}
-		var err error
-		if currentIndex == 0 {
-			err = printer.PrintResults(result.MatchingNodes, leadingContent)
-		} else {
-			err = printer.PrintResults(result.MatchingNodes, "")
-		}
+		err := printer.PrintResults(result.MatchingNodes)
 
 		if err != nil {
 			return currentIndex, err

pkg/yqlib/utils.go

@@ -11,8 +11,7 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-func readStream(filename string) (io.Reader, string, error) {
-	var commentLineRegEx = regexp.MustCompile(`^\s*#`)
+func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, string, error) {
 	var reader *bufio.Reader
 	if filename == "-" {
 		reader = bufio.NewReader(os.Stdin)
@@ -25,17 +24,33 @@ func readStream(filename string) (io.Reader, string, error) {
 		}
 		reader = bufio.NewReader(file)
 	}
-	var sb strings.Builder
 
+	if !leadingContentPreProcessing {
+		return reader, "", nil
+	}
+	return processReadStream(reader)
+}
+
+func processReadStream(reader *bufio.Reader) (io.Reader, string, error) {
+	var commentLineRegEx = regexp.MustCompile(`^\s*#`)
+	var sb strings.Builder
+	sb.WriteString("$yqLeadingContent$\n")
 	for {
 		peekBytes, err := reader.Peek(3)
-
 		if err == io.EOF {
 			// EOF are handled else where..
 			return reader, sb.String(), nil
 		} else if err != nil {
 			return reader, sb.String(), err
-		} else if string(peekBytes) == "---" || commentLineRegEx.MatchString(string(peekBytes)) {
+		} else if string(peekBytes) == "---" {
+			_, err := reader.ReadString('\n')
+			sb.WriteString("$yqDocSeperator$\n")
+			if err == io.EOF {
+				return reader, sb.String(), nil
+			} else if err != nil {
+				return reader, sb.String(), err
+			}
+		} else if commentLineRegEx.MatchString(string(peekBytes)) {
 			line, err := reader.ReadString('\n')
 			sb.WriteString(line)
 			if err == io.EOF {
@@ -47,7 +62,6 @@ func readStream(filename string) (io.Reader, string, error) {
 			return reader, sb.String(), nil
 		}
 	}
-
 }
 
 func readDocuments(reader io.Reader, filename string, fileIndex int) (*list.List, error) {