Include blank new lines in leading header preprocessing #1462

acceptance_tests/leading-seperator.sh

@@ -34,6 +34,64 @@ EOM
   assertEquals "$expected" "$X"
 }
 
+
+testLeadingSeperatorWithNewlinesNewDoc() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+
+
+---
+a: test
+---
+b: cool
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+
+
+---
+a: thing
+---
+b: cool
+EOM
+
+  X=$(./yq e '(select(di == 0) | .a) = "thing"' - < test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorWithNewlinesMoreComments() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+
+---
+# great
+
+a: test
+---
+b: cool
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+
+---
+# great
+
+a: thing
+---
+b: cool
+EOM
+
+  X=$(./yq e '(select(di == 0) | .a) = "thing"' - < test.yml)
+  assertEquals "$expected" "$X"
+}
+
+
 testLeadingSeperatorWithDirective() {
   cat >test.yml <<EOL
 %YAML 1.1

pkg/yqlib/decoder_yaml.go

@@ -39,6 +39,14 @@ func (dec *yamlDecoder) processReadStream(reader *bufio.Reader) (io.Reader, stri
 			return reader, sb.String(), nil
 		} else if err != nil {
 			return reader, sb.String(), err
+		} else if string(peekBytes[0]) == "\n" {
+			_, err := reader.ReadString('\n')
+			sb.WriteString("\n")
+			if errors.Is(err, io.EOF) {
+				return reader, sb.String(), nil
+			} else if err != nil {
+				return reader, sb.String(), err
+			}
 		} else if string(peekBytes) == "---" {
 			_, err := reader.ReadString('\n')
 			sb.WriteString("$yqDocSeperator$\n")

pkg/yqlib/doc/operators/comment-operators.md

@@ -247,6 +247,7 @@ Note the use of `...` to ensure key nodes are included.
 Given a sample.yml file of:
 ```yaml
 # hi
+
 a: cat # comment
 # great
 b: # key comment
@@ -265,6 +266,7 @@ b:
 Given a sample.yml file of:
 ```yaml
 # welcome!
+
 a: cat # meow
 # have a great day
 ```
@@ -293,6 +295,7 @@ yq '. | head_comment' sample.yml
 will output
 ```yaml
 welcome!
+
 ```
 
 ## Head comment with document split

pkg/yqlib/doc/operators/path.md

@@ -127,6 +127,7 @@ Like pick but recursive. This uses `ireduce` to deeply set the selected paths in
 
 Given a sample.yml file of:
 ```yaml
+
 parentA: bob
 parentB:
   child1: i am child1

pkg/yqlib/operator_anchors_aliases_test.go

@@ -12,8 +12,7 @@ var specDocument = `- &CENTER { x: 1, y: 2 }
 
 var expectedSpecResult = "D0, P[4], (!!map)::x: 1\ny: 2\nr: 10\n"
 
-var simpleArrayRef = `
-item_value: &item_value
+var simpleArrayRef = `item_value: &item_value
   value: true
 
 thingOne:

pkg/yqlib/operator_comments_test.go

@@ -221,7 +221,7 @@ var commentOperatorScenarios = []expressionScenario{
 		document:              "# welcome!\n\na: cat # meow\n\n# have a great day",
 		expression:            `. | head_comment`,
 		expected: []string{
-			"D0, P[], (!!str)::welcome!\n",
+			"D0, P[], (!!str)::welcome!\n\n",
 		},
 	},
 	{

pkg/yqlib/operator_multiply_test.go

@@ -108,7 +108,7 @@ var multiplyOperatorScenarios = []expressionScenario{
 		document2:  docNoComments,
 		expression: `select(fi == 0) * select(fi == 1)`,
 		expected: []string{
-			"D0, P[], (!!map)::# here\na: apple\nb: banana\n",
+			"D0, P[], (!!map)::# here\n\na: apple\nb: banana\n",
 		},
 	},
 	{
@@ -126,7 +126,7 @@ var multiplyOperatorScenarios = []expressionScenario{
 		document2:  docWithHeader,
 		expression: `select(fi == 0) * select(fi == 1)`,
 		expected: []string{
-			"D0, P[], (!!map)::# here\nb: banana\na: apple\n",
+			"D0, P[], (!!map)::# here\n\nb: banana\na: apple\n",
 		},
 	},
 	{

pkg/yqlib/operator_traverse_path_test.go

@@ -4,8 +4,7 @@ import (
 	"testing"
 )
 
-var mergeDocSample = `
-foo: &foo
+var mergeDocSample = `foo: &foo
   a: foo_a
   thing: foo_thing
   c: foo_c