Fixed merging arrays with merge anchors #899

.gitignore

@@ -38,3 +38,5 @@ parts/
 prime/
 .snapcraft/
 yq*.snap
+test.yml
+test2.yml

README.md

@@ -176,6 +176,7 @@ Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
 - Written in portable go, so you can download a lovely dependency free binary
 - Uses similar syntax as `jq` but works with YAML and JSON files
 - Fully supports multi document yaml files
+- Supports yaml [front matter](https://mikefarah.gitbook.io/yq/usage/front-matter) blocks (e.g. jekyll/assemble)
 - Colorized yaml output
 - [Deeply traverse yaml](https://mikefarah.gitbook.io/yq/operators/traverse-read)
 - [Sort yaml by keys](https://mikefarah.gitbook.io/yq/operators/sort-keys)
@@ -199,24 +200,26 @@ Usage:
   yq [command]
 
 Available Commands:
-  eval             Apply expression to each document in each yaml file given in sequence
+  eval             Apply the expression to each document in each yaml file in sequence
   eval-all         Loads _all_ yaml documents of _all_ yaml files and runs expression once
   help             Help about any command
   shell-completion Generate completion script
 
 Flags:
-  -C, --colors        force print with colors
-  -e, --exit-status   set exit status if there are no matches or null or false is returned
-  -h, --help          help for yq
-  -I, --indent int    sets indent level for output (default 2)
-  -i, --inplace       update the yaml file inplace of first yaml file given.
-  -M, --no-colors     force print with no colors
-  -N, --no-doc        Don't print document separators (---)
-  -n, --null-input    Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.
-  -P, --prettyPrint    pretty print, shorthand for '... style = ""' 
-  -j, --tojson        output as json. Set indent to 0 to print json in one line.
-  -v, --verbose       verbose mode
-  -V, --version       Print version information and quit
+  -C, --colors                force print with colors
+  -e, --exit-status           set exit status if there are no matches or null or false is returned
+  -f, --front-matter string   (extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact
+  -h, --help                  help for yq
+  -I, --indent int            sets indent level for output (default 2)
+  -i, --inplace               update the yaml file inplace of first yaml file given.
+  -M, --no-colors             force print with no colors
+  -N, --no-doc                Don't print document separators (---)
+  -n, --null-input            Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.
+  -P, --prettyPrint           pretty print, shorthand for '... style = ""'
+  -j, --tojson                output as json. Set indent to 0 to print json in one line.
+      --unwrapScalar          unwrap scalar, print the value with no quotes, colors or comments (default true)
+  -v, --verbose               verbose mode
+  -V, --version               Print version information and quit
 
 Use "yq [command] --help" for more information about a command.
 ```
@@ -230,6 +233,5 @@ yq e '.a.b | length' f1.yml f2.yml
 ## Known Issues / Missing Features
 - `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details)
 - Powershell has its own...opinions: https://mikefarah.gitbook.io/yq/usage/tips-and-tricks#quotes-in-windows-powershell
-- Running expressions against blank files does not work, because the file is empty, there are no matches for yq to run through the expression pipeline and so nothing happens. Instead, you can do something like `yq e -n '.someNew="content"' > newfile.yml` to create a new file.
 
 See [tips and tricks](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks) for more common problems and solutions.

acceptance_tests/basic.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+setUp() {
+  rm -f test.yml
+}
+
+testBasicEvalRoundTrip() {
+  ./yq e -n ".a = 123" > test.yml
+  X=$(./yq e '.a' test.yml)
+  assertEquals 123 "$X"
+}
+
+testBasicUpdateInPlaceSequence() {
+  cat >test.yml <<EOL
+a: 0
+EOL
+  ./yq e -i ".a = 10" test.yml
+  X=$(./yq e '.a' test.yml)
+  assertEquals "10" "$X"
+}
+
+testBasicUpdateInPlaceSequenceEvalAll() {
+  cat >test.yml <<EOL
+a: 0
+EOL
+  ./yq ea -i ".a = 10" test.yml
+  X=$(./yq e '.a' test.yml)
+  assertEquals "10" "$X"
+}
+
+testBasicNoExitStatus() {
+  echo "a: cat" > test.yml
+  X=$(./yq e '.z' test.yml)
+  assertEquals "null" "$X"
+}
+
+testBasicExitStatus() {
+  echo "a: cat" > test.yml
+  X=$(./yq e -e '.z' test.yml 2&>/dev/null)
+  assertEquals 1 "$?"
+}
+
+testBasicExtractFieldWithSeperator() {
+    cat >test.yml <<EOL
+---
+name: chart-name
+version: 1.2.3
+EOL
+  X=$(./yq e '.name' test.yml)
+  assertEquals "chart-name" "$X"
+}
+
+testBasicExtractMultipleFieldWithSeperator() {
+    cat >test.yml <<EOL
+---
+name: chart-name
+version: 1.2.3
+---
+name: thing
+version: 1.2.3
+EOL
+
+read -r -d '' expected << EOM
+chart-name
+---
+thing
+EOM
+  X=$(./yq e '.name' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+source ./scripts/shunit2

acceptance_tests/empty.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+setUp() {
+  cat >test.yml <<EOL
+# comment
+EOL
+}
+
+testEmptyEval() {
+  X=$(./yq e test.yml)
+  expected=$(cat test.yml)
+  assertEquals 0 $?
+  assertEquals "$expected" "$X"
+}
+
+testEmptyEvalNoNewLine() {
+  echo -n "#comment" >test.yml
+  X=$(./yq e test.yml)
+  expected=$(cat test.yml)
+  assertEquals 0 $?
+  assertEquals "$expected" "$X"
+}
+
+testEmptyEvalNoNewLineWithExpression() {
+  echo -n "# comment" >test.yml
+  X=$(./yq e '.apple = "tree"' test.yml)
+  read -r -d '' expected << EOM
+# comment
+apple: tree
+EOM
+  assertEquals "$expected" "$X"
+}
+
+testEmptyEvalPipe() {
+  X=$(./yq e - < test.yml)
+  assertEquals 0 $?
+}
+
+testEmptyCommentsWithExpressionEval() {
+  read -r -d '' expected << EOM
+# comment
+apple: tree
+EOM
+
+  X=$(./yq e '.apple="tree"' test.yml)
+
+  assertEquals "$expected" "$X"
+}
+
+testEmptyCommentsWithExpressionEvalAll() {
+  read -r -d '' expected << EOM
+# comment
+apple: tree
+EOM
+
+  X=$(./yq ea '.apple="tree"' test.yml)
+
+  assertEquals "$expected" "$X"
+}
+
+testEmptyWithExpressionEval() {
+  rm test.yml
+  touch test.yml
+  expected="apple: tree"
+
+  X=$(./yq e '.apple="tree"' test.yml)
+
+  assertEquals "$expected" "$X"
+}
+
+testEmptyWithExpressionEvalAll() {
+  rm test.yml
+  touch test.yml
+  expected="apple: tree"
+
+  X=$(./yq ea '.apple="tree"' test.yml)
+
+  assertEquals "$expected" "$X"
+}
+
+
+source ./scripts/shunit2

acceptance_tests/front-matter.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+setUp() {
+  cat >test.yml <<EOL
+---
+a: apple
+b: cat
+---
+not yaml
+c: at
+EOL
+}
+
+testFrontMatterProcessEval() {
+  read -r -d '' expected << EOM
+---
+a: apple
+b: dog
+---
+not yaml
+c: at
+EOM
+  ./yq e --front-matter="process" '.b = "dog"' test.yml -i
+  assertEquals "$expected" "$(cat test.yml)"
+}
+
+testFrontMatterProcessEvalAll() {
+  read -r -d '' expected << EOM
+---
+a: apple
+b: dog
+---
+not yaml
+c: at
+EOM
+  ./yq ea --front-matter="process" '.b = "dog"' test.yml -i
+  assertEquals "$expected" "$(cat test.yml)"
+}
+
+testFrontMatterExtractEval() {
+    cat >test.yml <<EOL
+a: apple
+b: cat
+---
+not yaml
+c: at
+EOL
+
+  read -r -d '' expected << EOM
+a: apple
+b: dog
+EOM
+  ./yq e --front-matter="extract" '.b = "dog"' test.yml -i
+  assertEquals "$expected" "$(cat test.yml)"
+}
+
+testFrontMatterExtractEvalAll() {
+  cat >test.yml <<EOL
+a: apple
+b: cat
+---
+not yaml
+c: at
+EOL
+
+  read -r -d '' expected << EOM
+a: apple
+b: dog
+EOM
+  ./yq ea --front-matter="extract" '.b = "dog"' test.yml -i
+  assertEquals "$expected" "$(cat test.yml)"
+}
+
+
+source ./scripts/shunit2

acceptance_tests/leading-seperator.sh

@@ -0,0 +1,420 @@
+#!/bin/bash
+
+setUp() {
+  cat >test.yml <<EOL
+---
+a: test
+EOL
+}
+
+testLeadingSeperatorWithDoc() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+---
+a: test
+---
+b: cool
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+---
+a: thing
+---
+b: cool
+EOM
+
+  X=$(./yq e '(select(di == 0) | .a) = "thing"' - < test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorPipeIntoEvalSeq() {
+  X=$(./yq e - < test.yml)
+  expected=$(cat test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorExtractField() {
+  X=$(./yq e '.a' - < test.yml)
+  assertEquals "test" "$X"
+}
+
+testLeadingSeperatorExtractFieldWithCommentsAfterSep() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+EOL
+  X=$(./yq e '.a' test.yml)
+  assertEquals "test" "$X"
+}
+
+testLeadingSeperatorExtractFieldWithCommentsBeforeSep() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+---
+a: test
+EOL
+  X=$(./yq e '.a' test.yml)
+  assertEquals "test" "$X"
+}
+
+
+testLeadingSeperatorExtractFieldMultiDoc() {
+  cat >test.yml <<EOL
+---
+a: test
+---
+a: test2
+EOL
+
+  read -r -d '' expected << EOM
+test
+---
+test2
+EOM
+  X=$(./yq e '.a' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorExtractFieldMultiDocWithComments() {
+  cat >test.yml <<EOL
+# here
+---
+# there
+a: test
+# whereever
+---
+# you are
+a: test2
+# woop
+EOL
+
+  read -r -d '' expected << EOM
+test
+---
+test2
+EOM
+  X=$(./yq e '.a' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+
+testLeadingSeperatorEvalSeq() {
+  X=$(./yq e test.yml)
+  expected=$(cat test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorPipeIntoEvalAll() {
+  X=$(./yq ea - < test.yml)
+  expected=$(cat test.yml)
+  assertEquals "$expected" "$X"
+}
+
+
+testLeadingSeperatorEvalAll() {
+  X=$(./yq ea test.yml)
+  expected=$(cat test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalSimple() {
+  read -r -d '' expected << EOM
+---
+a: test
+---
+version: 3
+application: MyApp
+EOM
+
+
+  X=$(./yq e '.' test.yml examples/order.yaml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocInOneFile() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+---
+b: things
+EOL
+  expected=$(cat test.yml)
+  X=$(./yq e '.' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocInOneFileEvalAll() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+---
+b: things
+EOL
+  expected=$(cat test.yml)
+  X=$(./yq ea '.' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalComments() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+a: test
+EOL
+
+cat >test2.yml <<EOL
+# this is another doc
+# great
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+a: test
+---
+# this is another doc
+# great
+b: sane
+EOM
+
+
+  X=$(./yq e '.' test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalCommentsTrailingSep() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+---
+a: test
+EOL
+
+cat >test2.yml <<EOL
+# this is another doc
+# great
+---
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+---
+a: test
+---
+# this is another doc
+# great
+---
+b: sane
+EOM
+
+
+  X=$(./yq e '.' test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiMultiDocEvalCommentsTrailingSep() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+---
+a: test
+---
+a1: test2
+EOL
+
+cat >test2.yml <<EOL
+# this is another doc
+# great
+---
+b: sane
+---
+b2: cool
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+---
+a: test
+---
+a1: test2
+---
+# this is another doc
+# great
+---
+b: sane
+---
+b2: cool
+EOM
+
+
+  X=$(./yq e '.' test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalCommentsLeadingSep() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+EOL
+
+cat >test2.yml <<EOL
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+---
+# hi peeps
+# cool
+a: test
+---
+# this is another doc
+# great
+b: sane
+EOM
+
+
+  X=$(./yq e '.' test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalCommentsStripComments() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  # it will be hard to remove that top level separator
+  read -r -d '' expected << EOM
+a: test
+---
+b: sane
+EOM
+
+  X=$(./yq e '... comments=""'  test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalCommentsLeadingSepNoDocFlag() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+a: test
+# this is another doc
+# great
+b: sane
+EOM
+
+
+  X=$(./yq e '.' --no-doc test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalJsonFlag() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+EOL
+
+cat >test2.yml <<EOL
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+{
+  "a": "test"
+}
+{
+  "b": "sane"
+}
+EOM
+
+
+  X=$(./yq e '.' -j test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalAllJsonFlag() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+EOL
+
+cat >test2.yml <<EOL
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+{
+  "a": "test"
+}
+{
+  "b": "sane"
+}
+EOM
+
+
+  X=$(./yq ea '.' -j test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalAll() {
+  read -r -d '' expected << EOM
+---
+a: test
+---
+version: 3
+application: MyApp
+EOM
+
+
+  X=$(./yq ea '.' test.yml examples/order.yaml)
+  assertEquals "$expected" "$X"
+}
+
+source ./scripts/shunit2

cmd/constant.go

@@ -1,5 +1,6 @@
 package cmd
 
+var leadingContentPreProcessing = true
 var unwrapScalar = true
 
 var writeInplace = false
@@ -16,4 +17,7 @@ var verbose = false
 var version = false
 var prettyPrint = false
 
+// can be either "" (off), "extract" or "process"
+var frontMatter = ""
+
 var completedSuccessfully = false

cmd/evaluate_all_command.go

@@ -89,23 +89,39 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 
 	printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
 
+	if frontMatter != "" {
+		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])
+		err = frontMatterHandler.Split()
+		if err != nil {
+			return err
+		}
+		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
+
+		if frontMatter == "process" {
+			reader := frontMatterHandler.GetContentReader()
+			printer.SetAppendix(reader)
+			defer yqlib.SafelyCloseReader(reader)
+		}
+		defer frontMatterHandler.CleanUp()
+	}
+
 	allAtOnceEvaluator := yqlib.NewAllAtOnceEvaluator()
 	switch len(args) {
 	case 0:
 		if pipingStdIn {
-			err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer)
+			err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer, leadingContentPreProcessing)
 		} else {
 			cmd.Println(cmd.UsageString())
 			return nil
 		}
 	case 1:
 		if nullInput {
-			err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(args[0]), printer)
+			err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(args[0]), printer, "")
 		} else {
-			err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer)
+			err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer, leadingContentPreProcessing)
 		}
 	default:
-		err = allAtOnceEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer)
+		err = allAtOnceEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing)
 	}
 
 	completedSuccessfully = err == nil

cmd/evalute_sequence_command.go

@@ -103,22 +103,38 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 		return errors.New("Cannot pass files in when using null-input flag")
 	}
 
+	if frontMatter != "" {
+		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])
+		err = frontMatterHandler.Split()
+		if err != nil {
+			return err
+		}
+		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
+
+		if frontMatter == "process" {
+			reader := frontMatterHandler.GetContentReader()
+			printer.SetAppendix(reader)
+			defer yqlib.SafelyCloseReader(reader)
+		}
+		defer frontMatterHandler.CleanUp()
+	}
+
 	switch len(args) {
 	case 0:
 		if pipingStdIn {
-			err = streamEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer)
+			err = streamEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer, leadingContentPreProcessing)
 		} else {
 			cmd.Println(cmd.UsageString())
 			return nil
 		}
 	case 1:
 		if nullInput {
-			err = streamEvaluator.EvaluateNew(processExpression(args[0]), printer)
+			err = streamEvaluator.EvaluateNew(processExpression(args[0]), printer, "")
 		} else {
-			err = streamEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer)
+			err = streamEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer, leadingContentPreProcessing)
 		}
 	default:
-		err = streamEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer)
+		err = streamEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing)
 	}
 	completedSuccessfully = err == nil
 

cmd/root.go

@@ -54,6 +54,8 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 
 	rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")
 	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
+	rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
+	rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and seperators before processing expression. This is a workaround for go-yaml to persist header content. This flag will be removed once this feature has been out in the wild for a while.")
 	rootCmd.AddCommand(
 		createEvaluateSequenceCommand(),
 		createEvaluateAllCommand(),

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.10.0"
+	Version = "4.11.1"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

examples/data1.yaml

@@ -1,2 +1,4 @@
 ---
-{"a":{"b":1}}
+# hi peeps
+# cool
+a: test

examples/empty.yaml

@@ -0,0 +1 @@
+# comment

examples/front-matter.yaml

@@ -0,0 +1,6 @@
+---
+a: apple
+b: bannana
+---
+hello there
+apples: great

github-action/Dockerfile

@@ -1,4 +1,4 @@
-FROM mikefarah/yq:4.10.0
+FROM mikefarah/yq:4.11.1
 
 COPY entrypoint.sh /entrypoint.sh
 

pkg/yqlib/all_at_once_evaluator.go

@@ -8,7 +8,7 @@ import (
 
 // A yaml expression evaluator that runs the expression once against all files/nodes in memory.
 type Evaluator interface {
-	EvaluateFiles(expression string, filenames []string, printer Printer) error
+	EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error
 
 	// EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes
 	EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error)
@@ -46,19 +46,19 @@ func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCand
 	return context.MatchingNodes, nil
 }
 
-func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error {
+func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error {
 	fileIndex := 0
-	firstFileLeadingSeperator := false
+	firstFileLeadingContent := ""
 
 	var allDocuments *list.List = list.New()
 	for _, filename := range filenames {
-		reader, leadingSeperator, err := readStream(filename)
+		reader, leadingContent, err := readStream(filename, leadingContentPreProcessing)
 		if err != nil {
 			return err
 		}
 
-		if fileIndex == 0 && leadingSeperator {
-			firstFileLeadingSeperator = leadingSeperator
+		if fileIndex == 0 {
+			firstFileLeadingContent = leadingContent
 		}
 
 		fileDocuments, err := readDocuments(reader, filename, fileIndex)
@@ -73,16 +73,17 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
 		candidateNode := &CandidateNode{
 			Document:  0,
 			Filename:  "",
-			Node:      &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode},
+			Node:      &yaml.Node{Kind: yaml.DocumentNode, HeadComment: firstFileLeadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
 			FileIndex: 0,
 		}
 		allDocuments.PushBack(candidateNode)
+	} else {
+		allDocuments.Front().Value.(*CandidateNode).Node.HeadComment = firstFileLeadingContent
 	}
 
 	matches, err := e.EvaluateCandidateNodes(expression, allDocuments)
 	if err != nil {
 		return err
 	}
-	printer.SetPrintLeadingSeperator(firstFileLeadingSeperator)
 	return printer.PrintResults(matches)
 }

pkg/yqlib/file_utils.go

@@ -2,6 +2,7 @@ package yqlib
 
 import (
 	"io"
+	"io/ioutil"
 	"os"
 )
 
@@ -23,6 +24,14 @@ func safelyRenameFile(from string, to string) {
 	}
 }
 
+func tryRemoveFile(filename string) {
+	log.Debug("Removing temp file: %v", filename)
+	removeErr := os.Remove(filename)
+	if removeErr != nil {
+		log.Errorf("Failed to remove temp file: %v", filename)
+	}
+}
+
 // thanks https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang
 func copyFileContents(src, dst string) (err error) {
 	// ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory,
@@ -44,6 +53,13 @@ func copyFileContents(src, dst string) (err error) {
 	return out.Sync()
 }
 
+func SafelyCloseReader(reader io.Reader) {
+	switch reader := reader.(type) {
+	case *os.File:
+		safelyCloseFile(reader)
+	}
+}
+
 func safelyCloseFile(file *os.File) {
 	err := file.Close()
 	if err != nil {
@@ -51,3 +67,22 @@ func safelyCloseFile(file *os.File) {
 		log.Error(err.Error())
 	}
 }
+
+func createTempFile() (*os.File, error) {
+	_, err := os.Stat(os.TempDir())
+	if os.IsNotExist(err) {
+		err = os.Mkdir(os.TempDir(), 0700)
+		if err != nil {
+			return nil, err
+		}
+	} else if err != nil {
+		return nil, err
+	}
+
+	file, err := ioutil.TempFile("", "temp")
+	if err != nil {
+		return nil, err
+	}
+
+	return file, err
+}

pkg/yqlib/front_matter.go

@@ -0,0 +1,93 @@
+package yqlib
+
+import (
+	"bufio"
+	"io"
+	"os"
+)
+
+type frontMatterHandler interface {
+	Split() error
+	GetYamlFrontMatterFilename() string
+	GetContentReader() io.Reader
+	CleanUp()
+}
+
+type frontMatterHandlerImpl struct {
+	originalFilename        string
+	yamlFrontMatterFilename string
+	contentReader           io.Reader
+}
+
+func NewFrontMatterHandler(originalFilename string) frontMatterHandler {
+	return &frontMatterHandlerImpl{originalFilename, "", nil}
+}
+
+func (f *frontMatterHandlerImpl) GetYamlFrontMatterFilename() string {
+	return f.yamlFrontMatterFilename
+}
+
+func (f *frontMatterHandlerImpl) GetContentReader() io.Reader {
+	return f.contentReader
+}
+
+func (f *frontMatterHandlerImpl) CleanUp() {
+	tryRemoveFile(f.yamlFrontMatterFilename)
+}
+
+// Splits the given file by yaml front matter
+// yaml content will be saved to first temporary file
+// remaining content will be saved to second temporary file
+func (f *frontMatterHandlerImpl) Split() error {
+	var reader *bufio.Reader
+	var err error
+	if f.originalFilename == "-" {
+		reader = bufio.NewReader(os.Stdin)
+	} else {
+		file, err := os.Open(f.originalFilename) // #nosec
+		if err != nil {
+			return err
+		}
+		reader = bufio.NewReader(file)
+	}
+	f.contentReader = reader
+
+	yamlTempFile, err := createTempFile()
+	if err != nil {
+		return err
+	}
+	f.yamlFrontMatterFilename = yamlTempFile.Name()
+	log.Debug("yamlTempFile: %v", yamlTempFile.Name())
+
+	lineCount := 0
+
+	for {
+		peekBytes, err := reader.Peek(3)
+		if err == io.EOF {
+			// we've finished reading the yaml content..I guess
+			break
+		} else if err != nil {
+			return err
+		}
+		if lineCount > 0 && string(peekBytes) == "---" {
+			// we've finished reading the yaml content..
+			break
+		}
+		line, errReading := reader.ReadString('\n')
+		lineCount = lineCount + 1
+		if errReading != nil && errReading != io.EOF {
+			return errReading
+		}
+
+		_, errWriting := yamlTempFile.Write([]byte(line))
+
+		if errWriting != nil {
+			return errWriting
+		}
+	}
+
+	safelyCloseFile(yamlTempFile)
+
+	return nil
+
+}

pkg/yqlib/front_matter_test.go

@@ -0,0 +1,142 @@
+package yqlib
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+func createTestFile(content string) string {
+	tempFile, err := createTempFile()
+	if err != nil {
+		panic(err)
+	}
+
+	_, err = tempFile.Write([]byte(content))
+	if err != nil {
+		panic(err)
+	}
+
+	safelyCloseFile(tempFile)
+
+	return tempFile.Name()
+}
+
+func readFile(filename string) string {
+	bytes, err := ioutil.ReadFile(filename)
+	if err != nil {
+		panic(err)
+	}
+	return string(bytes)
+}
+
+func TestFrontMatterSplitWithLeadingSep(t *testing.T) {
+	file := createTestFile(`---
+a: apple
+b: banana
+---
+not a 
+yaml: doc
+`)
+
+	expectedYamlFm := `---
+a: apple
+b: banana
+`
+
+	expectedContent := `---
+not a 
+yaml: doc
+`
+
+	fmHandler := NewFrontMatterHandler(file)
+	err := fmHandler.Split()
+	if err != nil {
+		panic(err)
+	}
+
+	yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename())
+
+	test.AssertResult(t, expectedYamlFm, yamlFm)
+
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
+	if err != nil {
+		panic(err)
+	}
+	test.AssertResult(t, expectedContent, string(contentBytes))
+
+	tryRemoveFile(file)
+	fmHandler.CleanUp()
+}
+
+func TestFrontMatterSplitWithNoLeadingSep(t *testing.T) {
+	file := createTestFile(`a: apple
+b: banana
+---
+not a 
+yaml: doc
+`)
+
+	expectedYamlFm := `a: apple
+b: banana
+`
+
+	expectedContent := `---
+not a 
+yaml: doc
+`
+
+	fmHandler := NewFrontMatterHandler(file)
+	err := fmHandler.Split()
+	if err != nil {
+		panic(err)
+	}
+
+	yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename())
+
+	test.AssertResult(t, expectedYamlFm, yamlFm)
+
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
+	if err != nil {
+		panic(err)
+	}
+	test.AssertResult(t, expectedContent, string(contentBytes))
+
+	tryRemoveFile(file)
+	fmHandler.CleanUp()
+}
+
+func TestFrontMatterSplitWithArray(t *testing.T) {
+	file := createTestFile(`[1,2,3]
+---
+not a 
+yaml: doc
+`)
+
+	expectedYamlFm := "[1,2,3]\n"
+
+	expectedContent := `---
+not a 
+yaml: doc
+`
+
+	fmHandler := NewFrontMatterHandler(file)
+	err := fmHandler.Split()
+	if err != nil {
+		panic(err)
+	}
+
+	yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename())
+
+	test.AssertResult(t, expectedYamlFm, yamlFm)
+
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
+	if err != nil {
+		panic(err)
+	}
+	test.AssertResult(t, expectedContent, string(contentBytes))
+
+	tryRemoveFile(file)
+	fmHandler.CleanUp()
+}

pkg/yqlib/operator_multiply.go

@@ -31,7 +31,7 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
 			(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
 
 			var newBlank = lhs.CreateChild(nil, &yaml.Node{})
-
+			log.Debugf("merge - merge lhs into blank")
 			var newThing, err = mergeObjects(d, context.WritableClone(), newBlank, lhs, multiplyPreferences{})
 			if err != nil {
 				return nil, err
@@ -83,12 +83,13 @@ func multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, e
 }
 
 func mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode, preferences multiplyPreferences) (*CandidateNode, error) {
-	shouldAppendArrays := preferences.AppendArrays
 	var results = list.New()
 
-	// shouldn't recurse arrays if appending
-	prefs := recursiveDescentPreferences{RecurseArray: !shouldAppendArrays,
+	// only need to recurse the array if we are doing a deep merge
+	prefs := recursiveDescentPreferences{RecurseArray: preferences.DeepMergeArrays,
 		TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: true}}
+	log.Debugf("merge - preferences.DeepMergeArrays %v", preferences.DeepMergeArrays)
+	log.Debugf("merge - preferences.AppendArrays %v", preferences.AppendArrays)
 	err := recursiveDecent(d, results, context.SingleChildContext(rhs), prefs)
 	if err != nil {
 		return nil, err
@@ -118,14 +119,22 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom
 	log.Debugf("merge - applyAssignment lhs %v, rhs: %v", lhs.GetKey(), rhs.GetKey())
 
 	lhsPath := rhs.Path[pathIndexToStartFrom:]
+	log.Debugf("merge - lhsPath %v", lhsPath)
 
 	assignmentOp := &Operation{OperationType: assignAttributesOpType}
 	if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode {
 		assignmentOp.OperationType = addAssignOpType
+		log.Debugf("merge - assignmentOp.OperationType = addAssignOpType")
 	} else if !preferences.DeepMergeArrays && rhs.Node.Kind == yaml.SequenceNode ||
 		(rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode) {
 		assignmentOp.OperationType = assignOpType
 		assignmentOp.UpdateAssign = false
+		log.Debugf("merge - rhs.Node.Kind == yaml.SequenceNode: %v", rhs.Node.Kind == yaml.SequenceNode)
+		log.Debugf("merge - rhs.Node.Kind == yaml.ScalarNode: %v", rhs.Node.Kind == yaml.ScalarNode)
+		log.Debugf("merge - rhs.Node.Kind == yaml.AliasNode: %v", rhs.Node.Kind == yaml.AliasNode)
+		log.Debugf("merge - assignmentOp.OperationType = assignOpType, no updateassign")
+	} else {
+		log.Debugf("merge - assignmentOp := &Operation{OperationType: assignAttributesOpType}")
 	}
 	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs}
 

pkg/yqlib/operator_multiply_test.go

@@ -24,6 +24,11 @@ list2:
     - "123"
 `
 
+var mergeArrayWithAnchors = `sample:
+- &a
+- <<: *a
+`
+
 var mergeArraysObjectKeysText = `It's a complex command, the trickyness comes from needing to have the right context in the expressions.
 First we save the second array into a variable '$two' which lets us reference it later.
 We then need to update the first array. We will use the relative update (|=) because we need to update relative to the current element of the array in the LHS in the RHS expression. 
@@ -31,6 +36,14 @@ We set the current element of the first array as $cur. Now we multiply (merge) $
 `
 
 var multiplyOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   mergeArrayWithAnchors,
+		expression: `. * .`,
+		expected: []string{
+			"D0, P[], (!!map)::sample:\n    - &a\n    - !!merge <<: *a\n",
+		},
+	},
 	{
 		description: "Multiply integers",
 		expression:  `3 * 4`,

pkg/yqlib/operators_test.go

@@ -114,7 +114,7 @@ func formatYaml(yaml string, filename string) string {
 		panic(err)
 	}
 	streamEvaluator := NewStreamEvaluator()
-	_, err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer)
+	_, err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer, "")
 	if err != nil {
 		panic(err)
 	}

pkg/yqlib/printer.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"container/list"
 	"io"
+	"strings"
 
 	yaml "gopkg.in/yaml.v3"
 )
@@ -11,7 +12,8 @@ import (
 type Printer interface {
 	PrintResults(matchingNodes *list.List) error
 	PrintedAnything() bool
-	SetPrintLeadingSeperator(bool)
+	//e.g. when given a front-matter doc, like jekyll
+	SetAppendix(reader io.Reader)
 }
 
 type resultsPrinter struct {
@@ -26,6 +28,7 @@ type resultsPrinter struct {
 	previousFileIndex  int
 	printedMatches     bool
 	treeNavigator      DataTreeNavigator
+	appendixReader     io.Reader
 }
 
 func NewPrinter(writer io.Writer, outputToJSON bool, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
@@ -41,11 +44,8 @@ func NewPrinter(writer io.Writer, outputToJSON bool, unwrapScalar bool, colorsEn
 	}
 }
 
-func (p *resultsPrinter) SetPrintLeadingSeperator(printLeadingSeperator bool) {
-	if printLeadingSeperator {
-		p.firstTimePrinting = false
-		p.previousFileIndex = -1
-	}
+func (p *resultsPrinter) SetAppendix(reader io.Reader) {
+	p.appendixReader = reader
 }
 
 func (p *resultsPrinter) PrintedAnything() bool {
@@ -109,13 +109,55 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 	for el := matchingNodes.Front(); el != nil; el = el.Next() {
 		mappedDoc := el.Value.(*CandidateNode)
 		log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v, printDocSeparators: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document, p.printDocSeparators)
-		if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators {
+
+		commentStartsWithSeparator := strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$\n$yqDocSeperator$")
+
+		if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators && !commentStartsWithSeparator {
 			log.Debug("-- writing doc sep")
 			if err := p.writeString(bufferedWriter, "---\n"); err != nil {
 				return err
 			}
 		}
 
+		if strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$") {
+			log.Debug("headcommentwas %v", mappedDoc.Node.HeadComment)
+			log.Debug("finished headcomment")
+			reader := bufio.NewReader(strings.NewReader(mappedDoc.Node.HeadComment))
+			mappedDoc.Node.HeadComment = ""
+
+			for {
+
+				readline, errReading := reader.ReadString('\n')
+				if errReading != nil && errReading != io.EOF {
+					return errReading
+				}
+				if strings.Contains(readline, "$yqLeadingContent$") {
+					// skip this
+
+				} else if strings.Contains(readline, "$yqDocSeperator$") {
+					if p.printDocSeparators {
+						if err := p.writeString(bufferedWriter, "---\n"); err != nil {
+							return err
+						}
+					}
+				} else if !p.outputToJSON {
+					if err := p.writeString(bufferedWriter, readline); err != nil {
+						return err
+					}
+				}
+
+				if errReading == io.EOF {
+					if readline != "" {
+						// the last comment we read didn't have a new line, put one in
+						if err := p.writeString(bufferedWriter, "\n"); err != nil {
+							return err
+						}
+					}
+					break
+				}
+			}
+
+		}
 		if err := p.printNode(mappedDoc.Node, bufferedWriter); err != nil {
 			return err
 		}
@@ -123,5 +165,14 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 		p.previousDocIndex = mappedDoc.Document
 	}
 
+	if p.appendixReader != nil && !p.outputToJSON {
+		log.Debug("Piping appendix reader...")
+		betterReader := bufio.NewReader(p.appendixReader)
+		_, err := io.Copy(bufferedWriter, betterReader)
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }

pkg/yqlib/printer_test.go

@@ -17,8 +17,14 @@ a: apple
 a: coconut
 `
 
-var leadingSeperatorSample = `---
-a: good doc
+var multiDocSampleLeadingExpected = `# go cats
+---
+a: banana
+---
+a: apple
+---
+# cool
+a: coconut
 `
 
 func nodeToList(candidate *CandidateNode) *list.List {
@@ -27,26 +33,6 @@ func nodeToList(candidate *CandidateNode) *list.List {
 	return elMap
 }
 
-func TestPrinterWithLeadingSeperator(t *testing.T) {
-	var output bytes.Buffer
-	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, false, true, false, 2, true)
-
-	inputs, err := readDocuments(strings.NewReader(leadingSeperatorSample), "sample.yml", 0)
-	if err != nil {
-		panic(err)
-	}
-	printer.SetPrintLeadingSeperator(true)
-
-	err = printer.PrintResults(inputs)
-	if err != nil {
-		panic(err)
-	}
-
-	writer.Flush()
-	test.AssertResult(t, leadingSeperatorSample, output.String())
-}
-
 func TestPrinterMultipleDocsInSequence(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
@@ -85,6 +71,48 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) {
 	test.AssertResult(t, multiDocSample, output.String())
 }
 
+func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
+	var output bytes.Buffer
+	var writer = bufio.NewWriter(&output)
+	printer := NewPrinter(writer, false, true, false, 2, true)
+
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
+	if err != nil {
+		panic(err)
+	}
+
+	el := inputs.Front()
+	el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
+	sample1 := nodeToList(el.Value.(*CandidateNode))
+
+	el = el.Next()
+	el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
+	sample2 := nodeToList(el.Value.(*CandidateNode))
+
+	el = el.Next()
+	el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n# cool\n"
+	sample3 := nodeToList(el.Value.(*CandidateNode))
+
+	err = printer.PrintResults(sample1)
+	if err != nil {
+		panic(err)
+	}
+
+	err = printer.PrintResults(sample2)
+	if err != nil {
+		panic(err)
+	}
+
+	err = printer.PrintResults(sample3)
+	if err != nil {
+		panic(err)
+	}
+
+	writer.Flush()
+
+	test.AssertResult(t, multiDocSampleLeadingExpected, output.String())
+}
+
 func TestPrinterMultipleFilesInSequence(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
@@ -132,6 +160,56 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) {
 	test.AssertResult(t, multiDocSample, output.String())
 }
 
+func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
+	var output bytes.Buffer
+	var writer = bufio.NewWriter(&output)
+	printer := NewPrinter(writer, false, true, false, 2, true)
+
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
+	if err != nil {
+		panic(err)
+	}
+
+	el := inputs.Front()
+	elNode := el.Value.(*CandidateNode)
+	elNode.Document = 0
+	elNode.FileIndex = 0
+	elNode.Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
+	sample1 := nodeToList(elNode)
+
+	el = el.Next()
+	elNode = el.Value.(*CandidateNode)
+	elNode.Document = 0
+	elNode.FileIndex = 1
+	elNode.Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
+	sample2 := nodeToList(elNode)
+
+	el = el.Next()
+	elNode = el.Value.(*CandidateNode)
+	elNode.Document = 0
+	elNode.FileIndex = 2
+	elNode.Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n# cool\n"
+	sample3 := nodeToList(elNode)
+
+	err = printer.PrintResults(sample1)
+	if err != nil {
+		panic(err)
+	}
+
+	err = printer.PrintResults(sample2)
+	if err != nil {
+		panic(err)
+	}
+
+	err = printer.PrintResults(sample3)
+	if err != nil {
+		panic(err)
+	}
+
+	writer.Flush()
+	test.AssertResult(t, multiDocSampleLeadingExpected, output.String())
+}
+
 func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
@@ -151,6 +229,86 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 	test.AssertResult(t, multiDocSample, output.String())
 }
 
+func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
+	var output bytes.Buffer
+	var writer = bufio.NewWriter(&output)
+	printer := NewPrinter(writer, false, true, false, 2, true)
+
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
+	if err != nil {
+		panic(err)
+	}
+
+	inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
+
+	err = printer.PrintResults(inputs)
+	if err != nil {
+		panic(err)
+	}
+
+	writer.Flush()
+	expected := `# go cats
+---
+a: banana
+---
+a: apple
+---
+a: coconut
+`
+	test.AssertResult(t, expected, output.String())
+}
+
+func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
+	var output bytes.Buffer
+	var writer = bufio.NewWriter(&output)
+	printer := NewPrinter(writer, false, true, false, 2, true)
+
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
+	if err != nil {
+		panic(err)
+	}
+	inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
+	err = printer.PrintResults(inputs)
+	if err != nil {
+		panic(err)
+	}
+
+	writer.Flush()
+	expected := `---
+a: banana
+---
+a: apple
+---
+a: coconut
+`
+	test.AssertResult(t, expected, output.String())
+}
+
+func TestPrinterScalarWithLeadingCont(t *testing.T) {
+	var output bytes.Buffer
+	var writer = bufio.NewWriter(&output)
+	printer := NewPrinter(writer, false, true, false, 2, true)
+
+	node, err := NewExpressionParser().ParseExpression(".a")
+	if err != nil {
+		panic(err)
+	}
+	streamEvaluator := NewStreamEvaluator()
+	_, err = streamEvaluator.Evaluate("sample", strings.NewReader(multiDocSample), node, printer, "# blah\n")
+	if err != nil {
+		panic(err)
+	}
+
+	writer.Flush()
+	expected := `banana
+---
+apple
+---
+coconut
+`
+	test.AssertResult(t, expected, output.String())
+}
+
 func TestPrinterMultipleDocsJson(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
@@ -163,6 +321,8 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 		panic(err)
 	}
 
+	inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# ignore this\n"
+
 	err = printer.PrintResults(inputs)
 	if err != nil {
 		panic(err)

pkg/yqlib/stream_evaluator.go

@@ -12,9 +12,9 @@ import (
 // Uses less memory than loading all documents and running the expression once, but this cannot process
 // cross document expressions.
 type StreamEvaluator interface {
-	Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer) (uint, error)
-	EvaluateFiles(expression string, filenames []string, printer Printer) error
-	EvaluateNew(expression string, printer Printer) error
+	Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, leadingContent string) (uint, error)
+	EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error
+	EvaluateNew(expression string, printer Printer, leadingContent string) error
 }
 
 type streamEvaluator struct {
@@ -27,7 +27,7 @@ func NewStreamEvaluator() StreamEvaluator {
 	return &streamEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: NewExpressionParser()}
 }
 
-func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error {
+func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadingContent string) error {
 	node, err := s.treeCreator.ParseExpression(expression)
 	if err != nil {
 		return err
@@ -35,7 +35,7 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error
 	candidateNode := &CandidateNode{
 		Document:  0,
 		Filename:  "",
-		Node:      &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode},
+		Node:      &yaml.Node{Kind: yaml.DocumentNode, HeadComment: leadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
 		FileIndex: 0,
 	}
 	inputList := list.New()
@@ -48,22 +48,26 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error
 	return printer.PrintResults(result.MatchingNodes)
 }
 
-func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error {
+func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error {
 	var totalProcessDocs uint = 0
 	node, err := s.treeCreator.ParseExpression(expression)
 	if err != nil {
 		return err
 	}
 
+	var firstFileLeadingContent string
+
 	for index, filename := range filenames {
-		reader, leadingSeperator, err := readStream(filename)
-		if index == 0 && leadingSeperator {
-			printer.SetPrintLeadingSeperator(leadingSeperator)
+		reader, leadingContent, err := readStream(filename, leadingContentPreProcessing)
+
+		if index == 0 {
+			firstFileLeadingContent = leadingContent
 		}
+
 		if err != nil {
 			return err
 		}
-		processedDocs, err := s.Evaluate(filename, reader, node, printer)
+		processedDocs, err := s.Evaluate(filename, reader, node, printer, leadingContent)
 		if err != nil {
 			return err
 		}
@@ -76,13 +80,13 @@ func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, p
 	}
 
 	if totalProcessDocs == 0 {
-		return s.EvaluateNew(expression, printer)
+		return s.EvaluateNew(expression, printer, firstFileLeadingContent)
 	}
 
 	return nil
 }
 
-func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer) (uint, error) {
+func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, leadingContent string) (uint, error) {
 
 	var currentIndex uint
 	decoder := yaml.NewDecoder(reader)
@@ -96,6 +100,9 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr
 		} else if errorReading != nil {
 			return currentIndex, errorReading
 		}
+		if currentIndex == 0 {
+			dataBucket.HeadComment = leadingContent
+		}
 		candidateNode := &CandidateNode{
 			Document:  currentIndex,
 			Filename:  filename,
@@ -110,6 +117,7 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr
 			return currentIndex, errorParsing
 		}
 		err := printer.PrintResults(result.MatchingNodes)
+
 		if err != nil {
 			return currentIndex, err
 		}

pkg/yqlib/utils.go

@@ -5,41 +5,62 @@ import (
 	"container/list"
 	"io"
 	"os"
+	"regexp"
+	"strings"
 
 	yaml "gopkg.in/yaml.v3"
 )
 
-func readStream(filename string) (io.Reader, bool, error) {
-
+func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, string, error) {
+	var reader *bufio.Reader
 	if filename == "-" {
-		reader := bufio.NewReader(os.Stdin)
-
-		seperatorBytes, err := reader.Peek(3)
-
-		if err == io.EOF {
-			// EOF are handled else where..
-			return reader, false, nil
-		}
-
-		return reader, string(seperatorBytes) == "---", err
+		reader = bufio.NewReader(os.Stdin)
 	} else {
 		// ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory,
 		// and ensuring that it's not possible to give a path to a file outside thar directory.
-		reader, err := os.Open(filename) // #nosec
+		file, err := os.Open(filename) // #nosec
 		if err != nil {
-			return nil, false, err
+			return nil, "", err
 		}
-		seperatorBytes := make([]byte, 3)
-		_, err = reader.Read(seperatorBytes)
+		reader = bufio.NewReader(file)
+	}
+
+	if !leadingContentPreProcessing {
+		return reader, "", nil
+	}
+	return processReadStream(reader)
+}
+
+func processReadStream(reader *bufio.Reader) (io.Reader, string, error) {
+	var commentLineRegEx = regexp.MustCompile(`^\s*#`)
+	var sb strings.Builder
+	sb.WriteString("$yqLeadingContent$\n")
+	for {
+		peekBytes, err := reader.Peek(3)
 		if err == io.EOF {
 			// EOF are handled else where..
-			return reader, false, nil
+			return reader, sb.String(), nil
 		} else if err != nil {
-			return nil, false, err
+			return reader, sb.String(), err
+		} else if string(peekBytes) == "---" {
+			_, err := reader.ReadString('\n')
+			sb.WriteString("$yqDocSeperator$\n")
+			if err == io.EOF {
+				return reader, sb.String(), nil
+			} else if err != nil {
+				return reader, sb.String(), err
+			}
+		} else if commentLineRegEx.MatchString(string(peekBytes)) {
+			line, err := reader.ReadString('\n')
+			sb.WriteString(line)
+			if err == io.EOF {
+				return reader, sb.String(), nil
+			} else if err != nil {
+				return reader, sb.String(), err
+			}
+		} else {
+			return reader, sb.String(), nil
 		}
-		_, err = reader.Seek(0, 0)
-
-		return reader, string(seperatorBytes) == "---", err
 	}
 }
 

pkg/yqlib/write_in_place_handler.go

@@ -1,7 +1,6 @@
 package yqlib
 
 import (
-	"io/ioutil"
 	"os"
 )
 
@@ -21,27 +20,21 @@ func NewWriteInPlaceHandler(inputFile string) writeInPlaceHandler {
 }
 
 func (w *writeInPlaceHandlerImpl) CreateTempFile() (*os.File, error) {
+	file, err := createTempFile()
+
+	if err != nil {
+		return nil, err
+	}
 	info, err := os.Stat(w.inputFilename)
 	if err != nil {
 		return nil, err
 	}
-	_, err = os.Stat(os.TempDir())
-	if os.IsNotExist(err) {
-		err = os.Mkdir(os.TempDir(), 0700)
-		if err != nil {
-			return nil, err
-		}
-	} else if err != nil {
-		return nil, err
-	}
+	err = os.Chmod(file.Name(), info.Mode())
 
-	file, err := ioutil.TempFile("", "temp")
 	if err != nil {
 		return nil, err
 	}
-
-	err = os.Chmod(file.Name(), info.Mode())
-	log.Debug("writing to tempfile: %v", file.Name())
+	log.Debug("WriteInPlaceHandler: writing to tempfile: %v", file.Name())
 	w.tempFile = file
 	return file, err
 }
@@ -50,13 +43,9 @@ func (w *writeInPlaceHandlerImpl) FinishWriteInPlace(evaluatedSuccessfully bool)
 	log.Debug("Going to write-inplace, evaluatedSuccessfully=%v, target=%v", evaluatedSuccessfully, w.inputFilename)
 	safelyCloseFile(w.tempFile)
 	if evaluatedSuccessfully {
-		log.Debug("moved temp file to target")
+		log.Debug("Moving temp file to target")
 		safelyRenameFile(w.tempFile.Name(), w.inputFilename)
 	} else {
-		log.Debug("removed temp file")
-		removeErr := os.Remove(w.tempFile.Name())
-		if removeErr != nil {
-			log.Errorf("failed removing temp file: %s", w.tempFile.Name())
-		}
+		tryRemoveFile(w.tempFile.Name())
 	}
 }

scripts/acceptance.sh

@@ -1,159 +1,10 @@
-#!/bin/bash
-
+#! /bin/bash
 set -e
 
-# acceptance test
+for test in acceptance_tests/*.sh; do
+  echo "--------------------------------------------------------------"
+  echo "$test"
+  echo "--------------------------------------------------------------"
+  (exec "$test");
+done
 
-
-
-echo "test eval-sequence"
-random=$((1 + $RANDOM % 10))
-./yq e -n ".a = $random" > test.yml
-X=$(./yq e '.a' test.yml)
-
-if [[ $X != $random ]]; then
-  echo "Failed create: expected $random but was $X"
-  exit 1
-fi
-
-echo "--success"
-
-echo "test update-in-place"
-
-update=$(($random + 1))
-./yq e -i ".a = $update" test.yml
-
-X=$(./yq e '.a' test.yml)
-if [[ $X != $update ]]; then
-  echo "Failed to update inplace test: expected $update but was $X"
-  exit 1
-fi
-
-echo "--success"
-
-echo "test eval-all"
-./yq ea -n ".a = $random" > test-eval-all.yml
-Y=$(./yq ea '.a' test-eval-all.yml)
-
-if [[ $Y != $random ]]; then
-  echo "Failed create with eval all: expected $random but was $X"
-  exit 1
-fi
-echo "--success"
-
-echo "test no exit status"
-./yq e '.z' test.yml
-echo "--success"
-
-echo "test exit status"
-set +e
-
-./yq e -e '.z' test.yml
-
-if [[ $? != 1 ]]; then
-  echo "Expected error code 1 but was $?"
-  exit 1
-fi
-
-echo "Test: leading seperator logic"
-expected=$(cat examples/leading-seperator.yaml)
-
-X=$(cat examples/leading-seperator.yaml | ./yq e '.' -)
-if [[ $X != $expected ]]; then
-  echo "Pipe into e"
-  echo "Expected $expected but was $X"
-  exit 1
-fi
-
-X=$(./yq e '.' examples/leading-seperator.yaml)
-expected=$(cat examples/leading-seperator.yaml)
-if [[ $X != $expected ]]; then
-  echo "read given file e"
-  echo "Expected $expected but was $X"
-  exit 1
-fi
-
-X=$(cat examples/leading-seperator.yaml | ./yq ea '.' -)
-if [[ $X != $expected ]]; then
-  echo "Pipe into e"
-  echo "Expected $expected but was $X"
-  exit 1
-fi
-
-X=$(./yq ea '.' examples/leading-seperator.yaml)
-expected=$(cat examples/leading-seperator.yaml)
-if [[ $X != $expected ]]; then
-  echo "read given file e"
-  echo "Expected $expected but was $X"
-  exit 1
-fi
-
-# multidoc
-read -r -d '' expected << EOM
----
-a: test
----
-version: 3
-application: MyApp
-EOM
-
-X=$(./yq e '.' examples/leading-seperator.yaml examples/order.yaml)
-
-if [[ $X != $expected ]]; then
-  echo "Multidoc with leading seperator"
-  echo "Expected $expected but was $X"
-  exit 1
-fi
-
-X=$(./yq ea '.' examples/leading-seperator.yaml examples/order.yaml)
-
-if [[ $X != $expected ]]; then
-  echo "Multidoc with leading seperator"
-  echo "Expected $expected but was $X"
-  exit 1
-fi
-
-echo "Test: handle empty files"
-./yq e '.' examples/empty.yaml
-if [[ $? != 0 ]]; then
-  echo "Expected no error when processing empty file but got one"
-  exit 1
-fi
-
-cat examples/empty.yaml | ./yq e '.' -
-if [[ $? != 0 ]]; then
-  echo "Expected no error when processing empty stdin but got one"
-  exit 1
-fi
-
-# run expression against empty file
-touch temp.yaml
-expected="apple: tree"
-
-./yq e '.apple = "tree"' temp.yaml -i
-X=$(cat temp.yaml)
-rm temp.yaml
-if [[ $X != $expected ]]; then
-  echo "Write empty doc"
-  echo "Expected $expected but was $X"
-  exit 1
-fi
-
-touch temp.yaml
-
-./yq ea '.apple = "tree"' temp.yaml -i
-X=$(cat temp.yaml)
-rm temp.yaml
-if [[ $X != $expected ]]; then
-  echo "Write all empty doc"
-  echo "Expected $expected but was $X"
-  exit 1
-fi
-
-echo "--success"
-
-set -e
-
-rm test.yml
-rm test-eval-all.yml
-echo "acceptance tests passed"

scripts/secure.sh

@@ -5,7 +5,7 @@ set -o pipefail
 
 if command -v gosec &> /dev/null
 then
-    gosec ${PWD} ./...
+    gosec "${PWD}" ./...
 else
-  ./bin/gosec ${PWD} ./...
+  ./bin/gosec "${PWD}" ./...
 fi

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.10.0'
+version: '4.11.1'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.