Front matter processor seems to be working!

This reverts commit ffb6cbf769.

.github/workflows/test-yq.yml

@@ -0,0 +1,19 @@
+name: Test Yq Action
+on: [push]
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get test
+        uses: mikefarah/yq@master
+        with:
+          cmd: yq eval '.a' examples/data1.yaml
+      - name: Write inplace test
+        id: lookupSdkVersion
+        uses: mikefarah/yq@master
+        with:
+          cmd: yq eval -i '.a.b = 5' examples/data1.yaml

Dockerfile

@@ -12,7 +12,12 @@ RUN CGO_ENABLED=0 make local build
 
 # Choose alpine as a base image to make this useful for CI, as many
 # CI tools expect an interactive shell inside the container
-FROM alpine:3.12.3 as production
+FROM alpine:3.13.5 as production
+
+RUN mkdir /home/yq/
+RUN addgroup -g 1000 yq && \
+    adduser -u 1000 -G yq -s /bin/bash -h /home/yq -D yq
+RUN chown -R yq:yq /home/yq/
 
 COPY --from=builder /go/src/mikefarah/yq/yq /usr/bin/yq
 RUN chmod +x /usr/bin/yq
@@ -20,6 +25,8 @@ RUN chmod +x /usr/bin/yq
 ARG VERSION=none
 LABEL version=${VERSION}
 
+USER yq
+
 WORKDIR /workdir
 
 ENTRYPOINT ["/usr/bin/yq"]

README.md

@@ -101,6 +101,25 @@ yq() {
 }
 ```
 
+#### Running as root:
+
+`yq`'s docker image no longer runs under root (https://github.com/mikefarah/yq/pull/860). If you'd like to install more things in the docker image, or you're having permissions issues when attempting to read/write files you'll need to either:
+
+
+```
+docker run --user="root" -it --entrypoint sh mikefarah/yq
+```
+
+Or, in your docker file:
+
+```
+FROM mikefarah/yq
+
+USER root
+RUN apk add bash
+USER yq
+```
+
 ### Go Get:
 ```
 GO111MODULE=on go get github.com/mikefarah/yq/v4
@@ -168,6 +187,7 @@ Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
 - [Pipe data in by using '-'](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate)
 - [General shell completion scripts (bash/zsh/fish/powershell)](https://mikefarah.gitbook.io/yq/v/v4.x/commands/shell-completion)
 - [Reduce](https://mikefarah.gitbook.io/yq/operators/reduce) to merge multiple files or sum an array or other fancy things.
+- [Github Action](https://mikefarah.gitbook.io/yq/usage/github-action) to use in your automated pipeline (thanks @devorbitus)
 
 ## [Usage](https://mikefarah.gitbook.io/yq/)
 
@@ -209,3 +229,7 @@ yq e '.a.b | length' f1.yml f2.yml
 
 ## Known Issues / Missing Features
 - `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details)
+- Powershell has its own...opinions: https://mikefarah.gitbook.io/yq/usage/tips-and-tricks#quotes-in-windows-powershell
+- Running expressions against blank files does not work, because the file is empty, there are no matches for yq to run through the expression pipeline and so nothing happens. Instead, you can do something like `yq e -n '.someNew="content"' > newfile.yml` to create a new file.
+
+See [tips and tricks](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks) for more common problems and solutions.

action.yml

@@ -1,11 +1,15 @@
 name: 'yq - portable yaml processor'
 description: 'create, read, update, delete, merge, validate and do more with yaml'
-icon: command
-color: gray-dark
+branding:
+  icon: command
+  color: gray-dark
 inputs:
   cmd:
     description: 'The Command which should be run'
     required: true
+outputs:
+  result:
+    description: "The complete result from the yq command being run"
 runs:
   using: 'docker'
   image: 'github-action/Dockerfile'

cmd/constant.go

@@ -16,4 +16,7 @@ var verbose = false
 var version = false
 var prettyPrint = false
 
+// can be either "" (off), "extract" or "process"
+var frontMatter = ""
+
 var completedSuccessfully = false

cmd/evaluate_all_command.go

@@ -15,13 +15,27 @@ func createEvaluateAllCommand() *cobra.Command {
 		Aliases: []string{"ea"},
 		Short:   "Loads _all_ yaml documents of _all_ yaml files and runs expression once",
 		Example: `
-# merges f2.yml into f1.yml (inplace)
-yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' f1.yml f2.yml		
+# Merge f2.yml into f1.yml (inplace)
+yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' f1.yml f2.yml
+## the same command and expression using shortened names:
+yq ea -i 'select(fi == 0) * select(fi == 1)' f1.yml f2.yml
 
-# use '-' as a filename to read from STDIN
+
+# Merge all given files
+yq ea '. as $item ireduce ({}; . * $item )' file1.yml file2.yml ...
+
+# Read from STDIN
+## use '-' as a filename to read from STDIN
 cat file2.yml | yq ea '.a.b' file1.yml - file3.yml
 `,
-		Long: "Evaluate All:\nUseful when you need to run an expression across several yaml documents or files. Consumes more memory than eval",
+		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
+See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.
+
+## Evaluate All ##
+This command loads _all_ yaml documents of _all_ yaml files and runs expression once
+Useful when you need to run an expression across several yaml documents or files (like merge).
+Note that it consumes more memory than eval.
+`,
 		RunE: evaluateAll,
 	}
 	return cmdEvalAll
@@ -75,6 +89,25 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 
 	printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
 
+	if frontMatter != "" {
+		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])
+		err = frontMatterHandler.Split()
+		if err != nil {
+			return err
+		}
+		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
+
+		if frontMatter == "process" {
+			reader, err := os.Open(frontMatterHandler.GetContentFilename()) // #nosec
+			if err != nil {
+				return err
+			}
+			printer.SetAppendix(reader)
+			defer yqlib.SafelyCloseReader(reader)
+		}
+		defer frontMatterHandler.CleanUp()
+	}
+
 	allAtOnceEvaluator := yqlib.NewAllAtOnceEvaluator()
 	switch len(args) {
 	case 0:

cmd/evalute_sequence_command.go

@@ -13,25 +13,31 @@ func createEvaluateSequenceCommand() *cobra.Command {
 	var cmdEvalSequence = &cobra.Command{
 		Use:     "eval [expression] [yaml_file1]...",
 		Aliases: []string{"e"},
-		Short:   "Apply expression to each document in each yaml file given in sequence",
+		Short:   "Apply the expression to each document in each yaml file in sequence",
 		Example: `
-# runs the expression against each file, in series
-yq e '.a.b | length' f1.yml f2.yml 
+# Reads field under the given path for each file
+yq e '.a.b' f1.yml f2.yml 
 
-# prints out the file
+# Prints out the file
 yq e sample.yaml 
 
-# use '-' as a filename to read from STDIN
+# Read from STDIN
+## use '-' as a filename to read from STDIN
 cat file2.yml | yq e '.a.b' file1.yml - file3.yml
 
-# prints a new yaml document
+# Creates a new yaml document
+## Note that editing an empty file does not work.
 yq e -n '.a.b.c = "cat"' 
 
-
-# updates file.yaml directly
+# Update a file inplace
 yq e '.a.b = "cool"' -i file.yaml 
 `,
-		Long: "Evaluate Sequence:\nIterate over each yaml document, apply the expression and print the results, in sequence.",
+		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
+See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.
+
+## Evaluate Sequence ##
+This command iterates over each yaml document from each given file, applies the 
+expression and prints the result in sequence.`,
 		RunE: evaluateSequence,
 	}
 	return cmdEvalSequence
@@ -97,6 +103,25 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 		return errors.New("Cannot pass files in when using null-input flag")
 	}
 
+	if frontMatter != "" {
+		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])
+		err = frontMatterHandler.Split()
+		if err != nil {
+			return err
+		}
+		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
+
+		if frontMatter == "process" {
+			reader, err := os.Open(frontMatterHandler.GetContentFilename()) // #nosec
+			if err != nil {
+				return err
+			}
+			printer.SetAppendix(reader)
+			defer yqlib.SafelyCloseReader(reader)
+		}
+		defer frontMatterHandler.CleanUp()
+	}
+
 	switch len(args) {
 	case 0:
 		if pipingStdIn {

cmd/root.go

@@ -11,7 +11,8 @@ func New() *cobra.Command {
 	var rootCmd = &cobra.Command{
 		Use:   "yq",
 		Short: "yq is a lightweight and portable command-line YAML processor.",
-		Long:  `yq is a lightweight and portable command-line YAML processor. It aims to be the jq or sed of yaml files.`,
+		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
+See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if version {
 				cmd.Print(GetVersionDisplay())
@@ -53,6 +54,7 @@ func New() *cobra.Command {
 
 	rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")
 	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
+	rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data in-tact")
 	rootCmd.AddCommand(
 		createEvaluateSequenceCommand(),
 		createEvaluateAllCommand(),

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.9.3"
+	Version = "4.10.0"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release
@@ -25,7 +25,7 @@ const ProductName = "yq"
 // GetVersionDisplay composes the parts of the version in a way that's suitable
 // for displaying to humans.
 func GetVersionDisplay() string {
-	return fmt.Sprintf("%s version %s\n", ProductName, getHumanVersion())
+	return fmt.Sprintf("yq (https://github.com/mikefarah/yq/) version %s\n", getHumanVersion())
 }
 
 func getHumanVersion() string {

cmd/version_test.go

@@ -3,7 +3,7 @@ package cmd
 import "testing"
 
 func TestGetVersionDisplay(t *testing.T) {
-	var expectedVersion = ProductName + " version " + Version
+	var expectedVersion = ProductName + " (https://github.com/mikefarah/yq/) version " + Version
 	if VersionPrerelease != "" {
 		expectedVersion = expectedVersion + "-" + VersionPrerelease
 	}

debian/changelog

@@ -1,3 +1,12 @@
+yq (4.9.6) focal; urgency=medium
+
+  * Added darwin/arm64 build, thanks @alecthomas
+  * Incremented docker alpine base version, thanks @da6d6i7-bronga
+  * Bug fix: multine expression
+  * Bug fix: special character
+
+ -- Roberto Mier Escandon <rmescandon@gmail.com>  Tue, 29 Jun 2021 21:32:14 +0200
+
 yq (3.3.2) focal; urgency=medium
 
   * Bug fix: existStatus bug (#459)

debian/control

@@ -3,8 +3,7 @@ Section: devel
 Priority: optional
 Maintainer: Roberto Mier Escandón <rmescandon@gmail.com>
 Build-Depends: debhelper (>=10),
-               dh-golang (>=1.34),
-               golang-1.13,
+               golang-1.15,
                rsync
 Standards-Version: 4.1.4
 Homepage: https://github.com/mikefarah/yq.git

debian/rules

@@ -17,7 +17,7 @@
 PROJECT := yq
 OWNER := mikefarah
 REPO := github.com
-GOVERSION := 1.13
+GOVERSION := 1.15
 
 export DH_OPTIONS
 export DH_GOPKG := ${REPO}/${OWNER}/${PROJECT}
@@ -34,7 +34,7 @@ BINDIR := /usr/bin
 ASSETSDIR := /usr/share/${PROJECT}
 
 %:
-	dh $@ --builddirectory=${GOPATH} --buildsystem=golang
+	dh $@
 
 override_dh_auto_build:
 	mkdir -p ${SRCDIR}

examples/data1.yaml

@@ -1,6 +1,2 @@
-a:
-  key1: "value1"
-  key2: 2.6
-  ab:
-    key1: 6
-    key2: "h"
+---
+{"a":{"b":1}}

examples/empty.yaml

@@ -1,2 +1 @@
-# a: apple
-# b: cat
+# comment

examples/front-matter.yaml

@@ -0,0 +1,5 @@
+---
+a: fruit
+b: banana
+---
+content: cool

examples/leading-seperator.yaml

@@ -0,0 +1,2 @@
+---
+a: test

github-action/Dockerfile

@@ -1,5 +1,8 @@
-FROM mikefarah/yq:4.9.3
+FROM mikefarah/yq:4.10.0
 
 COPY entrypoint.sh /entrypoint.sh
 
+# this seems to be the default user in github actions
+USER 1001
+
 ENTRYPOINT ["/entrypoint.sh"]

github-action/entrypoint.sh

@@ -1,4 +1,6 @@
 #!/bin/sh -l
 
-echo "$1"
-eval $1
+echo "::debug::\$cmd: $1"
+RESULT=$(eval "$1")
+echo "::debug::\$RESULT: $RESULT"
+echo ::set-output name=result::"$RESULT"

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/spf13/cobra v1.1.3
 	github.com/timtadh/data-structures v0.5.3 // indirect
 	github.com/timtadh/lexmachine v0.2.2
-	golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e // indirect
+	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )

go.sum

@@ -256,8 +256,8 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e h1:XNp2Flc/1eWQGk5BLzqTAN7fQIwIbfyVTuVxXxZh73M=
-golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=

pkg/yqlib/all_at_once_evaluator.go

@@ -2,6 +2,7 @@ package yqlib
 
 import (
 	"container/list"
+	"os"
 
 	yaml "gopkg.in/yaml.v3"
 )
@@ -48,13 +49,19 @@ func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCand
 
 func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error {
 	fileIndex := 0
+	firstFileLeadingSeperator := false
 
 	var allDocuments *list.List = list.New()
 	for _, filename := range filenames {
-		reader, err := readStream(filename)
+		reader, leadingSeperator, err := readStream(filename)
 		if err != nil {
 			return err
 		}
+
+		if fileIndex == 0 && leadingSeperator {
+			firstFileLeadingSeperator = leadingSeperator
+		}
+
 		fileDocuments, err := readDocuments(reader, filename, fileIndex)
 		if err != nil {
 			return err
@@ -62,9 +69,33 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
 		allDocuments.PushBackList(fileDocuments)
 		fileIndex = fileIndex + 1
 	}
+
+	if allDocuments.Len() == 0 {
+		candidateNode := &CandidateNode{
+			Document:  0,
+			Filename:  "",
+			Node:      &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode},
+			FileIndex: 0,
+		}
+		allDocuments.PushBack(candidateNode)
+
+		if len(filenames) > 0 {
+			reader, _, err := readStream(filenames[0])
+			if err != nil {
+				return err
+			}
+			switch reader := reader.(type) {
+			case *os.File:
+				defer safelyCloseFile(reader)
+			}
+			printer.SetPreamble(reader)
+		}
+	}
+
 	matches, err := e.EvaluateCandidateNodes(expression, allDocuments)
 	if err != nil {
 		return err
 	}
+	printer.SetPrintLeadingSeperator(firstFileLeadingSeperator)
 	return printer.PrintResults(matches)
 }

pkg/yqlib/context.go

@@ -66,3 +66,9 @@ func (n *Context) ReadOnlyClone() Context {
 	clone.DontAutoCreate = true
 	return clone
 }
+
+func (n *Context) WritableClone() Context {
+	clone := n.Clone()
+	clone.DontAutoCreate = false
+	return clone
+}

pkg/yqlib/doc/Add.md

@@ -127,6 +127,32 @@ b:
   - 4
 ```
 
+## Append another array using +=
+Given a sample.yml file of:
+```yaml
+a:
+  - 1
+  - 2
+b:
+  - 3
+  - 4
+```
+then
+```bash
+yq eval '.a += .b' sample.yml
+```
+will output
+```yaml
+a:
+  - 1
+  - 2
+  - 3
+  - 4
+b:
+  - 3
+  - 4
+```
+
 ## Relative append
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Anchor and Alias Operators.md

@@ -299,3 +299,33 @@ foobar:
   thing: foobar_thing
 ```
 
+## Dereference and update a field
+`Use explode with multiply to dereference an object
+
+Given a sample.yml file of:
+```yaml
+item_value: &item_value
+  value: true
+thingOne:
+  name: item_1
+  !!merge <<: *item_value
+thingTwo:
+  name: item_2
+  !!merge <<: *item_value
+```
+then
+```bash
+yq eval '.thingOne |= explode(.) * {"value": false}' sample.yml
+```
+will output
+```yaml
+item_value: &item_value
+  value: true
+thingOne:
+  name: item_1
+  value: false
+thingTwo:
+  name: item_2
+  !!merge <<: *item_value
+```
+

pkg/yqlib/doc/Assign (Update).md

@@ -28,6 +28,24 @@ a:
   g: foof
 ```
 
+## Double elements in an array
+Given a sample.yml file of:
+```yaml
+- 1
+- 2
+- 3
+```
+then
+```bash
+yq eval '.[] |= . * 2' sample.yml
+```
+will output
+```yaml
+- 2
+- 4
+- 6
+```
+
 ## Update node from another file
 Note this will also work when the second file is a scalar (string/number)
 
@@ -75,7 +93,7 @@ c: fieldC
 ```
 then
 ```bash
-yq eval '(.a, .c) |= "potatoe"' sample.yml
+yq eval '(.a, .c) = "potatoe"' sample.yml
 ```
 will output
 ```yaml

pkg/yqlib/doc/Entries.md

@@ -98,3 +98,21 @@ KEY_a: 1
 KEY_b: 2
 ```
 
+## Use with_entries to filter the map
+Given a sample.yml file of:
+```yaml
+a:
+  b: bird
+c:
+  d: dog
+```
+then
+```bash
+yq eval 'with_entries(select(.value | has("b")))' sample.yml
+```
+will output
+```yaml
+a:
+  b: bird
+```
+

pkg/yqlib/doc/String Operators.md

@@ -1,5 +1,8 @@
 # String Operators
 
+## RegEx
+This uses golangs native regex functions under the hood - See https://github.com/google/re2/wiki/Syntax for the supported syntax.
+
 ## Join strings
 Given a sample.yml file of:
 ```yaml
@@ -18,6 +21,171 @@ will output
 cat; meow; 1; ; true
 ```
 
+## Match string
+Given a sample.yml file of:
+```yaml
+foo bar foo
+```
+then
+```bash
+yq eval 'match("foo")' sample.yml
+```
+will output
+```yaml
+string: foo
+offset: 0
+length: 3
+captures: []
+```
+
+## Match string, case insensitive
+Given a sample.yml file of:
+```yaml
+foo bar FOO
+```
+then
+```bash
+yq eval '[match("(?i)foo"; "g")]' sample.yml
+```
+will output
+```yaml
+- string: foo
+  offset: 0
+  length: 3
+  captures: []
+- string: FOO
+  offset: 8
+  length: 3
+  captures: []
+```
+
+## Match with capture groups
+Given a sample.yml file of:
+```yaml
+abc abc
+```
+then
+```bash
+yq eval '[match("(abc)+"; "g")]' sample.yml
+```
+will output
+```yaml
+- string: abc
+  offset: 0
+  length: 3
+  captures:
+    - string: abc
+      offset: 0
+      length: 3
+- string: abc
+  offset: 4
+  length: 3
+  captures:
+    - string: abc
+      offset: 4
+      length: 3
+```
+
+## Match with named capture groups
+Given a sample.yml file of:
+```yaml
+foo bar foo foo  foo
+```
+then
+```bash
+yq eval '[match("foo (?P<bar123>bar)? foo"; "g")]' sample.yml
+```
+will output
+```yaml
+- string: foo bar foo
+  offset: 0
+  length: 11
+  captures:
+    - string: bar
+      offset: 4
+      length: 3
+      name: bar123
+- string: foo  foo
+  offset: 12
+  length: 8
+  captures:
+    - string: null
+      offset: -1
+      length: 0
+      name: bar123
+```
+
+## Capture named groups into a map
+Given a sample.yml file of:
+```yaml
+xyzzy-14
+```
+then
+```bash
+yq eval 'capture("(?P<a>[a-z]+)-(?P<n>[0-9]+)")' sample.yml
+```
+will output
+```yaml
+a: xyzzy
+n: "14"
+```
+
+## Match without global flag
+Given a sample.yml file of:
+```yaml
+cat cat
+```
+then
+```bash
+yq eval 'match("cat")' sample.yml
+```
+will output
+```yaml
+string: cat
+offset: 0
+length: 3
+captures: []
+```
+
+## Match with global flag
+Given a sample.yml file of:
+```yaml
+cat cat
+```
+then
+```bash
+yq eval '[match("cat"; "g")]' sample.yml
+```
+will output
+```yaml
+- string: cat
+  offset: 0
+  length: 3
+  captures: []
+- string: cat
+  offset: 4
+  length: 3
+  captures: []
+```
+
+## Test using regex
+Like jq'q equivalant, this works like match but only returns true/false instead of full match details
+
+Given a sample.yml file of:
+```yaml
+- cat
+- dog
+```
+then
+```bash
+yq eval '.[] | test("at")' sample.yml
+```
+will output
+```yaml
+true
+false
+```
+
 ## Substitute / Replace string
 This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)
 Note the use of `|=` to run in context of the current string value.

pkg/yqlib/doc/Traverse (Read).md

@@ -1,4 +1,4 @@
-This is the simplest (and perhaps most used) operator, it is used to navigate deeply into yaml structurse.
+This is the simplest (and perhaps most used) operator, it is used to navigate deeply into yaml structures.
 ## Simple map navigation
 Given a sample.yml file of:
 ```yaml
@@ -63,6 +63,22 @@ will output
 frog
 ```
 
+## Nested special characters
+Given a sample.yml file of:
+```yaml
+a:
+  "key.withdots":
+    "another.key": apple
+```
+then
+```bash
+yq eval '.a["key.withdots"]["another.key"]' sample.yml
+```
+will output
+```yaml
+apple
+```
+
 ## Keys with spaces
 Use quotes with brackets around path elements with special characters
 
@@ -211,6 +227,20 @@ will output
 1
 ```
 
+## Traversing nested arrays by index
+Given a sample.yml file of:
+```yaml
+[[], [cat]]
+```
+then
+```bash
+yq eval '.[1][0]' sample.yml
+```
+will output
+```yaml
+cat
+```
+
 ## Maps with numeric keys
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/aa.md

@@ -71,7 +71,7 @@ dog
 Both sides have now been evaluated, so now the operator copies across the value from the RHS to the value on the LHS, and it returns the now updated context:
 
 ```yaml
-a: cat
+a: dog
 b: dog
 ```
 

pkg/yqlib/doc/headers/String Operators.md

@@ -1 +1,4 @@
 # String Operators
+
+## RegEx
+This uses golangs native regex functions under the hood - See https://github.com/google/re2/wiki/Syntax for the supported syntax.

pkg/yqlib/doc/headers/Traverse (Read).md

@@ -1 +1 @@
-This is the simplest (and perhaps most used) operator, it is used to navigate deeply into yaml structurse.
+This is the simplest (and perhaps most used) operator, it is used to navigate deeply into yaml structures.

pkg/yqlib/expression_processing_test.go

@@ -7,11 +7,24 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
+var variableWithNewLine = `"cat
+"`
+
 var pathTests = []struct {
 	path            string
 	expectedTokens  []interface{}
 	expectedPostFix []interface{}
 }{
+	{
+		".a\n",
+		append(make([]interface{}, 0), "a"),
+		append(make([]interface{}, 0), "a"),
+	},
+	{
+		variableWithNewLine,
+		append(make([]interface{}, 0), "cat\n (string)"),
+		append(make([]interface{}, 0), "cat\n (string)"),
+	},
 	{
 		`.[0]`,
 		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),

pkg/yqlib/expression_tokeniser.go

@@ -276,6 +276,9 @@ func initLexer() (*lex.Lexer, error) {
 
 	lexer.Add([]byte(`join`), opToken(joinStringOpType))
 	lexer.Add([]byte(`sub`), opToken(subStringOpType))
+	lexer.Add([]byte(`match`), opToken(matchOpType))
+	lexer.Add([]byte(`capture`), opToken(captureOpType))
+	lexer.Add([]byte(`test`), opToken(testOpType))
 
 	lexer.Add([]byte(`any`), opToken(anyOpType))
 	lexer.Add([]byte(`any_c`), opToken(anyConditionOpType))
@@ -320,7 +323,7 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte("( |\t|\n|\r)+"), skip)
 
 	lexer.Add([]byte(`\."[^ "]+"\??`), pathToken(true))
-	lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=]+\??`), pathToken(false))
+	lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=\n]+\??`), pathToken(false))
 	lexer.Add([]byte(`\.`), selfToken())
 
 	lexer.Add([]byte(`\|`), opToken(pipeOpType))

pkg/yqlib/file_utils.go

@@ -2,6 +2,7 @@ package yqlib
 
 import (
 	"io"
+	"io/ioutil"
 	"os"
 )
 
@@ -23,9 +24,20 @@ func safelyRenameFile(from string, to string) {
 	}
 }
 
+func tryRemoveFile(filename string) {
+	log.Debug("Removing temp file: %v", filename)
+	removeErr := os.Remove(filename)
+	if removeErr != nil {
+		log.Errorf("Failed to remove temp file: %v", filename)
+	}
+}
+
 // thanks https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang
 func copyFileContents(src, dst string) (err error) {
-	in, err := os.Open(src) // nolint gosec
+	// ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory,
+	// and ensuring that it's not possible to give a path to a file outside thar directory.
+
+	in, err := os.Open(src) // #nosec
 	if err != nil {
 		return err
 	}
@@ -41,6 +53,13 @@ func copyFileContents(src, dst string) (err error) {
 	return out.Sync()
 }
 
+func SafelyCloseReader(reader io.Reader) {
+	switch reader := reader.(type) {
+	case *os.File:
+		safelyCloseFile(reader)
+	}
+}
+
 func safelyCloseFile(file *os.File) {
 	err := file.Close()
 	if err != nil {
@@ -48,3 +67,22 @@ func safelyCloseFile(file *os.File) {
 		log.Error(err.Error())
 	}
 }
+
+func createTempFile() (*os.File, error) {
+	_, err := os.Stat(os.TempDir())
+	if os.IsNotExist(err) {
+		err = os.Mkdir(os.TempDir(), 0700)
+		if err != nil {
+			return nil, err
+		}
+	} else if err != nil {
+		return nil, err
+	}
+
+	file, err := ioutil.TempFile("", "temp")
+	if err != nil {
+		return nil, err
+	}
+
+	return file, err
+}

pkg/yqlib/front_matter.go

@@ -0,0 +1,96 @@
+package yqlib
+
+import (
+	"bufio"
+	"io"
+	"os"
+)
+
+type frontMatterHandler interface {
+	Split() error
+	GetYamlFrontMatterFilename() string
+	GetContentFilename() string
+	CleanUp()
+}
+
+type frontMatterHandlerImpl struct {
+	originalFilename        string
+	yamlFrontMatterFilename string
+	contentFilename         string
+}
+
+func NewFrontMatterHandler(originalFilename string) frontMatterHandler {
+	return &frontMatterHandlerImpl{originalFilename, "", ""}
+}
+
+func (f *frontMatterHandlerImpl) GetYamlFrontMatterFilename() string {
+	return f.yamlFrontMatterFilename
+}
+
+func (f *frontMatterHandlerImpl) GetContentFilename() string {
+	return f.contentFilename
+}
+
+func (f *frontMatterHandlerImpl) CleanUp() {
+	tryRemoveFile(f.yamlFrontMatterFilename)
+	tryRemoveFile(f.contentFilename)
+}
+
+// Splits the given file by yaml front matter
+// yaml content will be saved to first temporary file
+// remaining content will be saved to second temporary file
+func (f *frontMatterHandlerImpl) Split() error {
+	var reader io.Reader
+	var err error
+	if f.originalFilename == "-" {
+		reader = bufio.NewReader(os.Stdin)
+	} else {
+		reader, err = os.Open(f.originalFilename) // #nosec
+		if err != nil {
+			return err
+		}
+	}
+
+	yamlTempFile, err := createTempFile()
+	if err != nil {
+		return err
+	}
+	f.yamlFrontMatterFilename = yamlTempFile.Name()
+	log.Debug("yamlTempFile: %v", yamlTempFile.Name())
+
+	contentTempFile, err := createTempFile()
+	if err != nil {
+		return err
+	}
+	f.contentFilename = contentTempFile.Name()
+	log.Debug("contentTempFile: %v", contentTempFile.Name())
+
+	scanner := bufio.NewScanner(reader)
+
+	lineCount := 0
+	yamlContentBlock := true
+
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		if lineCount > 0 && line == "---" {
+			//we've finished reading the yaml content
+			yamlContentBlock = false
+		}
+		if yamlContentBlock {
+			_, err = yamlTempFile.Write([]byte(line + "\n"))
+		} else {
+			_, err = contentTempFile.Write([]byte(line + "\n"))
+		}
+		if err != nil {
+			return err
+		}
+		lineCount = lineCount + 1
+	}
+
+	safelyCloseFile(yamlTempFile)
+	safelyCloseFile(contentTempFile)
+
+	return scanner.Err()
+
+}

pkg/yqlib/front_matter_test.go

@@ -0,0 +1,133 @@
+package yqlib
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+func createTestFile(content string) string {
+	tempFile, err := createTempFile()
+	if err != nil {
+		panic(err)
+	}
+
+	_, err = tempFile.Write([]byte(content))
+	if err != nil {
+		panic(err)
+	}
+
+	safelyCloseFile(tempFile)
+
+	return tempFile.Name()
+}
+
+func readFile(filename string) string {
+	bytes, err := ioutil.ReadFile(filename)
+	if err != nil {
+		panic(err)
+	}
+	return string(bytes)
+}
+
+func TestFrontMatterSplitWithLeadingSep(t *testing.T) {
+	file := createTestFile(`---
+a: apple
+b: banana
+---
+not a 
+yaml: doc
+`)
+
+	expectedYamlFm := `---
+a: apple
+b: banana
+`
+
+	expectedContent := `---
+not a 
+yaml: doc
+`
+
+	fmHandler := NewFrontMatterHandler(file)
+	err := fmHandler.Split()
+	if err != nil {
+		panic(err)
+	}
+
+	yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename())
+
+	test.AssertResult(t, expectedYamlFm, yamlFm)
+
+	content := readFile(fmHandler.GetContentFilename())
+	test.AssertResult(t, expectedContent, content)
+
+	tryRemoveFile(file)
+	fmHandler.CleanUp()
+}
+
+func TestFrontMatterSplitWithNoLeadingSep(t *testing.T) {
+	file := createTestFile(`a: apple
+b: banana
+---
+not a 
+yaml: doc
+`)
+
+	expectedYamlFm := `a: apple
+b: banana
+`
+
+	expectedContent := `---
+not a 
+yaml: doc
+`
+
+	fmHandler := NewFrontMatterHandler(file)
+	err := fmHandler.Split()
+	if err != nil {
+		panic(err)
+	}
+
+	yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename())
+
+	test.AssertResult(t, expectedYamlFm, yamlFm)
+
+	content := readFile(fmHandler.GetContentFilename())
+	test.AssertResult(t, expectedContent, content)
+
+	tryRemoveFile(file)
+	fmHandler.CleanUp()
+}
+
+func TestFrontMatterSplitWithArray(t *testing.T) {
+	file := createTestFile(`[1,2,3]
+---
+not a 
+yaml: doc
+`)
+
+	expectedYamlFm := "[1,2,3]\n"
+
+	expectedContent := `---
+not a 
+yaml: doc
+`
+
+	fmHandler := NewFrontMatterHandler(file)
+	err := fmHandler.Split()
+	if err != nil {
+		panic(err)
+	}
+
+	yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename())
+
+	test.AssertResult(t, expectedYamlFm, yamlFm)
+
+	content := readFile(fmHandler.GetContentFilename())
+	test.AssertResult(t, expectedContent, content)
+
+	tryRemoveFile(file)
+	fmHandler.CleanUp()
+}

pkg/yqlib/lib.go

@@ -83,6 +83,9 @@ var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50,
 var sortKeysOpType = &operationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: sortKeysOperator}
 var joinStringOpType = &operationType{Type: "JOIN", NumArgs: 1, Precedence: 50, Handler: joinStringOperator}
 var subStringOpType = &operationType{Type: "SUBSTR", NumArgs: 1, Precedence: 50, Handler: substituteStringOperator}
+var matchOpType = &operationType{Type: "MATCH", NumArgs: 1, Precedence: 50, Handler: matchOperator}
+var captureOpType = &operationType{Type: "CAPTURE", NumArgs: 1, Precedence: 50, Handler: captureOperator}
+var testOpType = &operationType{Type: "TEST", NumArgs: 1, Precedence: 50, Handler: testOperator}
 var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 50, Handler: splitStringOperator}
 
 var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 50, Handler: keysOperator}
@@ -114,8 +117,8 @@ type Operation struct {
 	UpdateAssign  bool // used for assign ops, when true it means we evaluate the rhs given the lhs
 }
 
-func createValueOperation(value interface{}, stringValue string) *Operation {
-	var node yaml.Node = yaml.Node{Kind: yaml.ScalarNode}
+func createScalarNode(value interface{}, stringValue string) *yaml.Node {
+	var node = &yaml.Node{Kind: yaml.ScalarNode}
 	node.Value = stringValue
 
 	switch value.(type) {
@@ -130,12 +133,17 @@ func createValueOperation(value interface{}, stringValue string) *Operation {
 	case nil:
 		node.Tag = "!!null"
 	}
+	return node
+}
+
+func createValueOperation(value interface{}, stringValue string) *Operation {
+	var node *yaml.Node = createScalarNode(value, stringValue)
 
 	return &Operation{
 		OperationType: valueOpType,
 		Value:         value,
 		StringValue:   stringValue,
-		CandidateNode: &CandidateNode{Node: &node},
+		CandidateNode: &CandidateNode{Node: node},
 	}
 }
 
@@ -179,7 +187,10 @@ func NodeToString(node *CandidateNode) string {
 	if errorEncoding != nil {
 		log.Error("Error debugging node, %v", errorEncoding.Error())
 	}
-	encoder.Close()
+	errorClosingEncoder := encoder.Close()
+	if errorClosingEncoder != nil {
+		log.Error("Error closing encoder: ", errorClosingEncoder.Error())
+	}
 	tag := value.Tag
 	if value.Kind == yaml.DocumentNode {
 		tag = "doc"

pkg/yqlib/operator_add.go

@@ -15,11 +15,7 @@ func createAddOp(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode {
 }
 
 func addAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	assignmentOp := &Operation{OperationType: assignOpType}
-	assignmentOp.UpdateAssign = true
-	selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
-	assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: expressionNode.Lhs, Rhs: createAddOp(selfExpression, expressionNode.Rhs)}
-	return d.GetMatchingNodes(context, assignmentOpNode)
+	return compoundAssignFunction(d, context, expressionNode, createAddOp)
 }
 
 func toNodes(candidate *CandidateNode) []*yaml.Node {

pkg/yqlib/operator_add_test.go

@@ -86,6 +86,14 @@ var addOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: [1, 2, 3, 4], b: [3, 4]}\n",
 		},
 	},
+	{
+		description: "Append another array using +=",
+		document:    `{a: [1,2], b: [3,4]}`,
+		expression:  `.a += .b`,
+		expected: []string{
+			"D0, P[], (doc)::{a: [1, 2, 3, 4], b: [3, 4]}\n",
+		},
+	},
 	{
 		description: "Relative append",
 		document:    `a: { a1: {b: [cat]}, a2: {b: [dog]}, a3: {} }`,

pkg/yqlib/operator_alternative.go

@@ -9,6 +9,9 @@ func alternativeFunc(d *dataTreeNavigator, context Context, lhs *CandidateNode,
 	if lhs == nil {
 		return rhs, nil
 	}
+	if rhs == nil {
+		return lhs, nil
+	}
 	lhs.Node = unwrapDoc(lhs.Node)
 	rhs.Node = unwrapDoc(rhs.Node)
 	log.Debugf("Alternative LHS: %v", lhs.Node.Tag)

pkg/yqlib/operator_alternative_test.go

@@ -5,6 +5,12 @@ import (
 )
 
 var alternativeOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		expression: `.b // .c`,
+		document:   `a: bridge`,
+		expected:   []string{},
+	},
 	{
 		skipDoc:    true,
 		expression: `(.b // "hello") as $x`,
@@ -13,6 +19,14 @@ var alternativeOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: bridge\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		expression: `.a // .b`,
+		document:   `a: 2`,
+		expected: []string{
+			"D0, P[a], (!!int)::2\n",
+		},
+	},
 	{
 		description: "LHS is defined",
 		expression:  `.a // "hello"`,

pkg/yqlib/operator_anchors_aliases_test.go

@@ -12,6 +12,29 @@ var specDocument = `- &CENTER { x: 1, y: 2 }
 
 var expectedSpecResult = "D0, P[4], (!!map)::x: 1\ny: 2\nr: 10\n"
 
+var simpleArrayRef = `
+item_value: &item_value
+  value: true
+
+thingOne:
+  name: item_1
+  <<: *item_value
+
+thingTwo:
+  name: item_2
+  <<: *item_value
+`
+
+var expectedUpdatedArrayRef = `D0, P[], (doc)::item_value: &item_value
+    value: true
+thingOne:
+    name: item_1
+    value: false
+thingTwo:
+    name: item_2
+    !!merge <<: *item_value
+`
+
 var anchorOperatorScenarios = []expressionScenario{
 	{
 		description:    "Merge one map",
@@ -197,6 +220,13 @@ foobar:
 			"D0, P[], (doc)::{f: {a: cat, b: {f: cat}, cat: {f: cat}}}\n",
 		},
 	},
+	{
+		description:    "Dereference and update a field",
+		subdescription: "`Use explode with multiply to dereference an object",
+		document:       simpleArrayRef,
+		expression:     `.thingOne |= explode(.) * {"value": false}`,
+		expected:       []string{expectedUpdatedArrayRef},
+	},
 }
 
 func TestAnchorAliasOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_assign.go

@@ -1,21 +1,27 @@
 package yqlib
 
+func assignUpdateFunc(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	rhs.Node = unwrapDoc(rhs.Node)
+	lhs.UpdateFrom(rhs)
+	return lhs, nil
+}
+
 func assignUpdateOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	lhs, err := d.GetMatchingNodes(context, expressionNode.Lhs)
 	if err != nil {
 		return Context{}, err
 	}
-	var rhs Context
+
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err = d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
+		// this works because we already ran against LHS with an editable context.
+		_, err := crossFunction(d, context.ReadOnlyClone(), expressionNode, assignUpdateFunc, false)
+		return context, err
 	}
 
 	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 
-		if expressionNode.Operation.UpdateAssign {
-			rhs, err = d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
-		}
+		rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
 
 		if err != nil {
 			return Context{}, err

pkg/yqlib/operator_assign_test.go

@@ -36,6 +36,14 @@ var assignOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: {g: foof}}\n",
 		},
 	},
+	{
+		description: "Double elements in an array",
+		document:    `[1,2,3]`,
+		expression:  `.[] |= . * 2`,
+		expected: []string{
+			"D0, P[], (doc)::[2, 4, 6]\n",
+		},
+	},
 	{
 		description:    "Update node from another file",
 		subdescription: "Note this will also work when the second file is a scalar (string/number)",
@@ -57,7 +65,7 @@ var assignOperatorScenarios = []expressionScenario{
 	{
 		description: "Updated multiple paths",
 		document:    `{a: fieldA, b: fieldB, c: fieldC}`,
-		expression:  `(.a, .c) |= "potatoe"`,
+		expression:  `(.a, .c) = "potatoe"`,
 		expected: []string{
 			"D0, P[], (doc)::{a: potatoe, b: fieldB, c: potatoe}\n",
 		},

pkg/yqlib/operator_booleans_test.go

@@ -12,6 +12,22 @@ var booleanOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!bool)::true\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   "b: hi",
+		expression: `.a or .c`,
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "b: false",
+		expression: `.b or .c`,
+		expected: []string{
+			"D0, P[b], (!!bool)::false\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   "b: hi",

pkg/yqlib/operator_entries_test.go

@@ -52,6 +52,14 @@ var entriesOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!map)::KEY_a: 1\nKEY_b: 2\n",
 		},
 	},
+	{
+		description: "Use with_entries to filter the map",
+		document:    `{a: { b: bird }, c: { d: dog }}`,
+		expression:  `with_entries(select(.value | has("b")))`,
+		expected: []string{
+			"D0, P[], (!!map)::a: {b: bird}\n",
+		},
+	},
 }
 
 func TestEntriesOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_equals_test.go

@@ -5,6 +5,29 @@ import (
 )
 
 var equalsOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		expression: ".a == .b",
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: cat`,
+		expression: ".a == .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: cat`,
+		expression: ".b == .a",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   "cat",

pkg/yqlib/operator_multiply.go

@@ -32,11 +32,11 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
 
 			var newBlank = lhs.CreateChild(nil, &yaml.Node{})
 
-			var newThing, err = mergeObjects(d, context, newBlank, lhs, multiplyPreferences{})
+			var newThing, err = mergeObjects(d, context.WritableClone(), newBlank, lhs, multiplyPreferences{})
 			if err != nil {
 				return nil, err
 			}
-			return mergeObjects(d, context, newThing, rhs, preferences)
+			return mergeObjects(d, context.WritableClone(), newThing, rhs, preferences)
 		} else if lhs.Node.Tag == "!!int" && rhs.Node.Tag == "!!int" {
 			return multiplyIntegers(lhs, rhs)
 		} else if (lhs.Node.Tag == "!!int" || lhs.Node.Tag == "!!float") && (rhs.Node.Tag == "!!int" || rhs.Node.Tag == "!!float") {

pkg/yqlib/operator_multiply_test.go

@@ -47,6 +47,20 @@ var multiplyOperatorScenarios = []expressionScenario{
 			docExpected,
 		},
 	},
+	{
+		skipDoc:    true,
+		expression: `.x =  {"things": "whatever"} * {}`,
+		expected: []string{
+			"D0, P[], ()::x:\n    things: whatever\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `.x = {} * {"things": "whatever"}`,
+		expected: []string{
+			"D0, P[], ()::x:\n    things: whatever\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		expression: `3 * 4.5`,

pkg/yqlib/operator_strings.go

@@ -74,6 +74,232 @@ func substituteStringOperator(d *dataTreeNavigator, context Context, expressionN
 
 }
 
+func addMatch(original []*yaml.Node, match string, offset int, name string) []*yaml.Node {
+
+	newContent := append(original,
+		createScalarNode("string", "string"))
+
+	if offset < 0 {
+		// offset of -1 means there was no match, force a null value like jq
+		newContent = append(newContent,
+			createScalarNode(nil, "null"),
+		)
+	} else {
+		newContent = append(newContent,
+			createScalarNode(match, match),
+		)
+	}
+
+	newContent = append(newContent,
+		createScalarNode("offset", "offset"),
+		createScalarNode(offset, fmt.Sprintf("%v", offset)),
+		createScalarNode("length", "length"),
+		createScalarNode(len(match), fmt.Sprintf("%v", len(match))))
+
+	if name != "" {
+		newContent = append(newContent,
+			createScalarNode("name", "name"),
+			createScalarNode(name, name),
+		)
+	}
+	return newContent
+}
+
+type matchPreferences struct {
+	Global bool
+}
+
+func getMatches(matchPrefs matchPreferences, regEx *regexp.Regexp, value string) ([][]string, [][]int) {
+	var allMatches [][]string
+	var allIndices [][]int
+
+	if matchPrefs.Global {
+		allMatches = regEx.FindAllStringSubmatch(value, -1)
+		allIndices = regEx.FindAllStringSubmatchIndex(value, -1)
+	} else {
+		allMatches = [][]string{regEx.FindStringSubmatch(value)}
+		allIndices = [][]int{regEx.FindStringSubmatchIndex(value)}
+	}
+
+	log.Debug("allMatches, %v", allMatches)
+	return allMatches, allIndices
+}
+
+func match(matchPrefs matchPreferences, regEx *regexp.Regexp, candidate *CandidateNode, value string, results *list.List) {
+	subNames := regEx.SubexpNames()
+	allMatches, allIndices := getMatches(matchPrefs, regEx, value)
+
+	// if all matches just has an empty array in it,
+	// then nothing matched
+	if len(allMatches) > 0 && len(allMatches[0]) == 0 {
+		return
+	}
+
+	for i, matches := range allMatches {
+		capturesNode := &yaml.Node{Kind: yaml.SequenceNode}
+		match, submatches := matches[0], matches[1:]
+		for j, submatch := range submatches {
+			captureNode := &yaml.Node{Kind: yaml.MappingNode}
+			captureNode.Content = addMatch(capturesNode.Content, submatch, allIndices[i][2+j*2], subNames[j+1])
+			capturesNode.Content = append(capturesNode.Content, captureNode)
+		}
+
+		node := &yaml.Node{Kind: yaml.MappingNode}
+		node.Content = addMatch(node.Content, match, allIndices[i][0], "")
+		node.Content = append(node.Content,
+			createScalarNode("captures", "captures"),
+			capturesNode,
+		)
+		results.PushBack(candidate.CreateChild(nil, node))
+
+	}
+
+}
+
+func capture(matchPrefs matchPreferences, regEx *regexp.Regexp, candidate *CandidateNode, value string, results *list.List) {
+	subNames := regEx.SubexpNames()
+	allMatches, allIndices := getMatches(matchPrefs, regEx, value)
+
+	// if all matches just has an empty array in it,
+	// then nothing matched
+	if len(allMatches) > 0 && len(allMatches[0]) == 0 {
+		return
+	}
+
+	for i, matches := range allMatches {
+		capturesNode := &yaml.Node{Kind: yaml.MappingNode}
+
+		_, submatches := matches[0], matches[1:]
+		for j, submatch := range submatches {
+			capturesNode.Content = append(capturesNode.Content,
+				createScalarNode(subNames[j+1], subNames[j+1]))
+
+			offset := allIndices[i][2+j*2]
+			// offset of -1 means there was no match, force a null value like jq
+			if offset < 0 {
+				capturesNode.Content = append(capturesNode.Content,
+					createScalarNode(nil, "null"),
+				)
+			} else {
+				capturesNode.Content = append(capturesNode.Content,
+					createScalarNode(submatch, submatch),
+				)
+			}
+		}
+
+		results.PushBack(candidate.CreateChild(nil, capturesNode))
+
+	}
+
+}
+
+func extractMatchArguments(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (*regexp.Regexp, matchPreferences, error) {
+	regExExpNode := expressionNode.Rhs
+
+	matchPrefs := matchPreferences{}
+
+	// we got given parameters e.g. match(exp; params)
+	if expressionNode.Rhs.Operation.OperationType == blockOpType {
+		block := expressionNode.Rhs
+		regExExpNode = block.Lhs
+		replacementNodes, err := d.GetMatchingNodes(context, block.Rhs)
+		if err != nil {
+			return nil, matchPrefs, err
+		}
+		paramText := ""
+		if replacementNodes.MatchingNodes.Front() != nil {
+			paramText = replacementNodes.MatchingNodes.Front().Value.(*CandidateNode).Node.Value
+		}
+		if strings.Contains(paramText, "g") {
+			paramText = strings.ReplaceAll(paramText, "g", "")
+			matchPrefs.Global = true
+		}
+		if strings.Contains(paramText, "i") {
+			return nil, matchPrefs, fmt.Errorf(`'i' is not a valid option for match. To ignore case, use an expression like match("(?i)cat")`)
+		}
+		if len(paramText) > 0 {
+			return nil, matchPrefs, fmt.Errorf(`Unrecognised match params '%v', please see docs at https://mikefarah.gitbook.io/yq/operators/string-operators`, paramText)
+		}
+	}
+
+	regExNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), regExExpNode)
+	if err != nil {
+		return nil, matchPrefs, err
+	}
+	log.Debug(NodesToString(regExNodes.MatchingNodes))
+	regExStr := ""
+	if regExNodes.MatchingNodes.Front() != nil {
+		regExStr = regExNodes.MatchingNodes.Front().Value.(*CandidateNode).Node.Value
+	}
+	log.Debug("regEx %v", regExStr)
+	regEx, err := regexp.Compile(regExStr)
+	return regEx, matchPrefs, err
+}
+
+func matchOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	regEx, matchPrefs, err := extractMatchArguments(d, context, expressionNode)
+	if err != nil {
+		return Context{}, err
+	}
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		node := unwrapDoc(candidate.Node)
+		if node.Tag != "!!str" {
+			return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
+		}
+
+		match(matchPrefs, regEx, candidate, node.Value, results)
+	}
+
+	return context.ChildContext(results), nil
+}
+
+func captureOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	regEx, matchPrefs, err := extractMatchArguments(d, context, expressionNode)
+	if err != nil {
+		return Context{}, err
+	}
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		node := unwrapDoc(candidate.Node)
+		if node.Tag != "!!str" {
+			return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
+		}
+		capture(matchPrefs, regEx, candidate, node.Value, results)
+
+	}
+
+	return context.ChildContext(results), nil
+}
+
+func testOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	regEx, _, err := extractMatchArguments(d, context, expressionNode)
+	if err != nil {
+		return Context{}, err
+	}
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		node := unwrapDoc(candidate.Node)
+		if node.Tag != "!!str" {
+			return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
+		}
+		matches := regEx.FindStringSubmatch(node.Value)
+		results.PushBack(createBooleanCandidate(candidate, len(matches) > 0))
+
+	}
+
+	return context.ChildContext(results), nil
+}
+
 func joinStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- joinStringOperator")
 	joinStr := ""

pkg/yqlib/operator_strings_test.go

@@ -13,6 +13,110 @@ var stringsOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!str)::cat; meow; 1; ; true\n",
 		},
 	},
+	{
+		description: "Match string",
+		document:    `foo bar foo`,
+		expression:  `match("foo")`,
+		expected: []string{
+			"D0, P[], ()::string: foo\noffset: 0\nlength: 3\ncaptures: []\n",
+		},
+	},
+	{
+		description: "Match string, case insensitive",
+		document:    `foo bar FOO`,
+		expression:  `[match("(?i)foo"; "g")]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- string: foo\n  offset: 0\n  length: 3\n  captures: []\n- string: FOO\n  offset: 8\n  length: 3\n  captures: []\n",
+		},
+	},
+	{
+		description: "Match with capture groups",
+		document:    `abc abc`,
+		expression:  `[match("(abc)+"; "g")]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- string: abc\n  offset: 0\n  length: 3\n  captures:\n    - string: abc\n      offset: 0\n      length: 3\n- string: abc\n  offset: 4\n  length: 3\n  captures:\n    - string: abc\n      offset: 4\n      length: 3\n",
+		},
+	},
+	{
+		description: "Match with named capture groups",
+		document:    `foo bar foo foo  foo`,
+		expression:  `[match("foo (?P<bar123>bar)? foo"; "g")]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- string: foo bar foo\n  offset: 0\n  length: 11\n  captures:\n    - string: bar\n      offset: 4\n      length: 3\n      name: bar123\n- string: foo  foo\n  offset: 12\n  length: 8\n  captures:\n    - string: null\n      offset: -1\n      length: 0\n      name: bar123\n",
+		},
+	},
+	{
+		description: "Capture named groups into a map",
+		document:    `xyzzy-14`,
+		expression:  `capture("(?P<a>[a-z]+)-(?P<n>[0-9]+)")`,
+		expected: []string{
+			"D0, P[], ()::a: xyzzy\nn: \"14\"\n",
+		},
+	},
+	{
+		skipDoc:     true,
+		description: "Capture named groups into a map, with null",
+		document:    `xyzzy-14`,
+		expression:  `capture("(?P<a>[a-z]+)-(?P<n>[0-9]+)(?P<bar123>bar)?")`,
+		expected: []string{
+			"D0, P[], ()::a: xyzzy\nn: \"14\"\nbar123: null\n",
+		},
+	},
+	{
+		description: "Match without global flag",
+		document:    `cat cat`,
+		expression:  `match("cat")`,
+		expected: []string{
+			"D0, P[], ()::string: cat\noffset: 0\nlength: 3\ncaptures: []\n",
+		},
+	},
+	{
+		description: "Match with global flag",
+		document:    `cat cat`,
+		expression:  `[match("cat"; "g")]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- string: cat\n  offset: 0\n  length: 3\n  captures: []\n- string: cat\n  offset: 4\n  length: 3\n  captures: []\n",
+		},
+	},
+	{
+		skipDoc:     true,
+		description: "No match",
+		document:    `dog`,
+		expression:  `match("cat"; "g")`,
+		expected:    []string{},
+	},
+	{
+		skipDoc:     true,
+		description: "No match",
+		expression:  `"dog" | match("cat", "g")`,
+		expected:    []string{},
+	},
+	{
+		skipDoc:     true,
+		description: "No match",
+		expression:  `"dog" | match("cat")`,
+		expected:    []string{},
+	},
+	{
+		description:    "Test using regex",
+		subdescription: "Like jq'q equivalant, this works like match but only returns true/false instead of full match details",
+		document:       `["cat", "dog"]`,
+		expression:     `.[] | test("at")`,
+		expected: []string{
+			"D0, P[0], (!!bool)::true\n",
+			"D0, P[1], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `["cat*", "cat*", "cat"]`,
+		expression: `.[] | test("cat\*")`,
+		expected: []string{
+			"D0, P[0], (!!bool)::true\n",
+			"D0, P[1], (!!bool)::true\n",
+			"D0, P[2], (!!bool)::false\n",
+		},
+	},
 	{
 		description:    "Substitute / Replace string",
 		subdescription: "This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)\nNote the use of `|=` to run in context of the current string value.",

pkg/yqlib/operator_subtract.go

@@ -15,11 +15,7 @@ func createSubtractOp(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode
 }
 
 func subtractAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	assignmentOp := &Operation{OperationType: assignOpType}
-	assignmentOp.UpdateAssign = true
-	selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
-	assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: expressionNode.Lhs, Rhs: createSubtractOp(selfExpression, expressionNode.Rhs)}
-	return d.GetMatchingNodes(context, assignmentOpNode)
+	return compoundAssignFunction(d, context, expressionNode, createSubtractOp)
 }
 
 func subtractOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {

pkg/yqlib/operator_traverse_path.go

@@ -127,9 +127,16 @@ func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesT
 	node := matchingNode.Node
 	if node.Tag == "!!null" {
 		log.Debugf("OperatorArrayTraverse got a null - turning it into an empty array")
-		// auto vivification, make it into an empty array
+		// auto vivification
 		node.Tag = ""
 		node.Kind = yaml.SequenceNode
+		//check that the indices are numeric, if not, then we should create an object
+		if len(indicesToTraverse) != 0 {
+			_, err := strconv.ParseInt(indicesToTraverse[0].Value, 10, 64)
+			if err != nil {
+				node.Kind = yaml.MappingNode
+			}
+		}
 	}
 
 	if node.Kind == yaml.AliasNode {
@@ -308,6 +315,6 @@ func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *C
 
 func traverseArray(candidate *CandidateNode, operation *Operation, prefs traversePreferences) (*list.List, error) {
 	log.Debug("operation Value %v", operation.Value)
-	indices := []*yaml.Node{&yaml.Node{Value: operation.StringValue}}
+	indices := []*yaml.Node{{Value: operation.StringValue}}
 	return traverseArrayWithIndices(candidate, indices, prefs)
 }

pkg/yqlib/operator_traverse_path_test.go

@@ -35,6 +35,14 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[0 0], (!!int)::1\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `b: cat`,
+		expression: ".b\n",
+		expected: []string{
+			"D0, P[b], (!!str)::cat\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `[[[1]]]`,
@@ -43,6 +51,13 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[0 0 0], (!!int)::1\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		expression: `.["cat"] = "thing"`,
+		expected: []string{
+			"D0, P[], ()::cat: thing\n",
+		},
+	},
 	{
 		description: "Simple map navigation",
 		document:    `{a: {b: apple}}`,
@@ -77,6 +92,14 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[{}], (!!str)::frog\n",
 		},
 	},
+	{
+		description: "Nested special characters",
+		document:    `a: {"key.withdots": {"another.key": apple}}`,
+		expression:  `.a["key.withdots"]["another.key"]`,
+		expected: []string{
+			"D0, P[a key.withdots another.key], (!!str)::apple\n",
+		},
+	},
 	{
 		description:    "Keys with spaces",
 		subdescription: "Use quotes with brackets around path elements with special characters",
@@ -244,6 +267,15 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[0], (!!int)::1\n",
 		},
 	},
+	{
+		description:           "Traversing nested arrays by index",
+		dontFormatInputForDoc: true,
+		document:              `[[], [cat]]`,
+		expression:            `.[1][0]`,
+		expected: []string{
+			"D0, P[1 0], (!!str)::cat\n",
+		},
+	},
 	{
 		description: "Maps with numeric keys",
 		document:    `{2: cat}`,

pkg/yqlib/operators.go

@@ -10,6 +10,32 @@ import (
 
 type operatorHandler func(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error)
 
+type compoundCalculation func(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode
+
+func compoundAssignFunction(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation compoundCalculation) (Context, error) {
+	lhs, err := d.GetMatchingNodes(context, expressionNode.Lhs)
+	if err != nil {
+		return Context{}, err
+	}
+
+	assignmentOp := &Operation{OperationType: assignOpType}
+	valueOp := &Operation{OperationType: valueOpType}
+
+	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		valueOp.CandidateNode = candidate
+		valueExpression := &ExpressionNode{Operation: valueOp}
+
+		assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: valueExpression, Rhs: calculation(valueExpression, expressionNode.Rhs)}
+
+		_, err = d.GetMatchingNodes(context, assignmentOpNode)
+		if err != nil {
+			return Context{}, err
+		}
+	}
+	return context, nil
+}
+
 func unwrapDoc(node *yaml.Node) *yaml.Node {
 	if node.Kind == yaml.DocumentNode {
 		return node.Content[0]
@@ -31,7 +57,9 @@ func resultsForRhs(d *dataTreeNavigator, context Context, lhsCandidate *Candidat
 		if err != nil {
 			return err
 		}
-		results.PushBack(resultCandidate)
+		if resultCandidate != nil {
+			results.PushBack(resultCandidate)
+		}
 		return nil
 	}
 
@@ -42,7 +70,9 @@ func resultsForRhs(d *dataTreeNavigator, context Context, lhsCandidate *Candidat
 		if err != nil {
 			return err
 		}
-		results.PushBack(resultCandidate)
+		if resultCandidate != nil {
+			results.PushBack(resultCandidate)
+		}
 	}
 	return nil
 }
@@ -91,9 +121,12 @@ func crossFunction(d *dataTreeNavigator, context Context, expressionNode *Expres
 		}
 	}
 	if evaluateAllTogether {
+		log.Debug("crossFunction evaluateAllTogether!")
 		return doCrossFunc(d, context, expressionNode, calculation, calcWhenEmpty)
 	}
 
+	log.Debug("crossFunction evaluate apart!")
+
 	for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {
 		innerResults, err := doCrossFunc(d, context.SingleChildContext(matchEl.Value.(*CandidateNode)), expressionNode, calculation, calcWhenEmpty)
 		if err != nil {

pkg/yqlib/operators_test.go

@@ -114,7 +114,7 @@ func formatYaml(yaml string, filename string) string {
 		panic(err)
 	}
 	streamEvaluator := NewStreamEvaluator()
-	err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer)
+	_, err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer)
 	if err != nil {
 		panic(err)
 	}

pkg/yqlib/printer.go

@@ -11,6 +11,12 @@ import (
 type Printer interface {
 	PrintResults(matchingNodes *list.List) error
 	PrintedAnything() bool
+	SetPrintLeadingSeperator(bool)
+
+	SetPreamble(reader io.Reader)
+
+	//e.g. when given a front-matter doc, like jekyll
+	SetAppendix(reader io.Reader)
 }
 
 type resultsPrinter struct {
@@ -25,6 +31,8 @@ type resultsPrinter struct {
 	previousFileIndex  int
 	printedMatches     bool
 	treeNavigator      DataTreeNavigator
+	preambleReader     io.Reader
+	appendixReader     io.Reader
 }
 
 func NewPrinter(writer io.Writer, outputToJSON bool, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
@@ -40,6 +48,21 @@ func NewPrinter(writer io.Writer, outputToJSON bool, unwrapScalar bool, colorsEn
 	}
 }
 
+func (p *resultsPrinter) SetPrintLeadingSeperator(printLeadingSeperator bool) {
+	if printLeadingSeperator {
+		p.firstTimePrinting = false
+		p.previousFileIndex = -1
+	}
+}
+
+func (p *resultsPrinter) SetPreamble(reader io.Reader) {
+	p.preambleReader = reader
+}
+
+func (p *resultsPrinter) SetAppendix(reader io.Reader) {
+	p.appendixReader = reader
+}
+
 func (p *resultsPrinter) PrintedAnything() bool {
 	return p.printedMatches
 }
@@ -87,6 +110,14 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 	bufferedWriter := bufio.NewWriter(p.writer)
 	defer p.safelyFlush(bufferedWriter)
 
+	if p.preambleReader != nil && !p.outputToJSON {
+		log.Debug("Piping preamble reader...")
+		_, err := io.Copy(bufferedWriter, p.preambleReader)
+		if err != nil {
+			return err
+		}
+	}
+
 	if matchingNodes.Len() == 0 {
 		log.Debug("no matching results, nothing to print")
 		return nil
@@ -115,5 +146,14 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 		p.previousDocIndex = mappedDoc.Document
 	}
 
+	if p.appendixReader != nil && !p.outputToJSON {
+		log.Debug("Piping appendix reader...")
+		betterReader := bufio.NewReader(p.appendixReader)
+		_, err := io.Copy(bufferedWriter, betterReader)
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }

pkg/yqlib/printer_test.go

@@ -17,12 +17,36 @@ a: apple
 a: coconut
 `
 
+var leadingSeperatorSample = `---
+a: good doc
+`
+
 func nodeToList(candidate *CandidateNode) *list.List {
 	elMap := list.New()
 	elMap.PushBack(candidate)
 	return elMap
 }
 
+func TestPrinterWithLeadingSeperator(t *testing.T) {
+	var output bytes.Buffer
+	var writer = bufio.NewWriter(&output)
+	printer := NewPrinter(writer, false, true, false, 2, true)
+
+	inputs, err := readDocuments(strings.NewReader(leadingSeperatorSample), "sample.yml", 0)
+	if err != nil {
+		panic(err)
+	}
+	printer.SetPrintLeadingSeperator(true)
+
+	err = printer.PrintResults(inputs)
+	if err != nil {
+		panic(err)
+	}
+
+	writer.Flush()
+	test.AssertResult(t, leadingSeperatorSample, output.String())
+}
+
 func TestPrinterMultipleDocsInSequence(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)

pkg/yqlib/stream_evaluator.go

@@ -12,7 +12,7 @@ import (
 // Uses less memory than loading all documents and running the expression once, but this cannot process
 // cross document expressions.
 type StreamEvaluator interface {
-	Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer) error
+	Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer) (uint, error)
 	EvaluateFiles(expression string, filenames []string, printer Printer) error
 	EvaluateNew(expression string, printer Printer) error
 }
@@ -49,34 +49,55 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error
 }
 
 func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error {
-
+	var totalProcessDocs uint = 0
 	node, err := s.treeCreator.ParseExpression(expression)
 	if err != nil {
 		return err
 	}
 
-	for _, filename := range filenames {
-		reader, err := readStream(filename)
+	for index, filename := range filenames {
+		reader, leadingSeperator, err := readStream(filename)
+		if index == 0 && leadingSeperator {
+			printer.SetPrintLeadingSeperator(leadingSeperator)
+		}
 		if err != nil {
 			return err
 		}
-		err = s.Evaluate(filename, reader, node, printer)
+		processedDocs, err := s.Evaluate(filename, reader, node, printer)
 		if err != nil {
 			return err
 		}
+		totalProcessDocs = totalProcessDocs + processedDocs
 
 		switch reader := reader.(type) {
 		case *os.File:
 			safelyCloseFile(reader)
 		}
 	}
+
+	if totalProcessDocs == 0 {
+
+		if len(filenames) > 0 {
+			reader, _, err := readStream(filenames[0])
+			if err != nil {
+				return err
+			}
+			switch reader := reader.(type) {
+			case *os.File:
+				defer safelyCloseFile(reader)
+			}
+			printer.SetPreamble(reader)
+		}
+
+		return s.EvaluateNew(expression, printer)
+	}
+
 	return nil
 }
 
-func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer) error {
+func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer) (uint, error) {
 
 	var currentIndex uint
-
 	decoder := yaml.NewDecoder(reader)
 	for {
 		var dataBucket yaml.Node
@@ -84,9 +105,9 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr
 
 		if errorReading == io.EOF {
 			s.fileIndex = s.fileIndex + 1
-			return nil
+			return currentIndex, nil
 		} else if errorReading != nil {
-			return errorReading
+			return currentIndex, errorReading
 		}
 		candidateNode := &CandidateNode{
 			Document:  currentIndex,
@@ -99,11 +120,11 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr
 
 		result, errorParsing := s.treeNavigator.GetMatchingNodes(Context{MatchingNodes: inputList}, node)
 		if errorParsing != nil {
-			return errorParsing
+			return currentIndex, errorParsing
 		}
 		err := printer.PrintResults(result.MatchingNodes)
 		if err != nil {
-			return err
+			return currentIndex, err
 		}
 		currentIndex = currentIndex + 1
 	}

pkg/yqlib/utils.go

@@ -9,11 +9,37 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-func readStream(filename string) (io.Reader, error) {
+func readStream(filename string) (io.Reader, bool, error) {
+
 	if filename == "-" {
-		return bufio.NewReader(os.Stdin), nil
+		reader := bufio.NewReader(os.Stdin)
+
+		seperatorBytes, err := reader.Peek(3)
+
+		if err == io.EOF {
+			// EOF are handled else where..
+			return reader, false, nil
+		}
+
+		return reader, string(seperatorBytes) == "---", err
 	} else {
-		return os.Open(filename) // nolint gosec
+		// ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory,
+		// and ensuring that it's not possible to give a path to a file outside thar directory.
+		reader, err := os.Open(filename) // #nosec
+		if err != nil {
+			return nil, false, err
+		}
+		seperatorBytes := make([]byte, 3)
+		_, err = reader.Read(seperatorBytes)
+		if err == io.EOF {
+			// EOF are handled else where..
+			return reader, false, nil
+		} else if err != nil {
+			return nil, false, err
+		}
+		_, err = reader.Seek(0, 0)
+
+		return reader, string(seperatorBytes) == "---", err
 	}
 }
 

pkg/yqlib/write_in_place_handler.go

@@ -1,7 +1,6 @@
 package yqlib
 
 import (
-	"io/ioutil"
 	"os"
 )
 
@@ -21,27 +20,21 @@ func NewWriteInPlaceHandler(inputFile string) writeInPlaceHandler {
 }
 
 func (w *writeInPlaceHandlerImpl) CreateTempFile() (*os.File, error) {
+	file, err := createTempFile()
+
+	if err != nil {
+		return nil, err
+	}
 	info, err := os.Stat(w.inputFilename)
 	if err != nil {
 		return nil, err
 	}
-	_, err = os.Stat(os.TempDir())
-	if os.IsNotExist(err) {
-		err = os.Mkdir(os.TempDir(), 0700)
-		if err != nil {
-			return nil, err
-		}
-	} else if err != nil {
-		return nil, err
-	}
+	err = os.Chmod(file.Name(), info.Mode())
 
-	file, err := ioutil.TempFile("", "temp")
 	if err != nil {
 		return nil, err
 	}
-
-	err = os.Chmod(file.Name(), info.Mode())
-	log.Debug("writing to tempfile: %v", file.Name())
+	log.Debug("WriteInPlaceHandler: writing to tempfile: %v", file.Name())
 	w.tempFile = file
 	return file, err
 }
@@ -50,10 +43,9 @@ func (w *writeInPlaceHandlerImpl) FinishWriteInPlace(evaluatedSuccessfully bool)
 	log.Debug("Going to write-inplace, evaluatedSuccessfully=%v, target=%v", evaluatedSuccessfully, w.inputFilename)
 	safelyCloseFile(w.tempFile)
 	if evaluatedSuccessfully {
-		log.Debug("moved temp file to target")
+		log.Debug("Moving temp file to target")
 		safelyRenameFile(w.tempFile.Name(), w.inputFilename)
 	} else {
-		log.Debug("removed temp file")
-		os.Remove(w.tempFile.Name())
+		tryRemoveFile(w.tempFile.Name())
 	}
 }

release_instructions.txt

@@ -12,6 +12,7 @@
 
 - snapcraft
   - will auto create a candidate, test it works then promote
+  - !! need to update v4/stable as well as latest
   
   sudo snap remove yq
   sudo snap install --edge yq

scripts/acceptance.sh

@@ -55,6 +55,212 @@ if [[ $? != 1 ]]; then
   exit 1
 fi
 
+echo "Test: leading seperator logic"
+expected=$(cat examples/leading-seperator.yaml)
+
+X=$(cat examples/leading-seperator.yaml | ./yq e '.' -)
+if [[ $X != $expected ]]; then
+  echo "Pipe into e"
+  echo "Expected $expected but was $X"
+  exit 1
+fi
+
+X=$(./yq e '.' examples/leading-seperator.yaml)
+expected=$(cat examples/leading-seperator.yaml)
+if [[ $X != $expected ]]; then
+  echo "read given file e"
+  echo "Expected $expected but was $X"
+  exit 1
+fi
+
+X=$(cat examples/leading-seperator.yaml | ./yq ea '.' -)
+if [[ $X != $expected ]]; then
+  echo "Pipe into e"
+  echo "Expected $expected but was $X"
+  exit 1
+fi
+
+X=$(./yq ea '.' examples/leading-seperator.yaml)
+expected=$(cat examples/leading-seperator.yaml)
+if [[ $X != $expected ]]; then
+  echo "read given file e"
+  echo "Expected $expected but was $X"
+  exit 1
+fi
+
+# multidoc
+read -r -d '' expected << EOM
+---
+a: test
+---
+version: 3
+application: MyApp
+EOM
+
+X=$(./yq e '.' examples/leading-seperator.yaml examples/order.yaml)
+
+if [[ $X != $expected ]]; then
+  echo "Multidoc with leading seperator"
+  echo "Expected $expected but was $X"
+  exit 1
+fi
+
+X=$(./yq ea '.' examples/leading-seperator.yaml examples/order.yaml)
+
+if [[ $X != $expected ]]; then
+  echo "Multidoc with leading seperator"
+  echo "Expected $expected but was $X"
+  exit 1
+fi
+
+echo "Test: handle empty files"
+./yq e '.' examples/empty.yaml
+if [[ $? != 0 ]]; then
+  echo "Expected no error when processing empty file but got one"
+  exit 1
+fi
+
+cat examples/empty.yaml | ./yq e '.' -
+if [[ $? != 0 ]]; then
+  echo "Expected no error when processing empty stdin but got one"
+  exit 1
+fi
+
+# run expression against empty file
+rm -f temp.yaml
+touch temp.yaml
+expected="apple: tree"
+
+./yq e '.apple = "tree"' temp.yaml -i
+X=$(cat temp.yaml)
+rm -f temp.yaml
+if [[ $X != $expected ]]; then
+  echo "Write empty doc"
+  echo "Expected $expected but was $X"
+  exit 1
+fi
+
+touch temp.yaml
+
+./yq ea '.apple = "tree"' temp.yaml -i
+X=$(cat temp.yaml)
+rm -f temp.yaml
+if [[ $X != $expected ]]; then
+  echo "Write all empty doc"
+  echo "Expected $expected but was $X"
+  exit 1
+fi
+
+echo "Test: handle empty files with just comments"
+rm -f temp.yaml
+echo "# comment" > temp.yaml
+read -r -d '' expected << EOM
+# comment
+apple: tree
+EOM
+
+./yq e '.apple = "tree"' temp.yaml -i
+X=$(cat temp.yaml)
+rm -f temp.yaml
+if [[ $X != $expected ]]; then
+  echo "Write empty doc"
+  echo "Expected $expected but was $X"
+  exit 1
+fi
+
+echo "# comment" > temp.yaml
+
+./yq ea '.apple = "tree"' temp.yaml -i
+X=$(cat temp.yaml)
+rm -f temp.yaml
+if [[ $X != $expected ]]; then
+  echo "Write all empty doc"
+  echo "Expected $expected but was $X"
+  exit 1
+fi
+
+echo "Test: eval front matter process"
+
+cat >temp.yaml <<EOL
+---
+a: apple
+b: cat
+---
+not yaml
+c: at
+EOL
+
+read -r -d '' expected << EOM
+---
+a: apple
+b: dog
+---
+not yaml
+c: at
+EOM
+
+./yq e --front-matter="process" '.b = "dog"' temp.yaml -i
+X=$(cat temp.yaml)
+rm -f temp.yaml
+if [[ $X != $expected ]]; then
+  echo "eval fail"
+  echo "Expected $expected but was $X"
+  exit 1
+fi
+
+echo "Test: eval front matter extract"
+
+cat >temp.yaml <<EOL
+a: apple
+b: cat
+---
+not yaml
+c: at
+EOL
+
+read -r -d '' expected << EOM
+a: apple
+b: dog
+EOM
+
+./yq e --front-matter="extract" '.b = "dog"' temp.yaml -i
+X=$(cat temp.yaml)
+rm -f temp.yaml
+if [[ $X != $expected ]]; then
+  echo "eval fail"
+  echo "Expected $expected but was $X"
+  exit 1
+fi
+
+echo "Test: eval-all front matter process"
+
+cat >temp.yaml <<EOL
+---
+a: apple
+b: cat
+---
+not yaml
+c: at
+EOL
+
+read -r -d '' expected << EOM
+---
+a: apple
+b: dog
+---
+not yaml
+c: at
+EOM
+
+./yq ea --front-matter="process" '.b = "dog"' temp.yaml -i
+X=$(cat temp.yaml)
+rm -f temp.yaml
+if [[ $X != $expected ]]; then
+  echo "eval fail"
+  echo "Expected $expected but was $X"
+  exit 1
+fi
+
 echo "--success"
 
 set -e

scripts/format.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
 find . \( -path ./vendor \) -prune -o -name "*.go" -exec goimports -w {} \;
+gofmt -w -s .
 go mod tidy
-go mod vendor
+go mod vendor

scripts/secure.sh

@@ -5,7 +5,7 @@ set -o pipefail
 
 if command -v gosec &> /dev/null
 then
-    gosec ${PWD}
+    gosec ${PWD} ./...
 else
-  ./bin/gosec ${PWD}
+  ./bin/gosec ${PWD} ./...
 fi

scripts/xcompile.sh

@@ -4,15 +4,16 @@ set -e
 # at https://github.com/inconshreveable/gonative
 
 
-CGO_ENABLED=0 gox -ldflags "${LDFLAGS}" -output="build/yq_{{.OS}}_{{.Arch}}" --osarch="darwin/amd64 freebsd/386 freebsd/amd64 freebsd/arm linux/386 linux/amd64 linux/arm linux/arm64 linux/mips linux/mips64 linux/mips64le linux/mipsle linux/ppc64 linux/ppc64le linux/s390x netbsd/386 netbsd/amd64 netbsd/arm openbsd/386 openbsd/amd64 windows/386 windows/amd64"
+CGO_ENABLED=0 gox -ldflags "${LDFLAGS}" -output="build/yq_{{.OS}}_{{.Arch}}" --osarch="darwin/amd64 darwin/arm64 freebsd/386 freebsd/amd64 freebsd/arm linux/386 linux/amd64 linux/arm linux/arm64 linux/mips linux/mips64 linux/mips64le linux/mipsle linux/ppc64 linux/ppc64le linux/s390x netbsd/386 netbsd/amd64 netbsd/arm openbsd/386 openbsd/amd64 windows/386 windows/amd64"
 
 cd build
+
+find . -executable -type f | xargs -I {} tar czvf {}.tar.gz {}
+
 rhash -r -a . -o checksums
 
 rhash --list-hashes > checksums_hashes_order
 
-find . -executable -type f | xargs -I {} tar czvf {}.tar.gz {}
-
 # just in case find thinks this is executable...
 rm -f checksums_hashes_order.tar.gz 
 rm -f checksums.tar.gz
@@ -21,4 +22,4 @@ rm yq_windows_386.exe.tar.gz
 rm yq_windows_amd64.exe.tar.gz
 
 zip yq_windows_386.zip yq_windows_386.exe
-zip yq_windows_amd64.zip yq_windows_amd64.exe
+zip yq_windows_amd64.zip yq_windows_amd64.exe

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.9.3'
+version: '4.10.0'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.

test/utils.go

@@ -81,7 +81,9 @@ func WriteTempYamlFile(content string) string {
 }
 
 func ReadTempYamlFile(name string) string {
-	content, _ := ioutil.ReadFile(name)
+	// ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory,
+	// and ensuring that it's not possible to give a path to a file outside thar directory.
+	content, _ := ioutil.ReadFile(name) // #nosec
 	return string(content)
 }