Better way of processing leading content

This reverts commit ffb6cbf769.

.github/workflows/test-yq.yml

@@ -0,0 +1,19 @@
+name: Test Yq Action
+on: [push]
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get test
+        uses: mikefarah/yq@master
+        with:
+          cmd: yq eval '.a' examples/data1.yaml
+      - name: Write inplace test
+        id: lookupSdkVersion
+        uses: mikefarah/yq@master
+        with:
+          cmd: yq eval -i '.a.b = 5' examples/data1.yaml

.gitignore

@@ -38,3 +38,5 @@ parts/
 prime/
 .snapcraft/
 yq*.snap
+test.yml
+test2.yml

Dockerfile

@@ -12,7 +12,12 @@ RUN CGO_ENABLED=0 make local build
 
 # Choose alpine as a base image to make this useful for CI, as many
 # CI tools expect an interactive shell inside the container
-FROM alpine:3.12.3 as production
+FROM alpine:3.13.5 as production
+
+RUN mkdir /home/yq/
+RUN addgroup -g 1000 yq && \
+    adduser -u 1000 -G yq -s /bin/bash -h /home/yq -D yq
+RUN chown -R yq:yq /home/yq/
 
 COPY --from=builder /go/src/mikefarah/yq/yq /usr/bin/yq
 RUN chmod +x /usr/bin/yq
@@ -20,6 +25,8 @@ RUN chmod +x /usr/bin/yq
 ARG VERSION=none
 LABEL version=${VERSION}
 
+USER yq
+
 WORKDIR /workdir
 
 ENTRYPOINT ["/usr/bin/yq"]

README.md

@@ -101,6 +101,25 @@ yq() {
 }
 ```
 
+#### Running as root:
+
+`yq`'s docker image no longer runs under root (https://github.com/mikefarah/yq/pull/860). If you'd like to install more things in the docker image, or you're having permissions issues when attempting to read/write files you'll need to either:
+
+
+```
+docker run --user="root" -it --entrypoint sh mikefarah/yq
+```
+
+Or, in your docker file:
+
+```
+FROM mikefarah/yq
+
+USER root
+RUN apk add bash
+USER yq
+```
+
 ### Go Get:
 ```
 GO111MODULE=on go get github.com/mikefarah/yq/v4
@@ -157,6 +176,7 @@ Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
 - Written in portable go, so you can download a lovely dependency free binary
 - Uses similar syntax as `jq` but works with YAML and JSON files
 - Fully supports multi document yaml files
+- Supports yaml [front matter](https://mikefarah.gitbook.io/yq/usage/front-matter) blocks (e.g. jekyll/assemble)
 - Colorized yaml output
 - [Deeply traverse yaml](https://mikefarah.gitbook.io/yq/operators/traverse-read)
 - [Sort yaml by keys](https://mikefarah.gitbook.io/yq/operators/sort-keys)
@@ -168,6 +188,7 @@ Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
 - [Pipe data in by using '-'](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate)
 - [General shell completion scripts (bash/zsh/fish/powershell)](https://mikefarah.gitbook.io/yq/v/v4.x/commands/shell-completion)
 - [Reduce](https://mikefarah.gitbook.io/yq/operators/reduce) to merge multiple files or sum an array or other fancy things.
+- [Github Action](https://mikefarah.gitbook.io/yq/usage/github-action) to use in your automated pipeline (thanks @devorbitus)
 
 ## [Usage](https://mikefarah.gitbook.io/yq/)
 
@@ -179,24 +200,26 @@ Usage:
   yq [command]
 
 Available Commands:
-  eval             Apply expression to each document in each yaml file given in sequence
+  eval             Apply the expression to each document in each yaml file in sequence
   eval-all         Loads _all_ yaml documents of _all_ yaml files and runs expression once
   help             Help about any command
   shell-completion Generate completion script
 
 Flags:
-  -C, --colors        force print with colors
-  -e, --exit-status   set exit status if there are no matches or null or false is returned
-  -h, --help          help for yq
-  -I, --indent int    sets indent level for output (default 2)
-  -i, --inplace       update the yaml file inplace of first yaml file given.
-  -M, --no-colors     force print with no colors
-  -N, --no-doc        Don't print document separators (---)
-  -n, --null-input    Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.
-  -P, --prettyPrint    pretty print, shorthand for '... style = ""' 
-  -j, --tojson        output as json. Set indent to 0 to print json in one line.
-  -v, --verbose       verbose mode
-  -V, --version       Print version information and quit
+  -C, --colors                force print with colors
+  -e, --exit-status           set exit status if there are no matches or null or false is returned
+  -f, --front-matter string   (extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact
+  -h, --help                  help for yq
+  -I, --indent int            sets indent level for output (default 2)
+  -i, --inplace               update the yaml file inplace of first yaml file given.
+  -M, --no-colors             force print with no colors
+  -N, --no-doc                Don't print document separators (---)
+  -n, --null-input            Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.
+  -P, --prettyPrint           pretty print, shorthand for '... style = ""'
+  -j, --tojson                output as json. Set indent to 0 to print json in one line.
+      --unwrapScalar          unwrap scalar, print the value with no quotes, colors or comments (default true)
+  -v, --verbose               verbose mode
+  -V, --version               Print version information and quit
 
 Use "yq [command] --help" for more information about a command.
 ```
@@ -209,3 +232,6 @@ yq e '.a.b | length' f1.yml f2.yml
 
 ## Known Issues / Missing Features
 - `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details)
+- Powershell has its own...opinions: https://mikefarah.gitbook.io/yq/usage/tips-and-tricks#quotes-in-windows-powershell
+
+See [tips and tricks](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks) for more common problems and solutions.

acceptance_tests/basic.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+setUp() {
+  rm -f test.yml
+}
+
+testBasicEvalRoundTrip() {
+  ./yq e -n ".a = 123" > test.yml
+  X=$(./yq e '.a' test.yml)
+  assertEquals 123 "$X"
+}
+
+testBasicUpdateInPlaceSequence() {
+  cat >test.yml <<EOL
+a: 0
+EOL
+  ./yq e -i ".a = 10" test.yml
+  X=$(./yq e '.a' test.yml)
+  assertEquals "10" "$X"
+}
+
+testBasicUpdateInPlaceSequenceEvalAll() {
+  cat >test.yml <<EOL
+a: 0
+EOL
+  ./yq ea -i ".a = 10" test.yml
+  X=$(./yq e '.a' test.yml)
+  assertEquals "10" "$X"
+}
+
+testBasicNoExitStatus() {
+  echo "a: cat" > test.yml
+  X=$(./yq e '.z' test.yml)
+  assertEquals "null" "$X"
+}
+
+testBasicExitStatus() {
+  echo "a: cat" > test.yml
+  X=$(./yq e -e '.z' test.yml 2&>/dev/null)
+  assertEquals 1 "$?"
+}
+
+testBasicExtractFieldWithSeperator() {
+    cat >test.yml <<EOL
+---
+name: chart-name
+version: 1.2.3
+EOL
+  X=$(./yq e '.name' test.yml)
+  assertEquals "chart-name" "$X"
+}
+
+testBasicExtractMultipleFieldWithSeperator() {
+    cat >test.yml <<EOL
+---
+name: chart-name
+version: 1.2.3
+---
+name: thing
+version: 1.2.3
+EOL
+
+read -r -d '' expected << EOM
+chart-name
+---
+thing
+EOM
+  X=$(./yq e '.name' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+source ./scripts/shunit2

acceptance_tests/empty.sh

@@ -0,0 +1,62 @@
+#!/bin/bash
+
+setUp() {
+  cat >test.yml <<EOL
+# comment
+EOL
+}
+
+testEmptyEval() {
+  X=$(./yq e test.yml)
+  assertEquals 0 $?
+}
+
+testEmptyEvalPipe() {
+  X=$(./yq e - < test.yml)
+  assertEquals 0 $?
+}
+
+testEmptyCommentsWithExpressionEval() {
+  read -r -d '' expected << EOM
+# comment
+apple: tree
+EOM
+
+  X=$(./yq e '.apple="tree"' test.yml)
+
+  assertEquals "$expected" "$X"
+}
+
+testEmptyCommentsWithExpressionEvalAll() {
+  read -r -d '' expected << EOM
+# comment
+apple: tree
+EOM
+
+  X=$(./yq ea '.apple="tree"' test.yml)
+
+  assertEquals "$expected" "$X"
+}
+
+testEmptyWithExpressionEval() {
+  rm test.yml
+  touch test.yml
+  expected="apple: tree"
+
+  X=$(./yq e '.apple="tree"' test.yml)
+
+  assertEquals "$expected" "$X"
+}
+
+testEmptyWithExpressionEvalAll() {
+  rm test.yml
+  touch test.yml
+  expected="apple: tree"
+
+  X=$(./yq ea '.apple="tree"' test.yml)
+
+  assertEquals "$expected" "$X"
+}
+
+
+source ./scripts/shunit2

acceptance_tests/front-matter.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+setUp() {
+  cat >test.yml <<EOL
+---
+a: apple
+b: cat
+---
+not yaml
+c: at
+EOL
+}
+
+testFrontMatterProcessEval() {
+  read -r -d '' expected << EOM
+---
+a: apple
+b: dog
+---
+not yaml
+c: at
+EOM
+  ./yq e --front-matter="process" '.b = "dog"' test.yml -i
+  assertEquals "$expected" "$(cat test.yml)"
+}
+
+testFrontMatterProcessEvalAll() {
+  read -r -d '' expected << EOM
+---
+a: apple
+b: dog
+---
+not yaml
+c: at
+EOM
+  ./yq ea --front-matter="process" '.b = "dog"' test.yml -i
+  assertEquals "$expected" "$(cat test.yml)"
+}
+
+testFrontMatterExtractEval() {
+    cat >test.yml <<EOL
+a: apple
+b: cat
+---
+not yaml
+c: at
+EOL
+
+  read -r -d '' expected << EOM
+a: apple
+b: dog
+EOM
+  ./yq e --front-matter="extract" '.b = "dog"' test.yml -i
+  assertEquals "$expected" "$(cat test.yml)"
+}
+
+testFrontMatterExtractEvalAll() {
+  cat >test.yml <<EOL
+a: apple
+b: cat
+---
+not yaml
+c: at
+EOL
+
+  read -r -d '' expected << EOM
+a: apple
+b: dog
+EOM
+  ./yq ea --front-matter="extract" '.b = "dog"' test.yml -i
+  assertEquals "$expected" "$(cat test.yml)"
+}
+
+
+source ./scripts/shunit2

acceptance_tests/leading-seperator.sh

@@ -0,0 +1,353 @@
+#!/bin/bash
+
+setUp() {
+  cat >test.yml <<EOL
+---
+a: test
+EOL
+}
+
+testLeadingSeperatorWithDoc() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+---
+a: test
+---
+b: cool
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+---
+a: thing
+---
+b: cool
+EOM
+
+  X=$(./yq e '(select(di == 0) | .a) = "thing"' - < test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorPipeIntoEvalSeq() {
+  X=$(./yq e - < test.yml)
+  expected=$(cat test.yml)
+  assertEquals "$expected" "$X"
+}
+
+
+testLeadingSeperatorEvalSeq() {
+  X=$(./yq e test.yml)
+  expected=$(cat test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorPipeIntoEvalAll() {
+  X=$(./yq ea - < test.yml)
+  expected=$(cat test.yml)
+  assertEquals "$expected" "$X"
+}
+
+
+testLeadingSeperatorEvalAll() {
+  X=$(./yq ea test.yml)
+  expected=$(cat test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalSimple() {
+  read -r -d '' expected << EOM
+---
+a: test
+---
+version: 3
+application: MyApp
+EOM
+
+
+  X=$(./yq e '.' test.yml examples/order.yaml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocInOneFile() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+---
+b: things
+EOL
+  expected=$(cat test.yml)
+  X=$(./yq e '.' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocInOneFileEvalAll() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+---
+b: things
+EOL
+  expected=$(cat test.yml)
+  X=$(./yq ea '.' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalComments() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+a: test
+EOL
+
+cat >test2.yml <<EOL
+# this is another doc
+# great
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+a: test
+---
+# this is another doc
+# great
+b: sane
+EOM
+
+
+  X=$(./yq e '.' test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalCommentsTrailingSep() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+---
+a: test
+EOL
+
+cat >test2.yml <<EOL
+# this is another doc
+# great
+---
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+---
+a: test
+---
+# this is another doc
+# great
+---
+b: sane
+EOM
+
+
+  X=$(./yq e '.' test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiMultiDocEvalCommentsTrailingSep() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+---
+a: test
+---
+a1: test2
+EOL
+
+cat >test2.yml <<EOL
+# this is another doc
+# great
+---
+b: sane
+---
+b2: cool
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+---
+a: test
+---
+a1: test2
+---
+# this is another doc
+# great
+---
+b: sane
+---
+b2: cool
+EOM
+
+
+  X=$(./yq e '.' test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalCommentsLeadingSep() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+EOL
+
+cat >test2.yml <<EOL
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+---
+# hi peeps
+# cool
+a: test
+---
+# this is another doc
+# great
+b: sane
+EOM
+
+
+  X=$(./yq e '.' test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalCommentsStripComments() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  # it will be hard to remove that top level separator
+  read -r -d '' expected << EOM
+a: test
+---
+b: sane
+EOM
+
+  X=$(./yq e '... comments=""'  test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalCommentsLeadingSepNoDocFlag() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+a: test
+# this is another doc
+# great
+b: sane
+EOM
+
+
+  X=$(./yq e '.' --no-doc test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalJsonFlag() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+EOL
+
+cat >test2.yml <<EOL
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+{
+  "a": "test"
+}
+{
+  "b": "sane"
+}
+EOM
+
+
+  X=$(./yq e '.' -j test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalAllJsonFlag() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+EOL
+
+cat >test2.yml <<EOL
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+{
+  "a": "test"
+}
+{
+  "b": "sane"
+}
+EOM
+
+
+  X=$(./yq ea '.' -j test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalAll() {
+  read -r -d '' expected << EOM
+---
+a: test
+---
+version: 3
+application: MyApp
+EOM
+
+
+  X=$(./yq ea '.' test.yml examples/order.yaml)
+  assertEquals "$expected" "$X"
+}
+
+source ./scripts/shunit2

action.yml

@@ -1,11 +1,15 @@
 name: 'yq - portable yaml processor'
 description: 'create, read, update, delete, merge, validate and do more with yaml'
-icon: command
-color: gray-dark
+branding:
+  icon: command
+  color: gray-dark
 inputs:
   cmd:
     description: 'The Command which should be run'
     required: true
+outputs:
+  result:
+    description: "The complete result from the yq command being run"
 runs:
   using: 'docker'
   image: 'github-action/Dockerfile'

cmd/constant.go

@@ -1,5 +1,6 @@
 package cmd
 
+var leadingContentPreProcessing = true
 var unwrapScalar = true
 
 var writeInplace = false
@@ -16,4 +17,7 @@ var verbose = false
 var version = false
 var prettyPrint = false
 
+// can be either "" (off), "extract" or "process"
+var frontMatter = ""
+
 var completedSuccessfully = false

cmd/evaluate_all_command.go

@@ -15,13 +15,27 @@ func createEvaluateAllCommand() *cobra.Command {
 		Aliases: []string{"ea"},
 		Short:   "Loads _all_ yaml documents of _all_ yaml files and runs expression once",
 		Example: `
-# merges f2.yml into f1.yml (inplace)
-yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' f1.yml f2.yml		
+# Merge f2.yml into f1.yml (inplace)
+yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' f1.yml f2.yml
+## the same command and expression using shortened names:
+yq ea -i 'select(fi == 0) * select(fi == 1)' f1.yml f2.yml
 
-# use '-' as a filename to read from STDIN
+
+# Merge all given files
+yq ea '. as $item ireduce ({}; . * $item )' file1.yml file2.yml ...
+
+# Read from STDIN
+## use '-' as a filename to read from STDIN
 cat file2.yml | yq ea '.a.b' file1.yml - file3.yml
 `,
-		Long: "Evaluate All:\nUseful when you need to run an expression across several yaml documents or files. Consumes more memory than eval",
+		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
+See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.
+
+## Evaluate All ##
+This command loads _all_ yaml documents of _all_ yaml files and runs expression once
+Useful when you need to run an expression across several yaml documents or files (like merge).
+Note that it consumes more memory than eval.
+`,
 		RunE: evaluateAll,
 	}
 	return cmdEvalAll
@@ -75,23 +89,42 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 
 	printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
 
+	if frontMatter != "" {
+		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])
+		err = frontMatterHandler.Split()
+		if err != nil {
+			return err
+		}
+		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
+
+		if frontMatter == "process" {
+			reader, err := os.Open(frontMatterHandler.GetContentFilename()) // #nosec
+			if err != nil {
+				return err
+			}
+			printer.SetAppendix(reader)
+			defer yqlib.SafelyCloseReader(reader)
+		}
+		defer frontMatterHandler.CleanUp()
+	}
+
 	allAtOnceEvaluator := yqlib.NewAllAtOnceEvaluator()
 	switch len(args) {
 	case 0:
 		if pipingStdIn {
-			err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer)
+			err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer, leadingContentPreProcessing)
 		} else {
 			cmd.Println(cmd.UsageString())
 			return nil
 		}
 	case 1:
 		if nullInput {
-			err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(args[0]), printer)
+			err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(args[0]), printer, "")
 		} else {
-			err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer)
+			err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer, leadingContentPreProcessing)
 		}
 	default:
-		err = allAtOnceEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer)
+		err = allAtOnceEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing)
 	}
 
 	completedSuccessfully = err == nil

cmd/evalute_sequence_command.go

@@ -13,25 +13,31 @@ func createEvaluateSequenceCommand() *cobra.Command {
 	var cmdEvalSequence = &cobra.Command{
 		Use:     "eval [expression] [yaml_file1]...",
 		Aliases: []string{"e"},
-		Short:   "Apply expression to each document in each yaml file given in sequence",
+		Short:   "Apply the expression to each document in each yaml file in sequence",
 		Example: `
-# runs the expression against each file, in series
-yq e '.a.b | length' f1.yml f2.yml 
+# Reads field under the given path for each file
+yq e '.a.b' f1.yml f2.yml 
 
-# prints out the file
+# Prints out the file
 yq e sample.yaml 
 
-# use '-' as a filename to read from STDIN
+# Read from STDIN
+## use '-' as a filename to read from STDIN
 cat file2.yml | yq e '.a.b' file1.yml - file3.yml
 
-# prints a new yaml document
+# Creates a new yaml document
+## Note that editing an empty file does not work.
 yq e -n '.a.b.c = "cat"' 
 
-
-# updates file.yaml directly
+# Update a file inplace
 yq e '.a.b = "cool"' -i file.yaml 
 `,
-		Long: "Evaluate Sequence:\nIterate over each yaml document, apply the expression and print the results, in sequence.",
+		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
+See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.
+
+## Evaluate Sequence ##
+This command iterates over each yaml document from each given file, applies the 
+expression and prints the result in sequence.`,
 		RunE: evaluateSequence,
 	}
 	return cmdEvalSequence
@@ -97,22 +103,41 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 		return errors.New("Cannot pass files in when using null-input flag")
 	}
 
+	if frontMatter != "" {
+		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])
+		err = frontMatterHandler.Split()
+		if err != nil {
+			return err
+		}
+		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
+
+		if frontMatter == "process" {
+			reader, err := os.Open(frontMatterHandler.GetContentFilename()) // #nosec
+			if err != nil {
+				return err
+			}
+			printer.SetAppendix(reader)
+			defer yqlib.SafelyCloseReader(reader)
+		}
+		defer frontMatterHandler.CleanUp()
+	}
+
 	switch len(args) {
 	case 0:
 		if pipingStdIn {
-			err = streamEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer)
+			err = streamEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer, leadingContentPreProcessing)
 		} else {
 			cmd.Println(cmd.UsageString())
 			return nil
 		}
 	case 1:
 		if nullInput {
-			err = streamEvaluator.EvaluateNew(processExpression(args[0]), printer)
+			err = streamEvaluator.EvaluateNew(processExpression(args[0]), printer, "")
 		} else {
-			err = streamEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer)
+			err = streamEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer, leadingContentPreProcessing)
 		}
 	default:
-		err = streamEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer)
+		err = streamEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing)
 	}
 	completedSuccessfully = err == nil
 

cmd/root.go

@@ -11,7 +11,8 @@ func New() *cobra.Command {
 	var rootCmd = &cobra.Command{
 		Use:   "yq",
 		Short: "yq is a lightweight and portable command-line YAML processor.",
-		Long:  `yq is a lightweight and portable command-line YAML processor. It aims to be the jq or sed of yaml files.`,
+		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
+See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if version {
 				cmd.Print(GetVersionDisplay())
@@ -53,6 +54,8 @@ func New() *cobra.Command {
 
 	rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")
 	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
+	rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
+	rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and seperators before processing expression. This is a workaround for go-yaml to persist header content. This flag will be removed once this feature has been out in the wild for a while.")
 	rootCmd.AddCommand(
 		createEvaluateSequenceCommand(),
 		createEvaluateAllCommand(),

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.7.1"
+	Version = "4.11.0"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release
@@ -25,7 +25,7 @@ const ProductName = "yq"
 // GetVersionDisplay composes the parts of the version in a way that's suitable
 // for displaying to humans.
 func GetVersionDisplay() string {
-	return fmt.Sprintf("%s version %s\n", ProductName, getHumanVersion())
+	return fmt.Sprintf("yq (https://github.com/mikefarah/yq/) version %s\n", getHumanVersion())
 }
 
 func getHumanVersion() string {

cmd/version_test.go

@@ -3,7 +3,7 @@ package cmd
 import "testing"
 
 func TestGetVersionDisplay(t *testing.T) {
-	var expectedVersion = ProductName + " version " + Version
+	var expectedVersion = ProductName + " (https://github.com/mikefarah/yq/) version " + Version
 	if VersionPrerelease != "" {
 		expectedVersion = expectedVersion + "-" + VersionPrerelease
 	}

debian/changelog

@@ -1,3 +1,12 @@
+yq (4.9.6) focal; urgency=medium
+
+  * Added darwin/arm64 build, thanks @alecthomas
+  * Incremented docker alpine base version, thanks @da6d6i7-bronga
+  * Bug fix: multine expression
+  * Bug fix: special character
+
+ -- Roberto Mier Escandon <rmescandon@gmail.com>  Tue, 29 Jun 2021 21:32:14 +0200
+
 yq (3.3.2) focal; urgency=medium
 
   * Bug fix: existStatus bug (#459)

debian/control

@@ -3,8 +3,7 @@ Section: devel
 Priority: optional
 Maintainer: Roberto Mier Escandón <rmescandon@gmail.com>
 Build-Depends: debhelper (>=10),
-               dh-golang (>=1.34),
-               golang-1.13,
+               golang-1.15,
                rsync
 Standards-Version: 4.1.4
 Homepage: https://github.com/mikefarah/yq.git

debian/rules

@@ -17,7 +17,7 @@
 PROJECT := yq
 OWNER := mikefarah
 REPO := github.com
-GOVERSION := 1.13
+GOVERSION := 1.15
 
 export DH_OPTIONS
 export DH_GOPKG := ${REPO}/${OWNER}/${PROJECT}
@@ -34,7 +34,7 @@ BINDIR := /usr/bin
 ASSETSDIR := /usr/share/${PROJECT}
 
 %:
-	dh $@ --builddirectory=${GOPATH} --buildsystem=golang
+	dh $@
 
 override_dh_auto_build:
 	mkdir -p ${SRCDIR}

examples/data1.yaml

@@ -1,6 +1,4 @@
-a:
-  key1: "value1"
-  key2: 2.6
-  ab:
-    key1: 6
-    key2: "h"
+---
+# hi peeps
+# cool
+a: test

examples/empty.yaml

@@ -1,2 +1 @@
-# a: apple
-# b: cat
+# comment

examples/front-matter.yaml

@@ -0,0 +1,5 @@
+---
+a: apple
+b: bannana
+---
+<h1>I like {{a}} and {{b}} </h1>

examples/leading-seperator.yaml

@@ -0,0 +1,2 @@
+---
+a: test

github-action/Dockerfile

@@ -1,5 +1,8 @@
-FROM mikefarah/yq:4.7.1
+FROM mikefarah/yq:4.11.0
 
 COPY entrypoint.sh /entrypoint.sh
 
+# this seems to be the default user in github actions
+USER 1001
+
 ENTRYPOINT ["/entrypoint.sh"]

github-action/entrypoint.sh

@@ -1,4 +1,6 @@
 #!/bin/sh -l
 
-echo "$1"
-eval $1
+echo "::debug::\$cmd: $1"
+RESULT=$(eval "$1")
+echo "::debug::\$RESULT: $RESULT"
+echo ::set-output name=result::"$RESULT"

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/spf13/cobra v1.1.3
 	github.com/timtadh/data-structures v0.5.3 // indirect
 	github.com/timtadh/lexmachine v0.2.2
-	golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e // indirect
+	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )

go.sum

@@ -256,8 +256,8 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e h1:XNp2Flc/1eWQGk5BLzqTAN7fQIwIbfyVTuVxXxZh73M=
-golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=

pkg/yqlib/all_at_once_evaluator.go

@@ -8,7 +8,7 @@ import (
 
 // A yaml expression evaluator that runs the expression once against all files/nodes in memory.
 type Evaluator interface {
-	EvaluateFiles(expression string, filenames []string, printer Printer) error
+	EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error
 
 	// EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes
 	EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error)
@@ -46,15 +46,21 @@ func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCand
 	return context.MatchingNodes, nil
 }
 
-func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error {
+func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error {
 	fileIndex := 0
+	firstFileLeadingContent := ""
 
 	var allDocuments *list.List = list.New()
 	for _, filename := range filenames {
-		reader, err := readStream(filename)
+		reader, leadingContent, err := readStream(filename, leadingContentPreProcessing)
 		if err != nil {
 			return err
 		}
+
+		if fileIndex == 0 {
+			firstFileLeadingContent = leadingContent
+		}
+
 		fileDocuments, err := readDocuments(reader, filename, fileIndex)
 		if err != nil {
 			return err
@@ -62,6 +68,19 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
 		allDocuments.PushBackList(fileDocuments)
 		fileIndex = fileIndex + 1
 	}
+
+	if allDocuments.Len() == 0 {
+		candidateNode := &CandidateNode{
+			Document:  0,
+			Filename:  "",
+			Node:      &yaml.Node{Kind: yaml.DocumentNode, HeadComment: firstFileLeadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
+			FileIndex: 0,
+		}
+		allDocuments.PushBack(candidateNode)
+	} else {
+		allDocuments.Front().Value.(*CandidateNode).Node.HeadComment = firstFileLeadingContent
+	}
+
 	matches, err := e.EvaluateCandidateNodes(expression, allDocuments)
 	if err != nil {
 		return err

pkg/yqlib/context.go

@@ -12,6 +12,14 @@ type Context struct {
 	DontAutoCreate bool
 }
 
+func (n *Context) SingleReadonlyChildContext(candidate *CandidateNode) Context {
+	list := list.New()
+	list.PushBack(candidate)
+	newContext := n.ChildContext(list)
+	newContext.DontAutoCreate = true
+	return newContext
+}
+
 func (n *Context) SingleChildContext(candidate *CandidateNode) Context {
 	list := list.New()
 	list.PushBack(candidate)
@@ -52,3 +60,15 @@ func (n *Context) Clone() Context {
 	}
 	return clone
 }
+
+func (n *Context) ReadOnlyClone() Context {
+	clone := n.Clone()
+	clone.DontAutoCreate = true
+	return clone
+}
+
+func (n *Context) WritableClone() Context {
+	clone := n.Clone()
+	clone.DontAutoCreate = false
+	return clone
+}

pkg/yqlib/doc/Add.md

@@ -127,6 +127,32 @@ b:
   - 4
 ```
 
+## Append another array using +=
+Given a sample.yml file of:
+```yaml
+a:
+  - 1
+  - 2
+b:
+  - 3
+  - 4
+```
+then
+```bash
+yq eval '.a += .b' sample.yml
+```
+will output
+```yaml
+a:
+  - 1
+  - 2
+  - 3
+  - 4
+b:
+  - 3
+  - 4
+```
+
 ## Relative append
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Anchor and Alias Operators.md

@@ -3,6 +3,97 @@ Use the `alias` and `anchor` operators to read and write yaml aliases and anchor
 `yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.
 
 
+## Merge one map
+see https://yaml.org/type/merge.html
+
+Given a sample.yml file of:
+```yaml
+- &CENTER
+  x: 1
+  y: 2
+- &LEFT
+  x: 0
+  y: 2
+- &BIG
+  r: 10
+- &SMALL
+  r: 1
+- !!merge <<: *CENTER
+  r: 10
+```
+then
+```bash
+yq eval '.[4] | explode(.)' sample.yml
+```
+will output
+```yaml
+x: 1
+y: 2
+r: 10
+```
+
+## Merge multiple maps
+see https://yaml.org/type/merge.html
+
+Given a sample.yml file of:
+```yaml
+- &CENTER
+  x: 1
+  y: 2
+- &LEFT
+  x: 0
+  y: 2
+- &BIG
+  r: 10
+- &SMALL
+  r: 1
+- !!merge <<:
+    - *CENTER
+    - *BIG
+```
+then
+```bash
+yq eval '.[4] | explode(.)' sample.yml
+```
+will output
+```yaml
+r: 10
+x: 1
+y: 2
+```
+
+## Override
+see https://yaml.org/type/merge.html
+
+Given a sample.yml file of:
+```yaml
+- &CENTER
+  x: 1
+  y: 2
+- &LEFT
+  x: 0
+  y: 2
+- &BIG
+  r: 10
+- &SMALL
+  r: 1
+- !!merge <<:
+    - *BIG
+    - *LEFT
+    - *SMALL
+  x: 1
+```
+then
+```bash
+yq eval '.[4] | explode(.)' sample.yml
+```
+will output
+```yaml
+r: 10
+x: 1
+y: 2
+```
+
 ## Get anchor
 Given a sample.yml file of:
 ```yaml
@@ -78,6 +169,22 @@ b: &meow purr
 a: *meow
 ```
 
+## Set alias to blank does nothing
+Given a sample.yml file of:
+```yaml
+b: &meow purr
+a: cat
+```
+then
+```bash
+yq eval '.a alias = ""' sample.yml
+```
+will output
+```yaml
+b: &meow purr
+a: cat
+```
+
 ## Set alias relatively using assign-update
 Given a sample.yml file of:
 ```yaml
@@ -183,12 +290,42 @@ bar:
   c: bar_c
 foobarList:
   b: bar_b
-  a: foo_a
-  thing: bar_thing
+  thing: foo_thing
   c: foobarList_c
+  a: foo_a
 foobar:
   c: foo_c
   a: foo_a
   thing: foobar_thing
 ```
 
+## Dereference and update a field
+`Use explode with multiply to dereference an object
+
+Given a sample.yml file of:
+```yaml
+item_value: &item_value
+  value: true
+thingOne:
+  name: item_1
+  !!merge <<: *item_value
+thingTwo:
+  name: item_2
+  !!merge <<: *item_value
+```
+then
+```bash
+yq eval '.thingOne |= explode(.) * {"value": false}' sample.yml
+```
+will output
+```yaml
+item_value: &item_value
+  value: true
+thingOne:
+  name: item_1
+  value: false
+thingTwo:
+  name: item_2
+  !!merge <<: *item_value
+```
+

pkg/yqlib/doc/Assign (Update).md

@@ -28,6 +28,24 @@ a:
   g: foof
 ```
 
+## Double elements in an array
+Given a sample.yml file of:
+```yaml
+- 1
+- 2
+- 3
+```
+then
+```bash
+yq eval '.[] |= . * 2' sample.yml
+```
+will output
+```yaml
+- 2
+- 4
+- 6
+```
+
 ## Update node from another file
 Note this will also work when the second file is a scalar (string/number)
 
@@ -75,7 +93,7 @@ c: fieldC
 ```
 then
 ```bash
-yq eval '(.a, .c) |= "potatoe"' sample.yml
+yq eval '(.a, .c) = "potatoe"' sample.yml
 ```
 will output
 ```yaml

pkg/yqlib/doc/Boolean Operators.md

@@ -1,5 +1,14 @@
-The `or` and `and` operators take two parameters and return a boolean result. `not` flips a boolean from true to false, or vice versa. These are most commonly used with the `select` operator to filter particular nodes.
-## OR example
+The `or` and `and` operators take two parameters and return a boolean result. 
+
+`not` flips a boolean from true to false, or vice versa. 
+
+`any` will return `true` if there are any `true` values in a array sequence, and `all` will return true if _all_ elements in an array are true.
+
+`any_c(condition)` and `all_c(condition)` are like `any` and `all` but they take a condition expression that is used against each element to determine if it's `true`. Note: in `jq` you can simply pass a condition to `any` or `all` and it simply works - `yq` isn't that clever..yet
+
+These are most commonly used with the `select` operator to filter particular nodes.
+
+## `or` example
 Running
 ```bash
 yq eval --null-input 'true or false'
@@ -9,7 +18,7 @@ will output
 true
 ```
 
-## AND example
+## `and` example
 Running
 ```bash
 yq eval --null-input 'true and false'
@@ -41,6 +50,104 @@ will output
   b: fly
 ```
 
+## `any` returns true if any boolean in a given array is true
+Given a sample.yml file of:
+```yaml
+- false
+- true
+```
+then
+```bash
+yq eval 'any' sample.yml
+```
+will output
+```yaml
+true
+```
+
+## `any` returns false for an empty array
+Given a sample.yml file of:
+```yaml
+[]
+```
+then
+```bash
+yq eval 'any' sample.yml
+```
+will output
+```yaml
+false
+```
+
+## `any_c` returns true if any element in the array is true for the given condition.
+Given a sample.yml file of:
+```yaml
+a:
+  - rad
+  - awesome
+b:
+  - meh
+  - whatever
+```
+then
+```bash
+yq eval '.[] |= any_c(. == "awesome")' sample.yml
+```
+will output
+```yaml
+a: true
+b: false
+```
+
+## `all` returns true if all booleans in a given array are true
+Given a sample.yml file of:
+```yaml
+- true
+- true
+```
+then
+```bash
+yq eval 'all' sample.yml
+```
+will output
+```yaml
+true
+```
+
+## `all` returns true for an empty array
+Given a sample.yml file of:
+```yaml
+[]
+```
+then
+```bash
+yq eval 'all' sample.yml
+```
+will output
+```yaml
+true
+```
+
+## `all_c` returns true if all elements in the array are true for the given condition.
+Given a sample.yml file of:
+```yaml
+a:
+  - rad
+  - awesome
+b:
+  - meh
+  - 12
+```
+then
+```bash
+yq eval '.[] |= all_c(tag == "!!str")' sample.yml
+```
+will output
+```yaml
+a: true
+b: false
+```
+
 ## Not true is false
 Running
 ```bash

pkg/yqlib/doc/Comment Operators.md

@@ -88,7 +88,7 @@ a: cat
 b: dog # leave this
 ```
 
-## Remove all comments
+## Remove (strip) all comments
 Note the use of `...` to ensure key nodes are included.
 
 Given a sample.yml file of:

pkg/yqlib/doc/Entries.md

@@ -0,0 +1,118 @@
+Similar to the same named functions in `jq` these functions convert to/from an object and an array of key-value pairs. This is most useful for performing operations on keys of maps.
+## to_entries Map
+Given a sample.yml file of:
+```yaml
+a: 1
+b: 2
+```
+then
+```bash
+yq eval 'to_entries' sample.yml
+```
+will output
+```yaml
+- key: a
+  value: 1
+- key: b
+  value: 2
+```
+
+## to_entries Array
+Given a sample.yml file of:
+```yaml
+- a
+- b
+```
+then
+```bash
+yq eval 'to_entries' sample.yml
+```
+will output
+```yaml
+- key: 0
+  value: a
+- key: 1
+  value: b
+```
+
+## to_entries null
+Given a sample.yml file of:
+```yaml
+null
+```
+then
+```bash
+yq eval 'to_entries' sample.yml
+```
+will output
+```yaml
+```
+
+## from_entries map
+Given a sample.yml file of:
+```yaml
+a: 1
+b: 2
+```
+then
+```bash
+yq eval 'to_entries | from_entries' sample.yml
+```
+will output
+```yaml
+a: 1
+b: 2
+```
+
+## from_entries with numeric key indexes
+from_entries always creates a map, even for numeric keys
+
+Given a sample.yml file of:
+```yaml
+- a
+- b
+```
+then
+```bash
+yq eval 'to_entries | from_entries' sample.yml
+```
+will output
+```yaml
+0: a
+1: b
+```
+
+## Use with_entries to update keys
+Given a sample.yml file of:
+```yaml
+a: 1
+b: 2
+```
+then
+```bash
+yq eval 'with_entries(.key |= "KEY_" + .)' sample.yml
+```
+will output
+```yaml
+KEY_a: 1
+KEY_b: 2
+```
+
+## Use with_entries to filter the map
+Given a sample.yml file of:
+```yaml
+a:
+  b: bird
+c:
+  d: dog
+```
+then
+```bash
+yq eval 'with_entries(select(.value | has("b")))' sample.yml
+```
+will output
+```yaml
+a:
+  b: bird
+```
+

pkg/yqlib/doc/Equals.md

@@ -93,3 +93,31 @@ will output
 true
 ```
 
+## Non exisitant key doesn't equal a value
+Given a sample.yml file of:
+```yaml
+a: frog
+```
+then
+```bash
+yq eval 'select(.b != "thing")' sample.yml
+```
+will output
+```yaml
+a: frog
+```
+
+## Two non existant keys are equal
+Given a sample.yml file of:
+```yaml
+a: frog
+```
+then
+```bash
+yq eval 'select(.b == .c)' sample.yml
+```
+will output
+```yaml
+a: frog
+```
+

pkg/yqlib/doc/Multiply (Merge).md

@@ -231,6 +231,46 @@ will output
   age: 32
 ```
 
+## Merge arrays of objects together, matching on a key
+It's a complex command, the trickyness comes from needing to have the right context in the expressions.
+First we save the second array into a variable '$two' which lets us reference it later.
+We then need to update the first array. We will use the relative update (|=) because we need to update relative to the current element of the array in the LHS in the RHS expression. 
+We set the current element of the first array as $cur. Now we multiply (merge) $cur with the matching entry in $two, by passing $two through a select filter.
+
+
+Given a sample.yml file of:
+```yaml
+- a: apple
+  b: appleB
+- a: kiwi
+  b: kiwiB
+- a: banana
+  b: bananaB
+```
+And another sample another.yml file of:
+```yaml
+- a: banana
+  c: bananaC
+- a: apple
+  b: appleB2
+- a: dingo
+  c: dingoC
+```
+then
+```bash
+yq eval-all '(select(fi==1) | .[]) as $two | select(fi==0) | .[] |= (. as $cur |  $cur * ($two | select(.a == $cur.a)))' sample.yml another.yml
+```
+will output
+```yaml
+- a: apple
+  b: appleB2
+- a: kiwi
+  b: kiwiB
+- a: banana
+  b: bananaB
+  c: bananaC
+```
+
 ## Merge to prefix an element
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/String Operators.md

@@ -1,5 +1,8 @@
 # String Operators
 
+## RegEx
+This uses golangs native regex functions under the hood - See https://github.com/google/re2/wiki/Syntax for the supported syntax.
+
 ## Join strings
 Given a sample.yml file of:
 ```yaml
@@ -18,6 +21,171 @@ will output
 cat; meow; 1; ; true
 ```
 
+## Match string
+Given a sample.yml file of:
+```yaml
+foo bar foo
+```
+then
+```bash
+yq eval 'match("foo")' sample.yml
+```
+will output
+```yaml
+string: foo
+offset: 0
+length: 3
+captures: []
+```
+
+## Match string, case insensitive
+Given a sample.yml file of:
+```yaml
+foo bar FOO
+```
+then
+```bash
+yq eval '[match("(?i)foo"; "g")]' sample.yml
+```
+will output
+```yaml
+- string: foo
+  offset: 0
+  length: 3
+  captures: []
+- string: FOO
+  offset: 8
+  length: 3
+  captures: []
+```
+
+## Match with capture groups
+Given a sample.yml file of:
+```yaml
+abc abc
+```
+then
+```bash
+yq eval '[match("(abc)+"; "g")]' sample.yml
+```
+will output
+```yaml
+- string: abc
+  offset: 0
+  length: 3
+  captures:
+    - string: abc
+      offset: 0
+      length: 3
+- string: abc
+  offset: 4
+  length: 3
+  captures:
+    - string: abc
+      offset: 4
+      length: 3
+```
+
+## Match with named capture groups
+Given a sample.yml file of:
+```yaml
+foo bar foo foo  foo
+```
+then
+```bash
+yq eval '[match("foo (?P<bar123>bar)? foo"; "g")]' sample.yml
+```
+will output
+```yaml
+- string: foo bar foo
+  offset: 0
+  length: 11
+  captures:
+    - string: bar
+      offset: 4
+      length: 3
+      name: bar123
+- string: foo  foo
+  offset: 12
+  length: 8
+  captures:
+    - string: null
+      offset: -1
+      length: 0
+      name: bar123
+```
+
+## Capture named groups into a map
+Given a sample.yml file of:
+```yaml
+xyzzy-14
+```
+then
+```bash
+yq eval 'capture("(?P<a>[a-z]+)-(?P<n>[0-9]+)")' sample.yml
+```
+will output
+```yaml
+a: xyzzy
+n: "14"
+```
+
+## Match without global flag
+Given a sample.yml file of:
+```yaml
+cat cat
+```
+then
+```bash
+yq eval 'match("cat")' sample.yml
+```
+will output
+```yaml
+string: cat
+offset: 0
+length: 3
+captures: []
+```
+
+## Match with global flag
+Given a sample.yml file of:
+```yaml
+cat cat
+```
+then
+```bash
+yq eval '[match("cat"; "g")]' sample.yml
+```
+will output
+```yaml
+- string: cat
+  offset: 0
+  length: 3
+  captures: []
+- string: cat
+  offset: 4
+  length: 3
+  captures: []
+```
+
+## Test using regex
+Like jq'q equivalant, this works like match but only returns true/false instead of full match details
+
+Given a sample.yml file of:
+```yaml
+- cat
+- dog
+```
+then
+```bash
+yq eval '.[] | test("at")' sample.yml
+```
+will output
+```yaml
+true
+false
+```
+
 ## Substitute / Replace string
 This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)
 Note the use of `|=` to run in context of the current string value.

pkg/yqlib/doc/Style.md

@@ -1,4 +1,42 @@
 The style operator can be used to get or set the style of nodes (e.g. string style, yaml style)
+## Update and set style of a particular node (simple)
+Given a sample.yml file of:
+```yaml
+a:
+  b: thing
+  c: something
+```
+then
+```bash
+yq eval '.a.b = "new" | .a.b style="double"' sample.yml
+```
+will output
+```yaml
+a:
+  b: "new"
+  c: something
+```
+
+## Update and set style of a particular node using path variables
+You can use a variable to re-use a path
+
+Given a sample.yml file of:
+```yaml
+a:
+  b: thing
+  c: something
+```
+then
+```bash
+yq eval '.a.b as $x | $x = "new" | $x style="double"' sample.yml
+```
+will output
+```yaml
+a:
+  b: "new"
+  c: something
+```
+
 ## Set tagged style
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Traverse (Read).md

@@ -1,4 +1,4 @@
-This is the simplest (and perhaps most used) operator, it is used to navigate deeply into yaml structurse.
+This is the simplest (and perhaps most used) operator, it is used to navigate deeply into yaml structures.
 ## Simple map navigation
 Given a sample.yml file of:
 ```yaml
@@ -32,6 +32,21 @@ b: apple
 c: banana
 ```
 
+## Optional Splat
+Just like splat, but won't error if you run it against scalars
+
+Given a sample.yml file of:
+```yaml
+cat
+```
+then
+```bash
+yq eval '.[]' sample.yml
+```
+will output
+```yaml
+```
+
 ## Special characters
 Use quotes with brackets around path elements with special characters
 
@@ -48,6 +63,22 @@ will output
 frog
 ```
 
+## Nested special characters
+Given a sample.yml file of:
+```yaml
+a:
+  "key.withdots":
+    "another.key": apple
+```
+then
+```bash
+yq eval '.a["key.withdots"]["another.key"]' sample.yml
+```
+will output
+```yaml
+apple
+```
+
 ## Keys with spaces
 Use quotes with brackets around path elements with special characters
 
@@ -98,6 +129,23 @@ will output
 null
 ```
 
+## Optional identifier
+Like jq, does not output an error when the yaml is not an array or object as expected
+
+Given a sample.yml file of:
+```yaml
+- 1
+- 2
+- 3
+```
+then
+```bash
+yq eval '.a?' sample.yml
+```
+will output
+```yaml
+```
+
 ## Wildcard matching
 Given a sample.yml file of:
 ```yaml
@@ -179,6 +227,20 @@ will output
 1
 ```
 
+## Traversing nested arrays by index
+Given a sample.yml file of:
+```yaml
+[[], [cat]]
+```
+then
+```bash
+yq eval '.[1][0]' sample.yml
+```
+will output
+```yaml
+cat
+```
+
 ## Maps with numeric keys
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Unique.md

@@ -0,0 +1,82 @@
+This is used to filter out duplicated items in an array.
+
+## Unique array of scalars (string/numbers)
+Given a sample.yml file of:
+```yaml
+- 1
+- 2
+- 3
+- 2
+```
+then
+```bash
+yq eval 'unique' sample.yml
+```
+will output
+```yaml
+- 1
+- 2
+- 3
+```
+
+## Unique nulls
+Unique works on the node value, so it considers different representations of nulls to be different
+
+Given a sample.yml file of:
+```yaml
+- ~
+- null
+- ~
+- null
+```
+then
+```bash
+yq eval 'unique' sample.yml
+```
+will output
+```yaml
+- ~
+- null
+```
+
+## Unique all nulls
+Run against the node tag to unique all the nulls
+
+Given a sample.yml file of:
+```yaml
+- ~
+- null
+- ~
+- null
+```
+then
+```bash
+yq eval 'unique_by(tag)' sample.yml
+```
+will output
+```yaml
+- ~
+```
+
+## Unique array object fields
+Given a sample.yml file of:
+```yaml
+- name: harry
+  pet: cat
+- name: billy
+  pet: dog
+- name: harry
+  pet: dog
+```
+then
+```bash
+yq eval 'unique_by(.name)' sample.yml
+```
+will output
+```yaml
+- name: harry
+  pet: cat
+- name: billy
+  pet: dog
+```
+

pkg/yqlib/doc/aa.md

@@ -4,7 +4,7 @@ In `yq` expressions are made up of operators and pipes. A context of nodes is pa
 
 Lets look at a couple of examples.
 
-## Example 1 - simple example
+## Example with a simple operator
 
 Given a document like:
 
@@ -37,10 +37,80 @@ This being the last operation in the expression, the results will be printed out
 3
 ```
 
-# Example 2 - operators with arguments.
+# Example with an operator that takes arguments.
+
+Given a document like:
+
+```yaml
+a: cat
+b: dog
+```
+
+with an expression:
+
+```
+.a = .b
+```
+
+The `=` operator takes two arguments, a `lhs` expression, which in this case is `.a` and `rhs` expression which is `.b`. 
+
+It pipes the current, lets call it 'root' context through the `lhs` expression of `.a` to return the node 
+
+```yaml
+cat
+```
+
+Note that this node holds not only its value 'cat', but comments and metadata too, including path and parent information.
+
+The `=` operator then pipes the 'root' context through the `rhs` expression of `.b` to return the node
+
+```yaml
+dog
+```
+
+Both sides have now been evaluated, so now the operator copies across the value from the RHS to the value on the LHS, and it returns the now updated context:
+
+```yaml
+a: dog
+b: dog
+```
+
+# Relative update (e.g. `|=`)
+There is another form of the `=` operator which we call the relative form. It's very similar to `=` but with one key difference when evaluating the RHS expression.
+
+In the plain form, we pass in the 'root' level context to the RHS expression. In relative form, we pass in _each result of the LHS_ to the RHS expression. Let's go through an example.
+
+Given a document like:
+
+```yaml
+a: 1
+b: thing
+```
+
+with an expression:
+
+```
+.a |= . + 1
+```
+
+Similar to the `=` operator, `|=` takes two operands, the LHS and RHS.
+
+It pipes the current context (the whole document) through the LHS expression  of `.a` to get the node value:
+
+```
+1
+```
+
+Now it pipes _that LHS context_ into the RHS expression `. + 1` (whereas in the `=` plain form it piped the original document context into the RHS) to yield:
 
 
-The `=` operator takes two arguments, a `lhs` expression and `rhs` expression. It runs the 'matching' nodes context against the `lhs` expression to find the nodes to update, lets call it `lhsNodes`, and then runs the matching nodes against the `rhs` to find the new values, lets call that `rhsNodes`. It updates the `lhsNodes` values with the `rhsNodes` values and _returns the original matching nodes_. This is important, where length changed the matching nodes to be new nodes with the length values, `=` returns the original matching nodes, albeit with some of the nodes values updated. So `.a = 3` will still return the parent matching node, but with the matching child updated.
+```
+2
+```
 
-Please see the individual operator docs for more information and examples.
+The assignment operator then copies across the value from the RHS to the value on the LHS, and it returns the now updated 'root' context:
 
+```yaml
+a: 2
+b: thing
+```

pkg/yqlib/doc/headers/Boolean Operators.md

@@ -1 +1,9 @@
-The `or` and `and` operators take two parameters and return a boolean result. `not` flips a boolean from true to false, or vice versa. These are most commonly used with the `select` operator to filter particular nodes.
+The `or` and `and` operators take two parameters and return a boolean result. 
+
+`not` flips a boolean from true to false, or vice versa. 
+
+`any` will return `true` if there are any `true` values in a array sequence, and `all` will return true if _all_ elements in an array are true.
+
+`any_c(condition)` and `all_c(condition)` are like `any` and `all` but they take a condition expression that is used against each element to determine if it's `true`. Note: in `jq` you can simply pass a condition to `any` or `all` and it simply works - `yq` isn't that clever..yet
+
+These are most commonly used with the `select` operator to filter particular nodes.

pkg/yqlib/doc/headers/Entries.md

@@ -0,0 +1 @@
+Similar to the same named functions in `jq` these functions convert to/from an object and an array of key-value pairs. This is most useful for performing operations on keys of maps.

pkg/yqlib/doc/headers/String Operators.md

@@ -1 +1,4 @@
 # String Operators
+
+## RegEx
+This uses golangs native regex functions under the hood - See https://github.com/google/re2/wiki/Syntax for the supported syntax.

pkg/yqlib/doc/headers/Traverse (Read).md

@@ -1 +1 @@
-This is the simplest (and perhaps most used) operator, it is used to navigate deeply into yaml structurse.
+This is the simplest (and perhaps most used) operator, it is used to navigate deeply into yaml structures.

pkg/yqlib/doc/headers/Unique.md

@@ -0,0 +1 @@
+This is used to filter out duplicated items in an array.

pkg/yqlib/expression_postfix.go

@@ -54,12 +54,26 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
 			opStack = opStack[0 : len(opStack)-1]
 			log.Debugf("deleteing open bracket from opstack")
 
-			//and append a collect to the opStack
-			result = append(result, &Operation{OperationType: collectOperator})
+			//and append a collect to the result
+			// hack - see if there's the optional traverse flag
+			// on the close op - move it to the collect op.
+			// allows for .["cat"]?
+			prefs := traversePreferences{}
+			closeTokenMatch := string(currentToken.Match.Bytes)
+			if closeTokenMatch[len(closeTokenMatch)-1:] == "?" {
+				prefs.OptionalTraverse = true
+			}
+			result = append(result, &Operation{OperationType: collectOperator, Preferences: prefs})
 			log.Debugf("put collect onto the result")
 			result = append(result, &Operation{OperationType: shortPipeOpType})
 			log.Debugf("put shortpipe onto the result")
 
+			//traverseArrayCollect is a sneaky op that needs to be included too
+			//when closing a []
+			if len(opStack) > 0 && opStack[len(opStack)-1].Operation != nil && opStack[len(opStack)-1].Operation.OperationType == traverseArrayOpType {
+				opStack, result = popOpToResult(opStack, result)
+			}
+
 		case closeBracket:
 			for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != openBracket {
 				opStack, result = popOpToResult(opStack, result)

pkg/yqlib/expression_processing_test.go

@@ -7,11 +7,39 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
+var variableWithNewLine = `"cat
+"`
+
 var pathTests = []struct {
 	path            string
 	expectedTokens  []interface{}
 	expectedPostFix []interface{}
 }{
+	{
+		".a\n",
+		append(make([]interface{}, 0), "a"),
+		append(make([]interface{}, 0), "a"),
+	},
+	{
+		variableWithNewLine,
+		append(make([]interface{}, 0), "cat\n (string)"),
+		append(make([]interface{}, 0), "cat\n (string)"),
+	},
+	{
+		`.[0]`,
+		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
+		append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
+	{
+		`.[0][1]`,
+		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]", "TRAVERSE_ARRAY", "[", "1 (int64)", "]"),
+		append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "1 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
+	{
+		`"\""`,
+		append(make([]interface{}, 0), "\" (string)"),
+		append(make([]interface{}, 0), "\" (string)"),
+	},
 	{
 		`[]|join(".")`,
 		append(make([]interface{}, 0), "[", "EMPTY", "]", "PIPE", "JOIN", "(", ". (string)", ")"),
@@ -62,6 +90,11 @@ var pathTests = []struct {
 		append(make([]interface{}, 0), "b", "TRAVERSE_ARRAY", "[", "a", "]"),
 		append(make([]interface{}, 0), "b", "a", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
 	},
+	{
+		`.b[.a]?`,
+		append(make([]interface{}, 0), "b", "TRAVERSE_ARRAY", "[", "a", "]"),
+		append(make([]interface{}, 0), "b", "a", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
 	{
 		`.[]`,
 		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
@@ -72,6 +105,11 @@ var pathTests = []struct {
 		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
 		append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
 	},
+	{
+		`.a[]?`,
+		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
+		append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
 	{
 		`.a.[]`,
 		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
@@ -82,6 +120,11 @@ var pathTests = []struct {
 		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
 		append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
 	},
+	{
+		`.a[0]?`,
+		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
+		append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
 	{
 		`.a.[0]`,
 		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),

pkg/yqlib/expression_tokeniser.go

@@ -29,8 +29,9 @@ const (
 type token struct {
 	TokenType            tokenType
 	Operation            *Operation
-	AssignOperation      *Operation // e.g. tag (GetTag) op becomes AssignTag if '=' follows it
-	CheckForPostTraverse bool       // e.g. [1]cat should really be [1].cat
+	AssignOperation      *Operation      // e.g. tag (GetTag) op becomes AssignTag if '=' follows it
+	CheckForPostTraverse bool            // e.g. [1]cat should really be [1].cat
+	Match                *machines.Match // match that created this token
 
 }
 
@@ -63,12 +64,19 @@ func (t *token) toString(detail bool) string {
 func pathToken(wrapped bool) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		value := string(m.Bytes)
+		prefs := traversePreferences{}
+
+		if value[len(value)-1:] == "?" {
+			prefs.OptionalTraverse = true
+			value = value[:len(value)-1]
+		}
+
 		value = value[1:]
 		if wrapped {
 			value = unwrap(value)
 		}
 		log.Debug("PathToken %v", value)
-		op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: traversePreferences{}}
+		op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: prefs}
 		return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
 	}
 }
@@ -138,7 +146,7 @@ func assignAllCommentsOp(updateAssign bool) lex.Action {
 
 func literalToken(pType tokenType, checkForPost bool) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &token{TokenType: pType, CheckForPostTraverse: checkForPost}, nil
+		return &token{TokenType: pType, CheckForPostTraverse: checkForPost, Match: m}, nil
 	}
 }
 
@@ -181,6 +189,7 @@ func stringValue(wrapped bool) lex.Action {
 		if wrapped {
 			value = unwrap(value)
 		}
+		value = strings.ReplaceAll(value, "\\\"", "\"")
 		return &token{TokenType: operationToken, Operation: createValueOperation(value, value)}, nil
 	}
 }
@@ -251,6 +260,8 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`sortKeys`), opToken(sortKeysOpType))
 	lexer.Add([]byte(`select`), opToken(selectOpType))
 	lexer.Add([]byte(`has`), opToken(hasOpType))
+	lexer.Add([]byte(`unique`), opToken(uniqueOpType))
+	lexer.Add([]byte(`unique_by`), opToken(uniqueByOpType))
 	lexer.Add([]byte(`explode`), opToken(explodeOpType))
 	lexer.Add([]byte(`or`), opToken(orOpType))
 	lexer.Add([]byte(`and`), opToken(andOpType))
@@ -265,6 +276,14 @@ func initLexer() (*lex.Lexer, error) {
 
 	lexer.Add([]byte(`join`), opToken(joinStringOpType))
 	lexer.Add([]byte(`sub`), opToken(subStringOpType))
+	lexer.Add([]byte(`match`), opToken(matchOpType))
+	lexer.Add([]byte(`capture`), opToken(captureOpType))
+	lexer.Add([]byte(`test`), opToken(testOpType))
+
+	lexer.Add([]byte(`any`), opToken(anyOpType))
+	lexer.Add([]byte(`any_c`), opToken(anyConditionOpType))
+	lexer.Add([]byte(`all`), opToken(allOpType))
+	lexer.Add([]byte(`all_c`), opToken(allConditionOpType))
 
 	lexer.Add([]byte(`split`), opToken(splitStringOpType))
 	lexer.Add([]byte(`keys`), opToken(keysOpType))
@@ -278,6 +297,9 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`fileIndex`), opToken(getFileIndexOpType))
 	lexer.Add([]byte(`fi`), opToken(getFileIndexOpType))
 	lexer.Add([]byte(`path`), opToken(getPathOpType))
+	lexer.Add([]byte(`to_entries`), opToken(toEntriesOpType))
+	lexer.Add([]byte(`from_entries`), opToken(fromEntriesOpType))
+	lexer.Add([]byte(`with_entries`), opToken(withEntriesOpType))
 
 	lexer.Add([]byte(`lineComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}))
 
@@ -300,8 +322,8 @@ func initLexer() (*lex.Lexer, error) {
 
 	lexer.Add([]byte("( |\t|\n|\r)+"), skip)
 
-	lexer.Add([]byte(`\."[^ "]+"`), pathToken(true))
-	lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=]+`), pathToken(false))
+	lexer.Add([]byte(`\."[^ "]+"\??`), pathToken(true))
+	lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=\n]+\??`), pathToken(false))
 	lexer.Add([]byte(`\.`), selfToken())
 
 	lexer.Add([]byte(`\|`), opToken(pipeOpType))
@@ -316,12 +338,12 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`[Nn][Uu][Ll][Ll]`), nullValue())
 	lexer.Add([]byte(`~`), nullValue())
 
-	lexer.Add([]byte(`"[^"]*"`), stringValue(true))
+	lexer.Add([]byte(`"([^"\\]*(\\.[^"\\]*)*)"`), stringValue(true))
 	lexer.Add([]byte(`strenv\([^\)]+\)`), envOp(true))
 	lexer.Add([]byte(`env\([^\)]+\)`), envOp(false))
 
 	lexer.Add([]byte(`\[`), literalToken(openCollect, false))
-	lexer.Add([]byte(`\]`), literalToken(closeCollect, true))
+	lexer.Add([]byte(`\]\??`), literalToken(closeCollect, true))
 	lexer.Add([]byte(`\{`), literalToken(openCollectObject, false))
 	lexer.Add([]byte(`\}`), literalToken(closeCollectObject, true))
 	lexer.Add([]byte(`\*[\+|\?d]*`), multiplyWithPrefs())
@@ -392,16 +414,19 @@ func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postPr
 	skipNextToken = false
 	currentToken := tokens[index]
 
+	log.Debug("processing %v", currentToken.toString(true))
+
 	if currentToken.TokenType == traverseArrayCollect {
 		//need to put a traverse array then a collect currentToken
 		// do this by adding traverse then converting currentToken to collect
 
 		if index == 0 || tokens[index-1].TokenType != operationToken ||
 			tokens[index-1].Operation.OperationType != traversePathOpType {
+			log.Debug("  adding self")
 			op := &Operation{OperationType: selfReferenceOpType, StringValue: "SELF"}
 			postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 		}
-
+		log.Debug("  adding traverse array")
 		op := &Operation{OperationType: traverseArrayOpType, StringValue: "TRAVERSE_ARRAY"}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 
@@ -412,17 +437,19 @@ func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postPr
 	if index != len(tokens)-1 && currentToken.AssignOperation != nil &&
 		tokens[index+1].TokenType == operationToken &&
 		tokens[index+1].Operation.OperationType == assignOpType {
+		log.Debug("  its an update assign")
 		currentToken.Operation = currentToken.AssignOperation
 		currentToken.Operation.UpdateAssign = tokens[index+1].Operation.UpdateAssign
 		skipNextToken = true
 	}
 
+	log.Debug("  adding token to the fixed list")
 	postProcessedTokens = append(postProcessedTokens, currentToken)
 
 	if index != len(tokens)-1 &&
 		((currentToken.TokenType == openCollect && tokens[index+1].TokenType == closeCollect) ||
 			(currentToken.TokenType == openCollectObject && tokens[index+1].TokenType == closeCollectObject)) {
-
+		log.Debug("  adding empty")
 		op := &Operation{OperationType: emptyOpType, StringValue: "EMPTY"}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 	}
@@ -430,12 +457,19 @@ func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postPr
 	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
 		tokens[index+1].TokenType == operationToken &&
 		tokens[index+1].Operation.OperationType == traversePathOpType {
+		log.Debug("  adding pipe because the next thing is traverse")
 		op := &Operation{OperationType: shortPipeOpType, Value: "PIPE"}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 	}
 	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
 		tokens[index+1].TokenType == openCollect {
 
+		// if tokens[index].TokenType == closeCollect {
+		// 	log.Debug("  adding pipe because next is opencollect")
+		// 	op := &Operation{OperationType: shortPipeOpType, Value: "PIPE"}
+		// 	postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
+		// }
+		log.Debug("  adding traverArray because next is opencollect")
 		op := &Operation{OperationType: traverseArrayOpType}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 	}

pkg/yqlib/file_utils.go

@@ -2,6 +2,7 @@ package yqlib
 
 import (
 	"io"
+	"io/ioutil"
 	"os"
 )
 
@@ -23,9 +24,20 @@ func safelyRenameFile(from string, to string) {
 	}
 }
 
+func tryRemoveFile(filename string) {
+	log.Debug("Removing temp file: %v", filename)
+	removeErr := os.Remove(filename)
+	if removeErr != nil {
+		log.Errorf("Failed to remove temp file: %v", filename)
+	}
+}
+
 // thanks https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang
 func copyFileContents(src, dst string) (err error) {
-	in, err := os.Open(src) // nolint gosec
+	// ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory,
+	// and ensuring that it's not possible to give a path to a file outside thar directory.
+
+	in, err := os.Open(src) // #nosec
 	if err != nil {
 		return err
 	}
@@ -41,6 +53,13 @@ func copyFileContents(src, dst string) (err error) {
 	return out.Sync()
 }
 
+func SafelyCloseReader(reader io.Reader) {
+	switch reader := reader.(type) {
+	case *os.File:
+		safelyCloseFile(reader)
+	}
+}
+
 func safelyCloseFile(file *os.File) {
 	err := file.Close()
 	if err != nil {
@@ -48,3 +67,22 @@ func safelyCloseFile(file *os.File) {
 		log.Error(err.Error())
 	}
 }
+
+func createTempFile() (*os.File, error) {
+	_, err := os.Stat(os.TempDir())
+	if os.IsNotExist(err) {
+		err = os.Mkdir(os.TempDir(), 0700)
+		if err != nil {
+			return nil, err
+		}
+	} else if err != nil {
+		return nil, err
+	}
+
+	file, err := ioutil.TempFile("", "temp")
+	if err != nil {
+		return nil, err
+	}
+
+	return file, err
+}

pkg/yqlib/front_matter.go

@@ -0,0 +1,96 @@
+package yqlib
+
+import (
+	"bufio"
+	"io"
+	"os"
+)
+
+type frontMatterHandler interface {
+	Split() error
+	GetYamlFrontMatterFilename() string
+	GetContentFilename() string
+	CleanUp()
+}
+
+type frontMatterHandlerImpl struct {
+	originalFilename        string
+	yamlFrontMatterFilename string
+	contentFilename         string
+}
+
+func NewFrontMatterHandler(originalFilename string) frontMatterHandler {
+	return &frontMatterHandlerImpl{originalFilename, "", ""}
+}
+
+func (f *frontMatterHandlerImpl) GetYamlFrontMatterFilename() string {
+	return f.yamlFrontMatterFilename
+}
+
+func (f *frontMatterHandlerImpl) GetContentFilename() string {
+	return f.contentFilename
+}
+
+func (f *frontMatterHandlerImpl) CleanUp() {
+	tryRemoveFile(f.yamlFrontMatterFilename)
+	tryRemoveFile(f.contentFilename)
+}
+
+// Splits the given file by yaml front matter
+// yaml content will be saved to first temporary file
+// remaining content will be saved to second temporary file
+func (f *frontMatterHandlerImpl) Split() error {
+	var reader io.Reader
+	var err error
+	if f.originalFilename == "-" {
+		reader = bufio.NewReader(os.Stdin)
+	} else {
+		reader, err = os.Open(f.originalFilename) // #nosec
+		if err != nil {
+			return err
+		}
+	}
+
+	yamlTempFile, err := createTempFile()
+	if err != nil {
+		return err
+	}
+	f.yamlFrontMatterFilename = yamlTempFile.Name()
+	log.Debug("yamlTempFile: %v", yamlTempFile.Name())
+
+	contentTempFile, err := createTempFile()
+	if err != nil {
+		return err
+	}
+	f.contentFilename = contentTempFile.Name()
+	log.Debug("contentTempFile: %v", contentTempFile.Name())
+
+	scanner := bufio.NewScanner(reader)
+
+	lineCount := 0
+	yamlContentBlock := true
+
+	for scanner.Scan() {
+		line := scanner.Text()
+
+		if lineCount > 0 && line == "---" {
+			//we've finished reading the yaml content
+			yamlContentBlock = false
+		}
+		if yamlContentBlock {
+			_, err = yamlTempFile.Write([]byte(line + "\n"))
+		} else {
+			_, err = contentTempFile.Write([]byte(line + "\n"))
+		}
+		if err != nil {
+			return err
+		}
+		lineCount = lineCount + 1
+	}
+
+	safelyCloseFile(yamlTempFile)
+	safelyCloseFile(contentTempFile)
+
+	return scanner.Err()
+
+}

pkg/yqlib/front_matter_test.go

@@ -0,0 +1,133 @@
+package yqlib
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+func createTestFile(content string) string {
+	tempFile, err := createTempFile()
+	if err != nil {
+		panic(err)
+	}
+
+	_, err = tempFile.Write([]byte(content))
+	if err != nil {
+		panic(err)
+	}
+
+	safelyCloseFile(tempFile)
+
+	return tempFile.Name()
+}
+
+func readFile(filename string) string {
+	bytes, err := ioutil.ReadFile(filename)
+	if err != nil {
+		panic(err)
+	}
+	return string(bytes)
+}
+
+func TestFrontMatterSplitWithLeadingSep(t *testing.T) {
+	file := createTestFile(`---
+a: apple
+b: banana
+---
+not a 
+yaml: doc
+`)
+
+	expectedYamlFm := `---
+a: apple
+b: banana
+`
+
+	expectedContent := `---
+not a 
+yaml: doc
+`
+
+	fmHandler := NewFrontMatterHandler(file)
+	err := fmHandler.Split()
+	if err != nil {
+		panic(err)
+	}
+
+	yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename())
+
+	test.AssertResult(t, expectedYamlFm, yamlFm)
+
+	content := readFile(fmHandler.GetContentFilename())
+	test.AssertResult(t, expectedContent, content)
+
+	tryRemoveFile(file)
+	fmHandler.CleanUp()
+}
+
+func TestFrontMatterSplitWithNoLeadingSep(t *testing.T) {
+	file := createTestFile(`a: apple
+b: banana
+---
+not a 
+yaml: doc
+`)
+
+	expectedYamlFm := `a: apple
+b: banana
+`
+
+	expectedContent := `---
+not a 
+yaml: doc
+`
+
+	fmHandler := NewFrontMatterHandler(file)
+	err := fmHandler.Split()
+	if err != nil {
+		panic(err)
+	}
+
+	yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename())
+
+	test.AssertResult(t, expectedYamlFm, yamlFm)
+
+	content := readFile(fmHandler.GetContentFilename())
+	test.AssertResult(t, expectedContent, content)
+
+	tryRemoveFile(file)
+	fmHandler.CleanUp()
+}
+
+func TestFrontMatterSplitWithArray(t *testing.T) {
+	file := createTestFile(`[1,2,3]
+---
+not a 
+yaml: doc
+`)
+
+	expectedYamlFm := "[1,2,3]\n"
+
+	expectedContent := `---
+not a 
+yaml: doc
+`
+
+	fmHandler := NewFrontMatterHandler(file)
+	err := fmHandler.Split()
+	if err != nil {
+		panic(err)
+	}
+
+	yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename())
+
+	test.AssertResult(t, expectedYamlFm, yamlFm)
+
+	content := readFile(fmHandler.GetContentFilename())
+	test.AssertResult(t, expectedContent, content)
+
+	tryRemoveFile(file)
+	fmHandler.CleanUp()
+}

pkg/yqlib/lib.go

@@ -20,9 +20,6 @@ type operationType struct {
 	Handler    operatorHandler
 }
 
-// operators TODO:
-// - mergeEmpty (sets only if the document is empty, do I do that now?)
-
 var orOpType = &operationType{Type: "OR", NumArgs: 2, Precedence: 20, Handler: orOperator}
 var andOpType = &operationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: andOperator}
 var reduceOpType = &operationType{Type: "REDUCE", NumArgs: 2, Precedence: 35, Handler: reduceOperator}
@@ -60,6 +57,16 @@ var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence:
 
 var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator}
 var collectOpType = &operationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: collectOperator}
+
+var anyOpType = &operationType{Type: "ANY", NumArgs: 0, Precedence: 50, Handler: anyOperator}
+var allOpType = &operationType{Type: "ALL", NumArgs: 0, Precedence: 50, Handler: allOperator}
+var anyConditionOpType = &operationType{Type: "ANY_CONDITION", NumArgs: 1, Precedence: 50, Handler: anyOperator}
+var allConditionOpType = &operationType{Type: "ALL_CONDITION", NumArgs: 1, Precedence: 50, Handler: allOperator}
+
+var toEntriesOpType = &operationType{Type: "TO_ENTRIES", NumArgs: 0, Precedence: 50, Handler: toEntriesOperator}
+var fromEntriesOpType = &operationType{Type: "FROM_ENTRIES", NumArgs: 0, Precedence: 50, Handler: fromEntriesOperator}
+var withEntriesOpType = &operationType{Type: "WITH_ENTRIES", NumArgs: 1, Precedence: 50, Handler: withEntriesOperator}
+
 var splitDocumentOpType = &operationType{Type: "SPLIT_DOC", NumArgs: 0, Precedence: 50, Handler: splitDocumentOperator}
 var getVariableOpType = &operationType{Type: "GET_VARIABLE", NumArgs: 0, Precedence: 55, Handler: getVariableOperator}
 var getStyleOpType = &operationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: getStyleOperator}
@@ -76,6 +83,9 @@ var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50,
 var sortKeysOpType = &operationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: sortKeysOperator}
 var joinStringOpType = &operationType{Type: "JOIN", NumArgs: 1, Precedence: 50, Handler: joinStringOperator}
 var subStringOpType = &operationType{Type: "SUBSTR", NumArgs: 1, Precedence: 50, Handler: substituteStringOperator}
+var matchOpType = &operationType{Type: "MATCH", NumArgs: 1, Precedence: 50, Handler: matchOperator}
+var captureOpType = &operationType{Type: "CAPTURE", NumArgs: 1, Precedence: 50, Handler: captureOperator}
+var testOpType = &operationType{Type: "TEST", NumArgs: 1, Precedence: 50, Handler: testOperator}
 var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 50, Handler: splitStringOperator}
 
 var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 50, Handler: keysOperator}
@@ -94,6 +104,8 @@ var recursiveDescentOpType = &operationType{Type: "RECURSIVE_DESCENT", NumArgs:
 
 var selectOpType = &operationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Handler: selectOperator}
 var hasOpType = &operationType{Type: "HAS", NumArgs: 1, Precedence: 50, Handler: hasOperator}
+var uniqueOpType = &operationType{Type: "UNIQUE", NumArgs: 0, Precedence: 50, Handler: unique}
+var uniqueByOpType = &operationType{Type: "UNIQUE_BY", NumArgs: 1, Precedence: 50, Handler: uniqueBy}
 var deleteChildOpType = &operationType{Type: "DELETE", NumArgs: 1, Precedence: 40, Handler: deleteChildOperator}
 
 type Operation struct {
@@ -105,8 +117,8 @@ type Operation struct {
 	UpdateAssign  bool // used for assign ops, when true it means we evaluate the rhs given the lhs
 }
 
-func createValueOperation(value interface{}, stringValue string) *Operation {
-	var node yaml.Node = yaml.Node{Kind: yaml.ScalarNode}
+func createScalarNode(value interface{}, stringValue string) *yaml.Node {
+	var node = &yaml.Node{Kind: yaml.ScalarNode}
 	node.Value = stringValue
 
 	switch value.(type) {
@@ -121,12 +133,17 @@ func createValueOperation(value interface{}, stringValue string) *Operation {
 	case nil:
 		node.Tag = "!!null"
 	}
+	return node
+}
+
+func createValueOperation(value interface{}, stringValue string) *Operation {
+	var node *yaml.Node = createScalarNode(value, stringValue)
 
 	return &Operation{
 		OperationType: valueOpType,
 		Value:         value,
 		StringValue:   stringValue,
-		CandidateNode: &CandidateNode{Node: &node},
+		CandidateNode: &CandidateNode{Node: node},
 	}
 }
 
@@ -170,7 +187,10 @@ func NodeToString(node *CandidateNode) string {
 	if errorEncoding != nil {
 		log.Error("Error debugging node, %v", errorEncoding.Error())
 	}
-	encoder.Close()
+	errorClosingEncoder := encoder.Close()
+	if errorClosingEncoder != nil {
+		log.Error("Error closing encoder: ", errorClosingEncoder.Error())
+	}
 	tag := value.Tag
 	if value.Kind == yaml.DocumentNode {
 		tag = "doc"

pkg/yqlib/operator_add.go

@@ -15,11 +15,7 @@ func createAddOp(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode {
 }
 
 func addAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	assignmentOp := &Operation{OperationType: assignOpType}
-	assignmentOp.UpdateAssign = true
-	selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
-	assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: expressionNode.Lhs, Rhs: createAddOp(selfExpression, expressionNode.Rhs)}
-	return d.GetMatchingNodes(context, assignmentOpNode)
+	return compoundAssignFunction(d, context, expressionNode, createAddOp)
 }
 
 func toNodes(candidate *CandidateNode) []*yaml.Node {
@@ -39,7 +35,7 @@ func toNodes(candidate *CandidateNode) []*yaml.Node {
 func addOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("Add operator")
 
-	return crossFunction(d, context, expressionNode, add, false)
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, add, false)
 }
 
 func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {

pkg/yqlib/operator_add_test.go

@@ -14,6 +14,22 @@ var addOperatorScenarios = []expressionScenario{
 			"D0, P[1 a], (!!int)::3\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `{}`,
+		expression: "(.a + .b) as $x",
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: 0`,
+		expression: ".a += .b.c",
+		expected: []string{
+			"D0, P[], (doc)::a: 0\n",
+		},
+	},
 	{
 		description: "Concatenate and assign arrays",
 		document:    `{a: {val: thing, b: [cat,dog]}}`,
@@ -70,6 +86,14 @@ var addOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: [1, 2, 3, 4], b: [3, 4]}\n",
 		},
 	},
+	{
+		description: "Append another array using +=",
+		document:    `{a: [1,2], b: [3,4]}`,
+		expression:  `.a += .b`,
+		expected: []string{
+			"D0, P[], (doc)::{a: [1, 2, 3, 4], b: [3, 4]}\n",
+		},
+	},
 	{
 		description: "Relative append",
 		document:    `a: { a1: {b: [cat]}, a2: {b: [dog]}, a3: {} }`,

pkg/yqlib/operator_alternative.go

@@ -2,13 +2,16 @@ package yqlib
 
 func alternativeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- alternative")
-	return crossFunction(d, context, expressionNode, alternativeFunc, true)
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, alternativeFunc, true)
 }
 
 func alternativeFunc(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	if lhs == nil {
 		return rhs, nil
 	}
+	if rhs == nil {
+		return lhs, nil
+	}
 	lhs.Node = unwrapDoc(lhs.Node)
 	rhs.Node = unwrapDoc(rhs.Node)
 	log.Debugf("Alternative LHS: %v", lhs.Node.Tag)

pkg/yqlib/operator_alternative_test.go

@@ -5,6 +5,28 @@ import (
 )
 
 var alternativeOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		expression: `.b // .c`,
+		document:   `a: bridge`,
+		expected:   []string{},
+	},
+	{
+		skipDoc:    true,
+		expression: `(.b // "hello") as $x`,
+		document:   `a: bridge`,
+		expected: []string{
+			"D0, P[], (doc)::a: bridge\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `.a // .b`,
+		document:   `a: 2`,
+		expected: []string{
+			"D0, P[a], (!!int)::2\n",
+		},
+	},
 	{
 		description: "LHS is defined",
 		expression:  `.a // "hello"`,

pkg/yqlib/operator_anchors_aliases.go

@@ -12,7 +12,7 @@ func assignAliasOperator(d *dataTreeNavigator, context Context, expressionNode *
 
 	aliasName := ""
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 		if err != nil {
 			return Context{}, err
 		}
@@ -32,7 +32,7 @@ func assignAliasOperator(d *dataTreeNavigator, context Context, expressionNode *
 		log.Debugf("Setting aliasName : %v", candidate.GetKey())
 
 		if expressionNode.Operation.UpdateAssign {
-			rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 			if err != nil {
 				return Context{}, err
 			}
@@ -41,8 +41,10 @@ func assignAliasOperator(d *dataTreeNavigator, context Context, expressionNode *
 			}
 		}
 
-		candidate.Node.Kind = yaml.AliasNode
-		candidate.Node.Value = aliasName
+		if aliasName != "" {
+			candidate.Node.Kind = yaml.AliasNode
+			candidate.Node.Value = aliasName
+		}
 	}
 	return context, nil
 }
@@ -66,7 +68,7 @@ func assignAnchorOperator(d *dataTreeNavigator, context Context, expressionNode
 
 	anchorName := ""
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 		if err != nil {
 			return Context{}, err
 		}
@@ -87,7 +89,7 @@ func assignAnchorOperator(d *dataTreeNavigator, context Context, expressionNode
 		log.Debugf("Setting anchorName of : %v", candidate.GetKey())
 
 		if expressionNode.Operation.UpdateAssign {
-			rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 			if err != nil {
 				return Context{}, err
 			}
@@ -176,7 +178,7 @@ func explodeNode(node *yaml.Node, context Context) error {
 			} else {
 				if valueNode.Kind == yaml.SequenceNode {
 					log.Debugf("an alias merge list!")
-					for index := 0; index < len(valueNode.Content); index = index + 1 {
+					for index := len(valueNode.Content) - 1; index >= 0; index = index - 1 {
 						aliasNode := valueNode.Content[index]
 						err := applyAlias(node, aliasNode.Alias, index, context.ChildContext(newContent))
 						if err != nil {

pkg/yqlib/operator_anchors_aliases_test.go

@@ -4,7 +4,59 @@ import (
 	"testing"
 )
 
+var specDocument = `- &CENTER { x: 1, y: 2 }
+- &LEFT { x: 0, y: 2 }
+- &BIG { r: 10 }
+- &SMALL { r: 1 }
+`
+
+var expectedSpecResult = "D0, P[4], (!!map)::x: 1\ny: 2\nr: 10\n"
+
+var simpleArrayRef = `
+item_value: &item_value
+  value: true
+
+thingOne:
+  name: item_1
+  <<: *item_value
+
+thingTwo:
+  name: item_2
+  <<: *item_value
+`
+
+var expectedUpdatedArrayRef = `D0, P[], (doc)::item_value: &item_value
+    value: true
+thingOne:
+    name: item_1
+    value: false
+thingTwo:
+    name: item_2
+    !!merge <<: *item_value
+`
+
 var anchorOperatorScenarios = []expressionScenario{
+	{
+		description:    "Merge one map",
+		subdescription: "see https://yaml.org/type/merge.html",
+		document:       specDocument + "- << : *CENTER\n  r: 10\n",
+		expression:     ".[4] | explode(.)",
+		expected:       []string{expectedSpecResult},
+	},
+	{
+		description:    "Merge multiple maps",
+		subdescription: "see https://yaml.org/type/merge.html",
+		document:       specDocument + "- << : [ *CENTER, *BIG ]\n",
+		expression:     ".[4] | explode(.)",
+		expected:       []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"},
+	},
+	{
+		description:    "Override",
+		subdescription: "see https://yaml.org/type/merge.html",
+		document:       specDocument + "- << : [ *BIG, *LEFT, *SMALL ]\n  x: 1\n",
+		expression:     ".[4] | explode(.)",
+		expected:       []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"},
+	},
 	{
 		description: "Get anchor",
 		document:    `a: &billyBob cat`,
@@ -29,6 +81,22 @@ var anchorOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: &cat {b: cat}\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `a: {c: cat}`,
+		expression: `.a anchor |= .b`,
+		expected: []string{
+			"D0, P[], (doc)::a: {c: cat}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: {c: cat}`,
+		expression: `.a anchor = .b`,
+		expected: []string{
+			"D0, P[], (doc)::a: {c: cat}\n",
+		},
+	},
 	{
 		description: "Get alias",
 		document:    `{b: &billyBob meow, a: *billyBob}`,
@@ -45,6 +113,30 @@ var anchorOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{b: &meow purr, a: *meow}\n",
 		},
 	},
+	{
+		description: "Set alias to blank does nothing",
+		document:    `{b: &meow purr, a: cat}`,
+		expression:  `.a alias = ""`,
+		expected: []string{
+			"D0, P[], (doc)::{b: &meow purr, a: cat}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{b: &meow purr, a: cat}`,
+		expression: `.a alias = .c`,
+		expected: []string{
+			"D0, P[], (doc)::{b: &meow purr, a: cat}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{b: &meow purr, a: cat}`,
+		expression: `.a alias |= .c`,
+		expected: []string{
+			"D0, P[], (doc)::{b: &meow purr, a: cat}\n",
+		},
+	},
 	{
 		description: "Set alias relatively using assign-update",
 		document:    `{b: &meow purr, a: {f: meow}}`,
@@ -91,9 +183,9 @@ bar:
     c: bar_c
 foobarList:
     b: bar_b
-    a: foo_a
-    thing: bar_thing
+    thing: foo_thing
     c: foobarList_c
+    a: foo_a
 foobar:
     c: foo_c
     a: foo_a
@@ -106,7 +198,7 @@ foobar:
 		expression: `.foo* | explode(.) | (. style="flow")`,
 		expected: []string{
 			"D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n",
-			"D0, P[foobarList], (!!map)::{b: bar_b, a: foo_a, thing: bar_thing, c: foobarList_c}\n",
+			"D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n",
 			"D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n",
 		},
 	},
@@ -116,7 +208,7 @@ foobar:
 		expression: `.foo* | explode(explode(.)) | (. style="flow")`,
 		expected: []string{
 			"D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n",
-			"D0, P[foobarList], (!!map)::{b: bar_b, a: foo_a, thing: bar_thing, c: foobarList_c}\n",
+			"D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n",
 			"D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n",
 		},
 	},
@@ -128,9 +220,16 @@ foobar:
 			"D0, P[], (doc)::{f: {a: cat, b: {f: cat}, cat: {f: cat}}}\n",
 		},
 	},
+	{
+		description:    "Dereference and update a field",
+		subdescription: "`Use explode with multiply to dereference an object",
+		document:       simpleArrayRef,
+		expression:     `.thingOne |= explode(.) * {"value": false}`,
+		expected:       []string{expectedUpdatedArrayRef},
+	},
 }
 
-func TestAnchorAliaseOperatorScenarios(t *testing.T) {
+func TestAnchorAliasOperatorScenarios(t *testing.T) {
 	for _, tt := range anchorOperatorScenarios {
 		testScenario(t, &tt)
 	}

pkg/yqlib/operator_assign.go

@@ -1,21 +1,27 @@
 package yqlib
 
+func assignUpdateFunc(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	rhs.Node = unwrapDoc(rhs.Node)
+	lhs.UpdateFrom(rhs)
+	return lhs, nil
+}
+
 func assignUpdateOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	lhs, err := d.GetMatchingNodes(context, expressionNode.Lhs)
 	if err != nil {
 		return Context{}, err
 	}
-	var rhs Context
+
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err = d.GetMatchingNodes(context, expressionNode.Rhs)
+		// this works because we already ran against LHS with an editable context.
+		_, err := crossFunction(d, context.ReadOnlyClone(), expressionNode, assignUpdateFunc, false)
+		return context, err
 	}
 
 	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 
-		if expressionNode.Operation.UpdateAssign {
-			rhs, err = d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
-		}
+		rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
 
 		if err != nil {
 			return Context{}, err
@@ -44,7 +50,7 @@ func assignAttributesOperator(d *dataTreeNavigator, context Context, expressionN
 	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 
-		rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 
 		if err != nil {
 			return Context{}, err

pkg/yqlib/operator_assign_test.go

@@ -12,6 +12,22 @@ var assignOperatorScenarios = []expressionScenario{
 			"D0, P[], ()::a:\n    b: cat\nx: frog\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: `.a |= .b`,
+		expected: []string{
+			"D0, P[], (doc)::{a: null}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: `.a = .b`,
+		expected: []string{
+			"D0, P[], (doc)::{a: null}\n",
+		},
+	},
 	{
 		description: "Update node to be the child value",
 		document:    `{a: {b: {g: foof}}}`,
@@ -20,6 +36,14 @@ var assignOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: {g: foof}}\n",
 		},
 	},
+	{
+		description: "Double elements in an array",
+		document:    `[1,2,3]`,
+		expression:  `.[] |= . * 2`,
+		expected: []string{
+			"D0, P[], (doc)::[2, 4, 6]\n",
+		},
+	},
 	{
 		description:    "Update node from another file",
 		subdescription: "Note this will also work when the second file is a scalar (string/number)",
@@ -41,7 +65,7 @@ var assignOperatorScenarios = []expressionScenario{
 	{
 		description: "Updated multiple paths",
 		document:    `{a: fieldA, b: fieldB, c: fieldC}`,
-		expression:  `(.a, .c) |= "potatoe"`,
+		expression:  `(.a, .c) = "potatoe"`,
 		expected: []string{
 			"D0, P[], (doc)::{a: potatoe, b: fieldB, c: potatoe}\n",
 		},

pkg/yqlib/operator_booleans.go

@@ -2,14 +2,13 @@ package yqlib
 
 import (
 	"container/list"
+	"fmt"
 
 	yaml "gopkg.in/yaml.v3"
 )
 
-func isTruthy(c *CandidateNode) (bool, error) {
-	node := unwrapDoc(c.Node)
+func isTruthyNode(node *yaml.Node) (bool, error) {
 	value := true
-
 	if node.Tag == "!!null" {
 		return false, nil
 	}
@@ -23,41 +22,131 @@ func isTruthy(c *CandidateNode) (bool, error) {
 	return value, nil
 }
 
+func isTruthy(c *CandidateNode) (bool, error) {
+	node := unwrapDoc(c.Node)
+	return isTruthyNode(node)
+}
+
 type boolOp func(bool, bool) bool
 
 func performBoolOp(op boolOp) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-		lhs.Node = unwrapDoc(lhs.Node)
-		rhs.Node = unwrapDoc(rhs.Node)
+		owner := lhs
 
-		lhsTrue, errDecoding := isTruthy(lhs)
-		if errDecoding != nil {
-			return nil, errDecoding
+		if lhs == nil && rhs == nil {
+			owner = &CandidateNode{}
+		} else if lhs == nil {
+			owner = rhs
 		}
 
-		rhsTrue, errDecoding := isTruthy(rhs)
-		if errDecoding != nil {
-			return nil, errDecoding
-		}
+		var errDecoding error
+		lhsTrue := false
+		if lhs != nil {
+			lhs.Node = unwrapDoc(lhs.Node)
+			lhsTrue, errDecoding = isTruthy(lhs)
 
-		return createBooleanCandidate(lhs, op(lhsTrue, rhsTrue)), nil
+			if errDecoding != nil {
+				return nil, errDecoding
+			}
+		}
+		log.Debugf("-- lhsTrue", lhsTrue)
+
+		rhsTrue := false
+		if rhs != nil {
+			rhs.Node = unwrapDoc(rhs.Node)
+			rhsTrue, errDecoding = isTruthy(rhs)
+			if errDecoding != nil {
+				return nil, errDecoding
+			}
+		}
+		log.Debugf("-- rhsTrue", rhsTrue)
+
+		return createBooleanCandidate(owner, op(lhsTrue, rhsTrue)), nil
 	}
 }
 
+func findBoolean(wantBool bool, d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, sequenceNode *yaml.Node) (bool, error) {
+	for _, node := range sequenceNode.Content {
+
+		if expressionNode != nil {
+			//need to evaluate the expression against the node
+			candidate := &CandidateNode{Node: node}
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode)
+			if err != nil {
+				return false, err
+			}
+			if rhs.MatchingNodes.Len() > 0 {
+				node = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node
+			} else {
+				// no results found, ignore this entry
+				continue
+			}
+		}
+
+		truthy, err := isTruthyNode(node)
+		if err != nil {
+			return false, err
+		}
+		if truthy == wantBool {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func allOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		candidateNode := unwrapDoc(candidate.Node)
+		if candidateNode.Kind != yaml.SequenceNode {
+			return Context{}, fmt.Errorf("any only supports arrays, was %v", candidateNode.Tag)
+		}
+		booleanResult, err := findBoolean(false, d, context, expressionNode.Rhs, candidateNode)
+		if err != nil {
+			return Context{}, err
+		}
+		result := createBooleanCandidate(candidate, !booleanResult)
+		results.PushBack(result)
+	}
+	return context.ChildContext(results), nil
+}
+
+func anyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		candidateNode := unwrapDoc(candidate.Node)
+		if candidateNode.Kind != yaml.SequenceNode {
+			return Context{}, fmt.Errorf("any only supports arrays, was %v", candidateNode.Tag)
+		}
+		booleanResult, err := findBoolean(true, d, context, expressionNode.Rhs, candidateNode)
+		if err != nil {
+			return Context{}, err
+		}
+		result := createBooleanCandidate(candidate, booleanResult)
+		results.PushBack(result)
+	}
+	return context.ChildContext(results), nil
+}
+
 func orOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- orOp")
-	return crossFunction(d, context, expressionNode, performBoolOp(
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, performBoolOp(
 		func(b1 bool, b2 bool) bool {
+			log.Debugf("-- peformingOrOp with %v and %v", b1, b2)
 			return b1 || b2
-		}), false)
+		}), true)
 }
 
 func andOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- AndOp")
-	return crossFunction(d, context, expressionNode, performBoolOp(
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, performBoolOp(
 		func(b1 bool, b2 bool) bool {
 			return b1 && b2
-		}), false)
+		}), true)
 }
 
 func notOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {

pkg/yqlib/operator_booleans_test.go

@@ -6,14 +6,46 @@ import (
 
 var booleanOperatorScenarios = []expressionScenario{
 	{
-		description: "OR example",
+		description: "`or` example",
 		expression:  `true or false`,
 		expected: []string{
 			"D0, P[], (!!bool)::true\n",
 		},
 	},
 	{
-		description: "AND example",
+		skipDoc:    true,
+		document:   "b: hi",
+		expression: `.a or .c`,
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "b: false",
+		expression: `.b or .c`,
+		expected: []string{
+			"D0, P[b], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "b: hi",
+		expression: `select(.a or .b)`,
+		expected: []string{
+			"D0, P[], (doc)::b: hi\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "b: hi",
+		expression: `select((.a and .b) | not)`,
+		expected: []string{
+			"D0, P[], (doc)::b: hi\n",
+		},
+	},
+	{
+		description: "`and` example",
 		expression:  `true and false`,
 		expected: []string{
 			"D0, P[], (!!bool)::false\n",
@@ -27,6 +59,86 @@ var booleanOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::- {a: bird, b: dog}\n- {a: cat, b: fly}\n",
 		},
 	},
+	{
+		description: "`any` returns true if any boolean in a given array is true",
+		document:    `[false, true]`,
+		expression:  "any",
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		description: "`any` returns false for an empty array",
+		document:    `[]`,
+		expression:  "any",
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		description: "`any_c` returns true if any element in the array is true for the given condition.",
+		document:    "a: [rad, awesome]\nb: [meh, whatever]",
+		expression:  `.[] |= any_c(. == "awesome")`,
+		expected: []string{
+			"D0, P[], (doc)::a: true\nb: false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{pet: cat}]`,
+		expression: `any_c(.name == "harry") as $c`,
+		expected: []string{
+			"D0, P[], (doc)::[{pet: cat}]\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{pet: cat}]`,
+		expression: `all_c(.name == "harry") as $c`,
+		expected: []string{
+			"D0, P[], (doc)::[{pet: cat}]\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[false, false]`,
+		expression: "any",
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		description: "`all` returns true if all booleans in a given array are true",
+		document:    `[true, true]`,
+		expression:  "all",
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[false, true]`,
+		expression: "all",
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		description: "`all` returns true for an empty array",
+		document:    `[]`,
+		expression:  "all",
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		description: "`all_c` returns true if all elements in the array are true for the given condition.",
+		document:    "a: [rad, awesome]\nb: [meh, 12]",
+		expression:  `.[] |= all_c(tag == "!!str")`,
+		expected: []string{
+			"D0, P[], (doc)::a: true\nb: false\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		expression: `false or false`,
@@ -45,6 +157,22 @@ var booleanOperatorScenarios = []expressionScenario{
 			"D0, P[b], (!!bool)::true\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `{}`,
+		expression: `(.a.b or .c) as $x`,
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{}`,
+		expression: `(.a.b and .c) as $x`,
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
 	{
 		description: "Not true is false",
 		expression:  `true | not`,

pkg/yqlib/operator_collect_object.go

@@ -17,9 +17,12 @@ import (
 ...
 */
 
-func collectObjectOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+func collectObjectOperator(d *dataTreeNavigator, originalContext Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- collectObjectOperation")
 
+	context := originalContext.Clone()
+	context.DontAutoCreate = false
+
 	if context.MatchingNodes.Len() == 0 {
 		node := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map", Value: "{}"}
 		candidate := &CandidateNode{Node: node}

pkg/yqlib/operator_collect_object_test.go

@@ -5,6 +5,30 @@ import (
 )
 
 var collectObjectOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   "a: []",
+		expression: `.a += [{"key": "att2", "value": "val2"}]`,
+		expected: []string{
+			"D0, P[], (doc)::a: [{key: att2, value: val2}]\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "",
+		expression: `.a += {"key": "att2", "value": "val2"}`,
+		expected: []string{
+			"D0, P[], ()::a:\n    key: att2\n    value: val2\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "",
+		expression: `.a += [0]`,
+		expected: []string{
+			"D0, P[], ()::a:\n    - 0\n",
+		},
+	},
 	{
 		description: `Collect empty object`,
 		document:    ``,

pkg/yqlib/operator_collect_test.go

@@ -5,6 +5,14 @@ import (
 )
 
 var collectOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   ``,
+		expression: `.a += [0]`,
+		expected: []string{
+			"D0, P[], ()::a:\n    - 0\n",
+		},
+	},
 	{
 		description: "Collect empty",
 		document:    ``,

pkg/yqlib/operator_comments.go

@@ -27,7 +27,7 @@ func assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNod
 
 	comment := ""
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 		if err != nil {
 			return Context{}, err
 		}
@@ -41,7 +41,7 @@ func assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNod
 		candidate := el.Value.(*CandidateNode)
 
 		if expressionNode.Operation.UpdateAssign {
-			rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 			if err != nil {
 				return Context{}, err
 			}

pkg/yqlib/operator_comments_test.go

@@ -62,6 +62,22 @@ var commentOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: cat\n\n# cat\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `a: cat`,
+		expression: `. footComment=.b.d`,
+		expected: []string{
+			"D0, P[], (doc)::a: cat\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: cat`,
+		expression: `. footComment|=.b.d`,
+		expected: []string{
+			"D0, P[], (doc)::a: cat\n",
+		},
+	},
 	{
 		description: "Remove comment",
 		document:    "a: cat # comment\nb: dog # leave this",
@@ -71,7 +87,7 @@ var commentOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description:    "Remove all comments",
+		description:    "Remove (strip) all comments",
 		subdescription: "Note the use of `...` to ensure key nodes are included.",
 		document:       "# hi\n\na: cat # comment\n\n# great\n\nb: # key comment",
 		expression:     `... comments=""`,

pkg/yqlib/operator_delete.go

@@ -7,9 +7,7 @@ import (
 )
 
 func deleteChildOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	contextToUse := context.Clone()
-	contextToUse.DontAutoCreate = true
-	nodesToDelete, err := d.GetMatchingNodes(contextToUse, expressionNode.Rhs)
+	nodesToDelete, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 
 	if err != nil {
 		return Context{}, err

pkg/yqlib/operator_entries.go

@@ -0,0 +1,162 @@
+package yqlib
+
+import (
+	"container/list"
+	"fmt"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+func entrySeqFor(key *yaml.Node, value *yaml.Node) *yaml.Node {
+	var keyKey = &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "key"}
+	var valueKey = &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "value"}
+
+	return &yaml.Node{
+		Kind:    yaml.MappingNode,
+		Tag:     "!!map",
+		Content: []*yaml.Node{keyKey, key, valueKey, value},
+	}
+}
+
+func toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode {
+	var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+	var entriesNode = candidateNode.CreateChild(nil, sequence)
+
+	var contents = unwrapDoc(candidateNode.Node).Content
+	for index := 0; index < len(contents); index = index + 2 {
+		key := contents[index]
+		value := contents[index+1]
+
+		sequence.Content = append(sequence.Content, entrySeqFor(key, value))
+	}
+	return entriesNode
+}
+
+func toEntriesfromSeq(candidateNode *CandidateNode) *CandidateNode {
+	var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+	var entriesNode = candidateNode.CreateChild(nil, sequence)
+
+	var contents = unwrapDoc(candidateNode.Node).Content
+	for index := 0; index < len(contents); index = index + 1 {
+		key := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!int", Value: fmt.Sprintf("%v", index)}
+		value := contents[index]
+
+		sequence.Content = append(sequence.Content, entrySeqFor(key, value))
+	}
+	return entriesNode
+}
+
+func toEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	var results = list.New()
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		candidateNode := unwrapDoc(candidate.Node)
+
+		switch candidateNode.Kind {
+		case yaml.MappingNode:
+			results.PushBack(toEntriesFromMap(candidate))
+
+		case yaml.SequenceNode:
+			results.PushBack(toEntriesfromSeq(candidate))
+		default:
+			if candidateNode.Tag != "!!null" {
+				return Context{}, fmt.Errorf("%v has no keys", candidate.Node.Tag)
+			}
+		}
+	}
+
+	return context.ChildContext(results), nil
+}
+
+func parseEntry(d *dataTreeNavigator, entry *yaml.Node, position int) (*yaml.Node, *yaml.Node, error) {
+	prefs := traversePreferences{DontAutoCreate: true}
+	candidateNode := &CandidateNode{Node: entry}
+
+	keyResults, err := traverseMap(Context{}, candidateNode, "key", prefs, false)
+
+	if err != nil {
+		return nil, nil, err
+	} else if keyResults.Len() != 1 {
+		return nil, nil, fmt.Errorf("Expected to find one 'key' entry but found %v in position %v", keyResults.Len(), position)
+	}
+
+	valueResults, err := traverseMap(Context{}, candidateNode, "value", prefs, false)
+
+	if err != nil {
+		return nil, nil, err
+	} else if valueResults.Len() != 1 {
+		return nil, nil, fmt.Errorf("Expected to find one 'value' entry but found %v in position %v", valueResults.Len(), position)
+	}
+
+	return keyResults.Front().Value.(*CandidateNode).Node, valueResults.Front().Value.(*CandidateNode).Node, nil
+
+}
+
+func fromEntries(d *dataTreeNavigator, candidateNode *CandidateNode) (*CandidateNode, error) {
+	var node = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
+	var mapCandidateNode = candidateNode.CreateChild(nil, node)
+
+	var contents = unwrapDoc(candidateNode.Node).Content
+
+	for index := 0; index < len(contents); index = index + 1 {
+		key, value, err := parseEntry(d, contents[index], index)
+		if err != nil {
+			return nil, err
+		}
+
+		node.Content = append(node.Content, key, value)
+	}
+	return mapCandidateNode, nil
+}
+
+func fromEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	var results = list.New()
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		candidateNode := unwrapDoc(candidate.Node)
+
+		switch candidateNode.Kind {
+		case yaml.SequenceNode:
+			mapResult, err := fromEntries(d, candidate)
+			if err != nil {
+				return Context{}, err
+			}
+			results.PushBack(mapResult)
+		default:
+			return Context{}, fmt.Errorf("from entries only runs against arrays")
+		}
+	}
+
+	return context.ChildContext(results), nil
+}
+
+func withEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	//to_entries on the context
+	toEntries, err := toEntriesOperator(d, context, expressionNode)
+	if err != nil {
+		return Context{}, err
+	}
+
+	//run expression against entries
+	// splat toEntries and pipe it into Rhs
+	splatted, err := splat(d, toEntries, traversePreferences{})
+	if err != nil {
+		return Context{}, err
+	}
+
+	result, err := d.GetMatchingNodes(splatted, expressionNode.Rhs)
+	log.Debug("expressionNode.Rhs %v", expressionNode.Rhs.Operation.OperationType)
+	log.Debug("result %v", result)
+	if err != nil {
+		return Context{}, err
+	}
+
+	collected, err := collectOperator(d, result, expressionNode)
+	if err != nil {
+		return Context{}, err
+	}
+
+	//from_entries on the result
+	return fromEntriesOperator(d, collected, expressionNode)
+}

pkg/yqlib/operator_entries_test.go

@@ -0,0 +1,70 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var entriesOperatorScenarios = []expressionScenario{
+	{
+		description: "to_entries Map",
+		document:    `{a: 1, b: 2}`,
+		expression:  `to_entries`,
+		expected: []string{
+			"D0, P[], (!!seq)::- key: a\n  value: 1\n- key: b\n  value: 2\n",
+		},
+	},
+	{
+		description: "to_entries Array",
+		document:    `[a, b]`,
+		expression:  `to_entries`,
+		expected: []string{
+			"D0, P[], (!!seq)::- key: 0\n  value: a\n- key: 1\n  value: b\n",
+		},
+	},
+	{
+		description: "to_entries null",
+		document:    `null`,
+		expression:  `to_entries`,
+		expected:    []string{},
+	},
+	{
+		description: "from_entries map",
+		document:    `{a: 1, b: 2}`,
+		expression:  `to_entries | from_entries`,
+		expected: []string{
+			"D0, P[], (!!map)::a: 1\nb: 2\n",
+		},
+	},
+	{
+		description:    "from_entries with numeric key indexes",
+		subdescription: "from_entries always creates a map, even for numeric keys",
+		document:       `[a,b]`,
+		expression:     `to_entries | from_entries`,
+		expected: []string{
+			"D0, P[], (!!map)::0: a\n1: b\n",
+		},
+	},
+	{
+		description: "Use with_entries to update keys",
+		document:    `{a: 1, b: 2}`,
+		expression:  `with_entries(.key |= "KEY_" + .)`,
+		expected: []string{
+			"D0, P[], (!!map)::KEY_a: 1\nKEY_b: 2\n",
+		},
+	},
+	{
+		description: "Use with_entries to filter the map",
+		document:    `{a: { b: bird }, c: { d: dog }}`,
+		expression:  `with_entries(select(.value | has("b")))`,
+		expected: []string{
+			"D0, P[], (!!map)::a: {b: bird}\n",
+		},
+	},
+}
+
+func TestEntriesOperatorScenarios(t *testing.T) {
+	for _, tt := range entriesOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "Entries", entriesOperatorScenarios)
+}

pkg/yqlib/operator_equals.go

@@ -4,12 +4,33 @@ import "gopkg.in/yaml.v3"
 
 func equalsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- equalsOperation")
-	return crossFunction(d, context, expressionNode, isEquals(false), false)
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, isEquals(false), true)
 }
 
 func isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 		value := false
+		log.Debugf("-- isEquals cross function")
+		if lhs == nil && rhs == nil {
+			owner := &CandidateNode{}
+			return createBooleanCandidate(owner, !flip), nil
+		} else if lhs == nil {
+			log.Debugf("lhs nil, but rhs is not")
+			rhsNode := unwrapDoc(rhs.Node)
+			value := rhsNode.Tag == "!!null"
+			if flip {
+				value = !value
+			}
+			return createBooleanCandidate(rhs, value), nil
+		} else if rhs == nil {
+			log.Debugf("lhs not nil, but rhs is")
+			lhsNode := unwrapDoc(lhs.Node)
+			value := lhsNode.Tag == "!!null"
+			if flip {
+				value = !value
+			}
+			return createBooleanCandidate(lhs, value), nil
+		}
 
 		lhsNode := unwrapDoc(lhs.Node)
 		rhsNode := unwrapDoc(rhs.Node)
@@ -28,6 +49,6 @@ func isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *Candid
 }
 
 func notEqualsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	log.Debugf("-- equalsOperation")
-	return crossFunction(d, context, expressionNode, isEquals(true), false)
+	log.Debugf("-- notEqualsOperator")
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, isEquals(true), true)
 }

pkg/yqlib/operator_equals_test.go

@@ -5,6 +5,29 @@ import (
 )
 
 var equalsOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		expression: ".a == .b",
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: cat`,
+		expression: ".a == .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: cat`,
+		expression: ".b == .a",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   "cat",
@@ -14,6 +37,60 @@ var equalsOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!bool)::false\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: "(.a == .b) as $x",
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: ".a == .b",
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: "(.a != .b) as $x",
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: ".a != .b",
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{a: {b: 10}}",
+		expression: "select(.c != null)",
+		expected:   []string{},
+	},
+	{
+		skipDoc:    true,
+		document:   "{a: {b: 10}}",
+		expression: "select(.d == .c)",
+		expected: []string{
+			"D0, P[], (doc)::{a: {b: 10}}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{a: {b: 10}}",
+		expression: "select(null == .c)",
+		expected: []string{
+			"D0, P[], (doc)::{a: {b: 10}}\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   "{a: { b: {things: \"\"}, f: [1], g: [] }}",
@@ -87,6 +164,22 @@ var equalsOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!bool)::true\n",
 		},
 	},
+	{
+		description: "Non exisitant key doesn't equal a value",
+		document:    "a: frog",
+		expression:  `select(.b != "thing")`,
+		expected: []string{
+			"D0, P[], (doc)::a: frog\n",
+		},
+	},
+	{
+		description: "Two non existant keys are equal",
+		document:    "a: frog",
+		expression:  `select(.b == .c)`,
+		expected: []string{
+			"D0, P[], (doc)::a: frog\n",
+		},
+	},
 }
 
 func TestEqualOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_has.go

@@ -12,14 +12,19 @@ func hasOperator(d *dataTreeNavigator, context Context, expressionNode *Expressi
 	log.Debugf("-- hasOperation")
 	var results = list.New()
 
-	rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
-	wanted := rhs.MatchingNodes.Front().Value.(*CandidateNode).Node
-	wantedKey := wanted.Value
+	rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 
 	if err != nil {
 		return Context{}, err
 	}
 
+	wantedKey := "null"
+	wanted := &yaml.Node{Tag: "!!null"}
+	if rhs.MatchingNodes.Len() != 0 {
+		wanted = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node
+		wantedKey = wanted.Value
+	}
+
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 

pkg/yqlib/operator_has_test.go

@@ -13,6 +13,22 @@ var hasOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!bool)::true\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `a: hello`,
+		expression: `has(.b) as $c`,
+		expected: []string{
+			"D0, P[], (doc)::a: hello\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: hello`,
+		expression: `has(.b)`,
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
 	{
 		description: "Has map key",
 		document: `- a: "yes"

pkg/yqlib/operator_multiply.go

@@ -32,11 +32,11 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
 
 			var newBlank = lhs.CreateChild(nil, &yaml.Node{})
 
-			var newThing, err = mergeObjects(d, context, newBlank, lhs, multiplyPreferences{})
+			var newThing, err = mergeObjects(d, context.WritableClone(), newBlank, lhs, multiplyPreferences{})
 			if err != nil {
 				return nil, err
 			}
-			return mergeObjects(d, context, newThing, rhs, preferences)
+			return mergeObjects(d, context.WritableClone(), newThing, rhs, preferences)
 		} else if lhs.Node.Tag == "!!int" && rhs.Node.Tag == "!!int" {
 			return multiplyIntegers(lhs, rhs)
 		} else if (lhs.Node.Tag == "!!int" || lhs.Node.Tag == "!!float") && (rhs.Node.Tag == "!!int" || rhs.Node.Tag == "!!float") {

pkg/yqlib/operator_multiply_test.go

@@ -24,6 +24,12 @@ list2:
     - "123"
 `
 
+var mergeArraysObjectKeysText = `It's a complex command, the trickyness comes from needing to have the right context in the expressions.
+First we save the second array into a variable '$two' which lets us reference it later.
+We then need to update the first array. We will use the relative update (|=) because we need to update relative to the current element of the array in the LHS in the RHS expression. 
+We set the current element of the first array as $cur. Now we multiply (merge) $cur with the matching entry in $two, by passing $two through a select filter.
+`
+
 var multiplyOperatorScenarios = []expressionScenario{
 	{
 		description: "Multiply integers",
@@ -41,6 +47,20 @@ var multiplyOperatorScenarios = []expressionScenario{
 			docExpected,
 		},
 	},
+	{
+		skipDoc:    true,
+		expression: `.x =  {"things": "whatever"} * {}`,
+		expected: []string{
+			"D0, P[], ()::x:\n    things: whatever\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `.x = {} * {"things": "whatever"}`,
+		expected: []string{
+			"D0, P[], ()::x:\n    things: whatever\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		expression: `3 * 4.5`,
@@ -245,6 +265,16 @@ var multiplyOperatorScenarios = []expressionScenario{
 			"D0, P[a], (!!seq)::[{name: fred, age: 34}, {name: bob, age: 32}]\n",
 		},
 	},
+	{
+		description:    "Merge arrays of objects together, matching on a key",
+		subdescription: mergeArraysObjectKeysText,
+		document:       `[{a: apple, b: appleB}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB}]`,
+		document2:      `[{a: banana, c: bananaC}, {a: apple, b: appleB2}, {a: dingo, c: dingoC}]`,
+		expression:     `(select(fi==1) | .[]) as $two | select(fi==0) | .[] |= (. as $cur |  $cur * ($two | select(.a == $cur.a)))`,
+		expected: []string{
+			"D0, P[], (doc)::[{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC}]\n",
+		},
+	},
 	{
 		description: "Merge to prefix an element",
 		document:    `{a: cat, b: dog}`,

pkg/yqlib/operator_select.go

@@ -11,9 +11,7 @@ func selectOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
-		childContext := context.SingleChildContext(candidate)
-		childContext.DontAutoCreate = true
-		rhs, err := d.GetMatchingNodes(childContext, expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 
 		if err != nil {
 			return Context{}, err
@@ -24,7 +22,9 @@ func selectOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 
 		if first != nil {
 			result := first.Value.(*CandidateNode)
+			log.Debugf("result %v", NodeToString(result))
 			includeResult, errDecoding := isTruthy(result)
+			log.Debugf("isTruthy %v", includeResult)
 			if errDecoding != nil {
 				return Context{}, errDecoding
 			}

pkg/yqlib/operator_select_test.go

@@ -14,6 +14,16 @@ var selectOperatorScenarios = []expressionScenario{
 			"D0, P[1], (!!str)::goat\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   "a: hello",
+		document2:  "b: world",
+		expression: `select(.a == "hello" or .b == "world")`,
+		expected: []string{
+			"D0, P[], (doc)::a: hello\n",
+			"D0, P[], (doc)::b: world\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `[{animal: cat, legs: {cool: true}}, {animal: fish}]`,

pkg/yqlib/operator_sort_keys.go

@@ -10,7 +10,7 @@ func sortKeysOperator(d *dataTreeNavigator, context Context, expressionNode *Exp
 
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
-		rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 		if err != nil {
 			return Context{}, err
 		}

pkg/yqlib/operator_sort_keys_test.go

@@ -13,6 +13,14 @@ var sortKeysOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: blah, b: bing, c: frog}\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `{c: frog}`,
+		expression: `sortKeys(.d)`,
+		expected: []string{
+			"D0, P[], (doc)::{c: frog}\n",
+		},
+	},
 	{
 		description:    "Sort keys recursively",
 		subdescription: "Note the array elements are left unsorted, but maps inside arrays are sorted",

pkg/yqlib/operator_strings.go

@@ -13,7 +13,7 @@ func getSubstituteParameters(d *dataTreeNavigator, block *ExpressionNode, contex
 	regEx := ""
 	replacementText := ""
 
-	regExNodes, err := d.GetMatchingNodes(context, block.Lhs)
+	regExNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), block.Lhs)
 	if err != nil {
 		return "", "", err
 	}
@@ -74,11 +74,237 @@ func substituteStringOperator(d *dataTreeNavigator, context Context, expressionN
 
 }
 
+func addMatch(original []*yaml.Node, match string, offset int, name string) []*yaml.Node {
+
+	newContent := append(original,
+		createScalarNode("string", "string"))
+
+	if offset < 0 {
+		// offset of -1 means there was no match, force a null value like jq
+		newContent = append(newContent,
+			createScalarNode(nil, "null"),
+		)
+	} else {
+		newContent = append(newContent,
+			createScalarNode(match, match),
+		)
+	}
+
+	newContent = append(newContent,
+		createScalarNode("offset", "offset"),
+		createScalarNode(offset, fmt.Sprintf("%v", offset)),
+		createScalarNode("length", "length"),
+		createScalarNode(len(match), fmt.Sprintf("%v", len(match))))
+
+	if name != "" {
+		newContent = append(newContent,
+			createScalarNode("name", "name"),
+			createScalarNode(name, name),
+		)
+	}
+	return newContent
+}
+
+type matchPreferences struct {
+	Global bool
+}
+
+func getMatches(matchPrefs matchPreferences, regEx *regexp.Regexp, value string) ([][]string, [][]int) {
+	var allMatches [][]string
+	var allIndices [][]int
+
+	if matchPrefs.Global {
+		allMatches = regEx.FindAllStringSubmatch(value, -1)
+		allIndices = regEx.FindAllStringSubmatchIndex(value, -1)
+	} else {
+		allMatches = [][]string{regEx.FindStringSubmatch(value)}
+		allIndices = [][]int{regEx.FindStringSubmatchIndex(value)}
+	}
+
+	log.Debug("allMatches, %v", allMatches)
+	return allMatches, allIndices
+}
+
+func match(matchPrefs matchPreferences, regEx *regexp.Regexp, candidate *CandidateNode, value string, results *list.List) {
+	subNames := regEx.SubexpNames()
+	allMatches, allIndices := getMatches(matchPrefs, regEx, value)
+
+	// if all matches just has an empty array in it,
+	// then nothing matched
+	if len(allMatches) > 0 && len(allMatches[0]) == 0 {
+		return
+	}
+
+	for i, matches := range allMatches {
+		capturesNode := &yaml.Node{Kind: yaml.SequenceNode}
+		match, submatches := matches[0], matches[1:]
+		for j, submatch := range submatches {
+			captureNode := &yaml.Node{Kind: yaml.MappingNode}
+			captureNode.Content = addMatch(capturesNode.Content, submatch, allIndices[i][2+j*2], subNames[j+1])
+			capturesNode.Content = append(capturesNode.Content, captureNode)
+		}
+
+		node := &yaml.Node{Kind: yaml.MappingNode}
+		node.Content = addMatch(node.Content, match, allIndices[i][0], "")
+		node.Content = append(node.Content,
+			createScalarNode("captures", "captures"),
+			capturesNode,
+		)
+		results.PushBack(candidate.CreateChild(nil, node))
+
+	}
+
+}
+
+func capture(matchPrefs matchPreferences, regEx *regexp.Regexp, candidate *CandidateNode, value string, results *list.List) {
+	subNames := regEx.SubexpNames()
+	allMatches, allIndices := getMatches(matchPrefs, regEx, value)
+
+	// if all matches just has an empty array in it,
+	// then nothing matched
+	if len(allMatches) > 0 && len(allMatches[0]) == 0 {
+		return
+	}
+
+	for i, matches := range allMatches {
+		capturesNode := &yaml.Node{Kind: yaml.MappingNode}
+
+		_, submatches := matches[0], matches[1:]
+		for j, submatch := range submatches {
+			capturesNode.Content = append(capturesNode.Content,
+				createScalarNode(subNames[j+1], subNames[j+1]))
+
+			offset := allIndices[i][2+j*2]
+			// offset of -1 means there was no match, force a null value like jq
+			if offset < 0 {
+				capturesNode.Content = append(capturesNode.Content,
+					createScalarNode(nil, "null"),
+				)
+			} else {
+				capturesNode.Content = append(capturesNode.Content,
+					createScalarNode(submatch, submatch),
+				)
+			}
+		}
+
+		results.PushBack(candidate.CreateChild(nil, capturesNode))
+
+	}
+
+}
+
+func extractMatchArguments(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (*regexp.Regexp, matchPreferences, error) {
+	regExExpNode := expressionNode.Rhs
+
+	matchPrefs := matchPreferences{}
+
+	// we got given parameters e.g. match(exp; params)
+	if expressionNode.Rhs.Operation.OperationType == blockOpType {
+		block := expressionNode.Rhs
+		regExExpNode = block.Lhs
+		replacementNodes, err := d.GetMatchingNodes(context, block.Rhs)
+		if err != nil {
+			return nil, matchPrefs, err
+		}
+		paramText := ""
+		if replacementNodes.MatchingNodes.Front() != nil {
+			paramText = replacementNodes.MatchingNodes.Front().Value.(*CandidateNode).Node.Value
+		}
+		if strings.Contains(paramText, "g") {
+			paramText = strings.ReplaceAll(paramText, "g", "")
+			matchPrefs.Global = true
+		}
+		if strings.Contains(paramText, "i") {
+			return nil, matchPrefs, fmt.Errorf(`'i' is not a valid option for match. To ignore case, use an expression like match("(?i)cat")`)
+		}
+		if len(paramText) > 0 {
+			return nil, matchPrefs, fmt.Errorf(`Unrecognised match params '%v', please see docs at https://mikefarah.gitbook.io/yq/operators/string-operators`, paramText)
+		}
+	}
+
+	regExNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), regExExpNode)
+	if err != nil {
+		return nil, matchPrefs, err
+	}
+	log.Debug(NodesToString(regExNodes.MatchingNodes))
+	regExStr := ""
+	if regExNodes.MatchingNodes.Front() != nil {
+		regExStr = regExNodes.MatchingNodes.Front().Value.(*CandidateNode).Node.Value
+	}
+	log.Debug("regEx %v", regExStr)
+	regEx, err := regexp.Compile(regExStr)
+	return regEx, matchPrefs, err
+}
+
+func matchOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	regEx, matchPrefs, err := extractMatchArguments(d, context, expressionNode)
+	if err != nil {
+		return Context{}, err
+	}
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		node := unwrapDoc(candidate.Node)
+		if node.Tag != "!!str" {
+			return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
+		}
+
+		match(matchPrefs, regEx, candidate, node.Value, results)
+	}
+
+	return context.ChildContext(results), nil
+}
+
+func captureOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	regEx, matchPrefs, err := extractMatchArguments(d, context, expressionNode)
+	if err != nil {
+		return Context{}, err
+	}
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		node := unwrapDoc(candidate.Node)
+		if node.Tag != "!!str" {
+			return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
+		}
+		capture(matchPrefs, regEx, candidate, node.Value, results)
+
+	}
+
+	return context.ChildContext(results), nil
+}
+
+func testOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	regEx, _, err := extractMatchArguments(d, context, expressionNode)
+	if err != nil {
+		return Context{}, err
+	}
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		node := unwrapDoc(candidate.Node)
+		if node.Tag != "!!str" {
+			return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
+		}
+		matches := regEx.FindStringSubmatch(node.Value)
+		results.PushBack(createBooleanCandidate(candidate, len(matches) > 0))
+
+	}
+
+	return context.ChildContext(results), nil
+}
+
 func joinStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- joinStringOperator")
 	joinStr := ""
 
-	rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+	rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 	if err != nil {
 		return Context{}, err
 	}
@@ -119,7 +345,7 @@ func splitStringOperator(d *dataTreeNavigator, context Context, expressionNode *
 	log.Debugf("-- splitStringOperator")
 	splitStr := ""
 
-	rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+	rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 	if err != nil {
 		return Context{}, err
 	}

pkg/yqlib/operator_strings_test.go

@@ -13,6 +13,110 @@ var stringsOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!str)::cat; meow; 1; ; true\n",
 		},
 	},
+	{
+		description: "Match string",
+		document:    `foo bar foo`,
+		expression:  `match("foo")`,
+		expected: []string{
+			"D0, P[], ()::string: foo\noffset: 0\nlength: 3\ncaptures: []\n",
+		},
+	},
+	{
+		description: "Match string, case insensitive",
+		document:    `foo bar FOO`,
+		expression:  `[match("(?i)foo"; "g")]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- string: foo\n  offset: 0\n  length: 3\n  captures: []\n- string: FOO\n  offset: 8\n  length: 3\n  captures: []\n",
+		},
+	},
+	{
+		description: "Match with capture groups",
+		document:    `abc abc`,
+		expression:  `[match("(abc)+"; "g")]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- string: abc\n  offset: 0\n  length: 3\n  captures:\n    - string: abc\n      offset: 0\n      length: 3\n- string: abc\n  offset: 4\n  length: 3\n  captures:\n    - string: abc\n      offset: 4\n      length: 3\n",
+		},
+	},
+	{
+		description: "Match with named capture groups",
+		document:    `foo bar foo foo  foo`,
+		expression:  `[match("foo (?P<bar123>bar)? foo"; "g")]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- string: foo bar foo\n  offset: 0\n  length: 11\n  captures:\n    - string: bar\n      offset: 4\n      length: 3\n      name: bar123\n- string: foo  foo\n  offset: 12\n  length: 8\n  captures:\n    - string: null\n      offset: -1\n      length: 0\n      name: bar123\n",
+		},
+	},
+	{
+		description: "Capture named groups into a map",
+		document:    `xyzzy-14`,
+		expression:  `capture("(?P<a>[a-z]+)-(?P<n>[0-9]+)")`,
+		expected: []string{
+			"D0, P[], ()::a: xyzzy\nn: \"14\"\n",
+		},
+	},
+	{
+		skipDoc:     true,
+		description: "Capture named groups into a map, with null",
+		document:    `xyzzy-14`,
+		expression:  `capture("(?P<a>[a-z]+)-(?P<n>[0-9]+)(?P<bar123>bar)?")`,
+		expected: []string{
+			"D0, P[], ()::a: xyzzy\nn: \"14\"\nbar123: null\n",
+		},
+	},
+	{
+		description: "Match without global flag",
+		document:    `cat cat`,
+		expression:  `match("cat")`,
+		expected: []string{
+			"D0, P[], ()::string: cat\noffset: 0\nlength: 3\ncaptures: []\n",
+		},
+	},
+	{
+		description: "Match with global flag",
+		document:    `cat cat`,
+		expression:  `[match("cat"; "g")]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- string: cat\n  offset: 0\n  length: 3\n  captures: []\n- string: cat\n  offset: 4\n  length: 3\n  captures: []\n",
+		},
+	},
+	{
+		skipDoc:     true,
+		description: "No match",
+		document:    `dog`,
+		expression:  `match("cat"; "g")`,
+		expected:    []string{},
+	},
+	{
+		skipDoc:     true,
+		description: "No match",
+		expression:  `"dog" | match("cat", "g")`,
+		expected:    []string{},
+	},
+	{
+		skipDoc:     true,
+		description: "No match",
+		expression:  `"dog" | match("cat")`,
+		expected:    []string{},
+	},
+	{
+		description:    "Test using regex",
+		subdescription: "Like jq'q equivalant, this works like match but only returns true/false instead of full match details",
+		document:       `["cat", "dog"]`,
+		expression:     `.[] | test("at")`,
+		expected: []string{
+			"D0, P[0], (!!bool)::true\n",
+			"D0, P[1], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `["cat*", "cat*", "cat"]`,
+		expression: `.[] | test("cat\*")`,
+		expected: []string{
+			"D0, P[0], (!!bool)::true\n",
+			"D0, P[1], (!!bool)::true\n",
+			"D0, P[2], (!!bool)::false\n",
+		},
+	},
 	{
 		description:    "Substitute / Replace string",
 		subdescription: "This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)\nNote the use of `|=` to run in context of the current string value.",

pkg/yqlib/operator_style.go

@@ -31,7 +31,7 @@ func assignStyleOperator(d *dataTreeNavigator, context Context, expressionNode *
 	log.Debugf("AssignStyleOperator: %v")
 	var style yaml.Style
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 		if err != nil {
 			return Context{}, err
 		}
@@ -54,7 +54,7 @@ func assignStyleOperator(d *dataTreeNavigator, context Context, expressionNode *
 		candidate := el.Value.(*CandidateNode)
 		log.Debugf("Setting style of : %v", candidate.GetKey())
 		if expressionNode.Operation.UpdateAssign {
-			rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 			if err != nil {
 				return Context{}, err
 			}

pkg/yqlib/operator_style_test.go

@@ -5,6 +5,23 @@ import (
 )
 
 var styleOperatorScenarios = []expressionScenario{
+	{
+		description: "Update and set style of a particular node (simple)",
+		document:    `a: {b: thing, c: something}`,
+		expression:  `.a.b = "new" | .a.b style="double"`,
+		expected: []string{
+			"D0, P[], (doc)::a: {b: \"new\", c: something}\n",
+		},
+	},
+	{
+		description:    "Update and set style of a particular node using path variables",
+		subdescription: "You can use a variable to re-use a path",
+		document:       `a: {b: thing, c: something}`,
+		expression:     `.a.b as $x | $x = "new" | $x style="double"`,
+		expected: []string{
+			"D0, P[], (doc)::a: {b: \"new\", c: something}\n",
+		},
+	},
 	{
 		description: "Set tagged style",
 		document:    `{a: cat, b: 5, c: 3.2, e: true}`,

pkg/yqlib/operator_subtract.go

@@ -15,17 +15,13 @@ func createSubtractOp(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode
 }
 
 func subtractAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	assignmentOp := &Operation{OperationType: assignOpType}
-	assignmentOp.UpdateAssign = true
-	selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
-	assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: expressionNode.Lhs, Rhs: createSubtractOp(selfExpression, expressionNode.Rhs)}
-	return d.GetMatchingNodes(context, assignmentOpNode)
+	return compoundAssignFunction(d, context, expressionNode, createSubtractOp)
 }
 
 func subtractOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("Subtract operator")
 
-	return crossFunction(d, context, expressionNode, subtract, false)
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, subtract, false)
 }
 
 func subtract(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {

pkg/yqlib/operator_subtract_test.go

@@ -5,6 +5,14 @@ import (
 )
 
 var subtractOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   `{}`,
+		expression: "(.a - .b) as $x",
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
 	{
 		description:    "Number subtraction - float",
 		subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.",

pkg/yqlib/operator_tag.go

@@ -12,7 +12,7 @@ func assignTagOperator(d *dataTreeNavigator, context Context, expressionNode *Ex
 	tag := ""
 
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 		if err != nil {
 			return Context{}, err
 		}
@@ -32,7 +32,7 @@ func assignTagOperator(d *dataTreeNavigator, context Context, expressionNode *Ex
 		candidate := el.Value.(*CandidateNode)
 		log.Debugf("Setting tag of : %v", candidate.GetKey())
 		if expressionNode.Operation.UpdateAssign {
-			rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 			if err != nil {
 				return Context{}, err
 			}

pkg/yqlib/operator_traverse_path.go

@@ -14,6 +14,7 @@ type traversePreferences struct {
 	IncludeMapKeys       bool
 	DontAutoCreate       bool // by default, we automatically create entries on the fly.
 	DontIncludeMapValues bool
+	OptionalTraverse     bool // e.g. .adf?
 }
 
 func splat(d *dataTreeNavigator, context Context, prefs traversePreferences) (Context, error) {
@@ -60,7 +61,7 @@ func traverse(d *dataTreeNavigator, context Context, matchingNode *CandidateNode
 
 	case yaml.SequenceNode:
 		log.Debug("its a sequence of %v things!", len(value.Content))
-		return traverseArray(matchingNode, operation)
+		return traverseArray(matchingNode, operation, operation.Preferences.(traversePreferences))
 
 	case yaml.AliasNode:
 		log.Debug("its an alias!")
@@ -88,15 +89,20 @@ func traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode
 
 	// rhs is a collect expression that will yield indexes to retreive of the arrays
 
-	rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+	rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
+
 	if err != nil {
 		return Context{}, err
 	}
+	prefs := traversePreferences{}
 
+	if expressionNode.Rhs.Rhs != nil && expressionNode.Rhs.Rhs.Operation.Preferences != nil {
+		prefs = expressionNode.Rhs.Rhs.Operation.Preferences.(traversePreferences)
+	}
 	var indicesToTraverse = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node.Content
 
 	//now we traverse the result of the lhs against the indices we found
-	result, err := traverseNodesWithArrayIndices(lhs, indicesToTraverse, traversePreferences{})
+	result, err := traverseNodesWithArrayIndices(lhs, indicesToTraverse, prefs)
 	if err != nil {
 		return Context{}, err
 	}
@@ -121,16 +127,23 @@ func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesT
 	node := matchingNode.Node
 	if node.Tag == "!!null" {
 		log.Debugf("OperatorArrayTraverse got a null - turning it into an empty array")
-		// auto vivification, make it into an empty array
+		// auto vivification
 		node.Tag = ""
 		node.Kind = yaml.SequenceNode
+		//check that the indices are numeric, if not, then we should create an object
+		if len(indicesToTraverse) != 0 {
+			_, err := strconv.ParseInt(indicesToTraverse[0].Value, 10, 64)
+			if err != nil {
+				node.Kind = yaml.MappingNode
+			}
+		}
 	}
 
 	if node.Kind == yaml.AliasNode {
 		matchingNode.Node = node.Alias
 		return traverseArrayIndices(context, matchingNode, indicesToTraverse, prefs)
 	} else if node.Kind == yaml.SequenceNode {
-		return traverseArrayWithIndices(matchingNode, indicesToTraverse)
+		return traverseArrayWithIndices(matchingNode, indicesToTraverse, prefs)
 	} else if node.Kind == yaml.MappingNode {
 		return traverseMapWithIndices(context, matchingNode, indicesToTraverse, prefs)
 	} else if node.Kind == yaml.DocumentNode {
@@ -159,7 +172,7 @@ func traverseMapWithIndices(context Context, candidate *CandidateNode, indices [
 	return matchingNodeMap, nil
 }
 
-func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node) (*list.List, error) {
+func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node, prefs traversePreferences) (*list.List, error) {
 	log.Debug("traverseArrayWithIndices")
 	var newMatches = list.New()
 	node := unwrapDoc(candidate.Node)
@@ -177,6 +190,9 @@ func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node) (*
 	for _, indexNode := range indices {
 		log.Debug("traverseArrayWithIndices: '%v'", indexNode.Value)
 		index, err := strconv.ParseInt(indexNode.Value, 10, 64)
+		if err != nil && prefs.OptionalTraverse {
+			continue
+		}
 		if err != nil {
 			return nil, fmt.Errorf("Cannot index array with '%v' (%v)", indexNode.Value, err)
 		}
@@ -297,8 +313,8 @@ func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *C
 	return nil
 }
 
-func traverseArray(candidate *CandidateNode, operation *Operation) (*list.List, error) {
+func traverseArray(candidate *CandidateNode, operation *Operation, prefs traversePreferences) (*list.List, error) {
 	log.Debug("operation Value %v", operation.Value)
-	indices := []*yaml.Node{&yaml.Node{Value: operation.StringValue}}
-	return traverseArrayWithIndices(candidate, indices)
+	indices := []*yaml.Node{{Value: operation.StringValue}}
+	return traverseArrayWithIndices(candidate, indices, prefs)
 }

pkg/yqlib/operator_traverse_path_test.go

@@ -27,6 +27,37 @@ foobar:
 `
 
 var traversePathOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   `[[1]]`,
+		expression: `.[0][0]`,
+		expected: []string{
+			"D0, P[0 0], (!!int)::1\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `b: cat`,
+		expression: ".b\n",
+		expected: []string{
+			"D0, P[b], (!!str)::cat\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[[[1]]]`,
+		expression: `.[0][0][0]`,
+		expected: []string{
+			"D0, P[0 0 0], (!!int)::1\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `.["cat"] = "thing"`,
+		expected: []string{
+			"D0, P[], ()::cat: thing\n",
+		},
+	},
 	{
 		description: "Simple map navigation",
 		document:    `{a: {b: apple}}`,
@@ -45,6 +76,13 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[1], (!!map)::{c: banana}\n",
 		},
 	},
+	{
+		description:    "Optional Splat",
+		subdescription: "Just like splat, but won't error if you run it against scalars",
+		document:       `"cat"`,
+		expression:     `.[]`,
+		expected:       []string{},
+	},
 	{
 		description:    "Special characters",
 		subdescription: "Use quotes with brackets around path elements with special characters",
@@ -54,6 +92,14 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[{}], (!!str)::frog\n",
 		},
 	},
+	{
+		description: "Nested special characters",
+		document:    `a: {"key.withdots": {"another.key": apple}}`,
+		expression:  `.a["key.withdots"]["another.key"]`,
+		expected: []string{
+			"D0, P[a key.withdots another.key], (!!str)::apple\n",
+		},
+	},
 	{
 		description:    "Keys with spaces",
 		subdescription: "Use quotes with brackets around path elements with special characters",
@@ -71,6 +117,14 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[flying fox], (!!str)::frog\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `c: dog`,
+		expression: `.[.a.b] as $x`,
+		expected: []string{
+			"D0, P[], (doc)::c: dog\n",
+		},
+	},
 	{
 		description:    "Dynamic keys",
 		subdescription: `Expressions within [] can be used to dynamically lookup / calculate keys`,
@@ -97,6 +151,19 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[a b], (!!null)::null\n",
 		},
 	},
+	{
+		description:    "Optional identifier",
+		subdescription: "Like jq, does not output an error when the yaml is not an array or object as expected",
+		document:       `[1,2,3]`,
+		expression:     `.a?`,
+		expected:       []string{},
+	},
+	{
+		skipDoc:    true,
+		document:   `[[1,2,3], {a: frog}]`,
+		expression: `.[] | .["a"]?`,
+		expected:   []string{"D0, P[1 a], (!!str)::frog\n"},
+	},
 	{
 		skipDoc:    true,
 		document:   ``,
@@ -200,6 +267,15 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[0], (!!int)::1\n",
 		},
 	},
+	{
+		description:           "Traversing nested arrays by index",
+		dontFormatInputForDoc: true,
+		document:              `[[], [cat]]`,
+		expression:            `.[1][0]`,
+		expected: []string{
+			"D0, P[1 0], (!!str)::cat\n",
+		},
+	},
 	{
 		description: "Maps with numeric keys",
 		document:    `{2: cat}`,

pkg/yqlib/operator_union_test.go

@@ -5,6 +5,20 @@ import (
 )
 
 var unionOperatorScenarios = []expressionScenario{
+	// {
+	// 	skipDoc:    true,
+	// 	document:   "{}",
+	// 	expression: `(.a, .b.c) as $x`,
+	// 	expected: []string{
+	// 		"D0, P[], (doc)::{}\n",
+	// 	},
+	// },
+	// {
+	// 	skipDoc:    true,
+	// 	document:   "{}",
+	// 	expression: `(.a, .b.c)`,
+	// 	expected:   []string{},
+	// },
 	{
 		description: "Combine scalars",
 		expression:  `1, true, "cat"`,

pkg/yqlib/operator_unique.go

@@ -0,0 +1,64 @@
+package yqlib
+
+import (
+	"container/list"
+	"fmt"
+
+	"github.com/elliotchance/orderedmap"
+	yaml "gopkg.in/yaml.v3"
+)
+
+func unique(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
+	uniqueByExpression := &ExpressionNode{Operation: &Operation{OperationType: uniqueByOpType}, Rhs: selfExpression}
+	return uniqueBy(d, context, uniqueByExpression)
+
+}
+
+func uniqueBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	log.Debugf("-- uniqueBy Operator")
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		candidateNode := unwrapDoc(candidate.Node)
+
+		if candidateNode.Kind != yaml.SequenceNode {
+			return Context{}, fmt.Errorf("Only arrays are supported for unique")
+		}
+
+		var newMatches = orderedmap.NewOrderedMap()
+		for _, node := range candidateNode.Content {
+			child := &CandidateNode{Node: node}
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(child), expressionNode.Rhs)
+
+			if err != nil {
+				return Context{}, err
+			}
+
+			keyValue := "null"
+
+			if rhs.MatchingNodes.Len() > 0 {
+				first := rhs.MatchingNodes.Front()
+				keyCandidate := first.Value.(*CandidateNode)
+				keyValue = keyCandidate.Node.Value
+			}
+
+			_, exists := newMatches.Get(keyValue)
+
+			if !exists {
+				newMatches.Set(keyValue, child.Node)
+			}
+		}
+		resultNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+		for el := newMatches.Front(); el != nil; el = el.Next() {
+			resultNode.Content = append(resultNode.Content, el.Value.(*yaml.Node))
+		}
+
+		results.PushBack(candidate.CreateChild(nil, resultNode))
+	}
+
+	return context.ChildContext(results), nil
+
+}

pkg/yqlib/operator_unique_test.go

@@ -0,0 +1,65 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var uniqueOperatorScenarios = []expressionScenario{
+	{
+		description: "Unique array of scalars (string/numbers)",
+		document:    `[1,2,3,2]`,
+		expression:  `unique`,
+		expected: []string{
+			"D0, P[], (!!seq)::- 1\n- 2\n- 3\n",
+		},
+	},
+	{
+		description:    "Unique nulls",
+		subdescription: "Unique works on the node value, so it considers different representations of nulls to be different",
+		document:       `[~,null, ~, null]`,
+		expression:     `unique`,
+		expected: []string{
+			"D0, P[], (!!seq)::- ~\n- null\n",
+		},
+	},
+	{
+		description:    "Unique all nulls",
+		subdescription: "Run against the node tag to unique all the nulls",
+		document:       `[~,null, ~, null]`,
+		expression:     `unique_by(tag)`,
+		expected: []string{
+			"D0, P[], (!!seq)::- ~\n",
+		},
+	},
+	{
+		description: "Unique array object fields",
+		document:    `[{name: harry, pet: cat}, {name: billy, pet: dog}, {name: harry, pet: dog}]`,
+		expression:  `unique_by(.name)`,
+		expected: []string{
+			"D0, P[], (!!seq)::- {name: harry, pet: cat}\n- {name: billy, pet: dog}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{name: harry, pet: cat}, {pet: fish}, {name: harry, pet: dog}]`,
+		expression: `unique_by(.name)`,
+		expected: []string{
+			"D0, P[], (!!seq)::- {name: harry, pet: cat}\n- {pet: fish}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{name: harry, pet: cat}, {pet: fish}, {name: harry, pet: dog}]`,
+		expression: `unique_by(.cat.dog)`,
+		expected: []string{
+			"D0, P[], (!!seq)::- {name: harry, pet: cat}\n",
+		},
+	},
+}
+
+func TestUniqueOperatorScenarios(t *testing.T) {
+	for _, tt := range uniqueOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "Unique", uniqueOperatorScenarios)
+}

pkg/yqlib/operator_variables.go

@@ -16,7 +16,7 @@ func getVariableOperator(d *dataTreeNavigator, context Context, expressionNode *
 }
 
 func assignVariableOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	lhs, err := d.GetMatchingNodes(context, expressionNode.Lhs)
+	lhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Lhs)
 	if err != nil {
 		return Context{}, nil
 	}

pkg/yqlib/operator_variables_test.go

@@ -5,6 +5,14 @@ import (
 )
 
 var variableOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   `{}`,
+		expression: `.a.b as $foo`,
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
 	{
 		description: "Single value variable",
 		document:    `a: cat`,

pkg/yqlib/operators.go

@@ -10,6 +10,32 @@ import (
 
 type operatorHandler func(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error)
 
+type compoundCalculation func(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode
+
+func compoundAssignFunction(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation compoundCalculation) (Context, error) {
+	lhs, err := d.GetMatchingNodes(context, expressionNode.Lhs)
+	if err != nil {
+		return Context{}, err
+	}
+
+	assignmentOp := &Operation{OperationType: assignOpType}
+	valueOp := &Operation{OperationType: valueOpType}
+
+	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		valueOp.CandidateNode = candidate
+		valueExpression := &ExpressionNode{Operation: valueOp}
+
+		assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: valueExpression, Rhs: calculation(valueExpression, expressionNode.Rhs)}
+
+		_, err = d.GetMatchingNodes(context, assignmentOpNode)
+		if err != nil {
+			return Context{}, err
+		}
+	}
+	return context, nil
+}
+
 func unwrapDoc(node *yaml.Node) *yaml.Node {
 	if node.Kind == yaml.DocumentNode {
 		return node.Content[0]
@@ -24,7 +50,19 @@ func emptyOperator(d *dataTreeNavigator, context Context, expressionNode *Expres
 
 type crossFunctionCalculation func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error)
 
-func resultsForRhs(d *dataTreeNavigator, context Context, lhsCandidate *CandidateNode, rhs Context, calculation crossFunctionCalculation, results *list.List) error {
+func resultsForRhs(d *dataTreeNavigator, context Context, lhsCandidate *CandidateNode, rhs Context, calculation crossFunctionCalculation, results *list.List, calcWhenEmpty bool) error {
+
+	if calcWhenEmpty && rhs.MatchingNodes.Len() == 0 {
+		resultCandidate, err := calculation(d, context, lhsCandidate, nil)
+		if err != nil {
+			return err
+		}
+		if resultCandidate != nil {
+			results.PushBack(resultCandidate)
+		}
+		return nil
+	}
+
 	for rightEl := rhs.MatchingNodes.Front(); rightEl != nil; rightEl = rightEl.Next() {
 		log.Debugf("Applying calc")
 		rhsCandidate := rightEl.Value.(*CandidateNode)
@@ -32,7 +70,9 @@ func resultsForRhs(d *dataTreeNavigator, context Context, lhsCandidate *Candidat
 		if err != nil {
 			return err
 		}
-		results.PushBack(resultCandidate)
+		if resultCandidate != nil {
+			results.PushBack(resultCandidate)
+		}
 	}
 	return nil
 }
@@ -52,14 +92,7 @@ func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *Expressi
 	}
 
 	if calcWhenEmpty && lhs.MatchingNodes.Len() == 0 {
-		if rhs.MatchingNodes.Len() == 0 {
-			resultCandidate, err := calculation(d, context, nil, nil)
-			if err != nil {
-				return Context{}, err
-			}
-			results.PushBack(resultCandidate)
-		}
-		err := resultsForRhs(d, context, nil, rhs, calculation, results)
+		err := resultsForRhs(d, context, nil, rhs, calculation, results, calcWhenEmpty)
 		if err != nil {
 			return Context{}, err
 		}
@@ -68,7 +101,7 @@ func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *Expressi
 	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
 		lhsCandidate := el.Value.(*CandidateNode)
 
-		err := resultsForRhs(d, context, lhsCandidate, rhs, calculation, results)
+		err := resultsForRhs(d, context, lhsCandidate, rhs, calculation, results, calcWhenEmpty)
 		if err != nil {
 			return Context{}, err
 		}
@@ -88,9 +121,12 @@ func crossFunction(d *dataTreeNavigator, context Context, expressionNode *Expres
 		}
 	}
 	if evaluateAllTogether {
+		log.Debug("crossFunction evaluateAllTogether!")
 		return doCrossFunc(d, context, expressionNode, calculation, calcWhenEmpty)
 	}
 
+	log.Debug("crossFunction evaluate apart!")
+
 	for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {
 		innerResults, err := doCrossFunc(d, context.SingleChildContext(matchEl.Value.(*CandidateNode)), expressionNode, calculation, calcWhenEmpty)
 		if err != nil {

pkg/yqlib/operators_test.go

@@ -71,7 +71,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
 		t.Error(fmt.Errorf("%v: %v", err, s.expression))
 		return
 	}
-	test.AssertResultComplexWithContext(t, s.expected, resultsToString(context.MatchingNodes), fmt.Sprintf("exp: %v\ndoc: %v", s.expression, s.document))
+	test.AssertResultComplexWithContext(t, s.expected, resultsToString(context.MatchingNodes), fmt.Sprintf("desc: %v\nexp: %v\ndoc: %v", s.description, s.expression, s.document))
 }
 
 func resultsToString(results *list.List) []string {
@@ -114,7 +114,7 @@ func formatYaml(yaml string, filename string) string {
 		panic(err)
 	}
 	streamEvaluator := NewStreamEvaluator()
-	err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer)
+	_, err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer, "")
 	if err != nil {
 		panic(err)
 	}

pkg/yqlib/printer.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"container/list"
 	"io"
+	"strings"
 
 	yaml "gopkg.in/yaml.v3"
 )
@@ -11,6 +12,8 @@ import (
 type Printer interface {
 	PrintResults(matchingNodes *list.List) error
 	PrintedAnything() bool
+	//e.g. when given a front-matter doc, like jekyll
+	SetAppendix(reader io.Reader)
 }
 
 type resultsPrinter struct {
@@ -25,6 +28,8 @@ type resultsPrinter struct {
 	previousFileIndex  int
 	printedMatches     bool
 	treeNavigator      DataTreeNavigator
+	preambleReader     io.Reader
+	appendixReader     io.Reader
 }
 
 func NewPrinter(writer io.Writer, outputToJSON bool, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
@@ -40,6 +45,14 @@ func NewPrinter(writer io.Writer, outputToJSON bool, unwrapScalar bool, colorsEn
 	}
 }
 
+func (p *resultsPrinter) SetPreamble(reader io.Reader) {
+	p.preambleReader = reader
+}
+
+func (p *resultsPrinter) SetAppendix(reader io.Reader) {
+	p.appendixReader = reader
+}
+
 func (p *resultsPrinter) PrintedAnything() bool {
 	return p.printedMatches
 }
@@ -101,13 +114,49 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 	for el := matchingNodes.Front(); el != nil; el = el.Next() {
 		mappedDoc := el.Value.(*CandidateNode)
 		log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v, printDocSeparators: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document, p.printDocSeparators)
-		if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators {
+
+		commentStartsWithSeparator := strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$\n$yqDocSeperator$")
+
+		if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators && !commentStartsWithSeparator {
 			log.Debug("-- writing doc sep")
 			if err := p.writeString(bufferedWriter, "---\n"); err != nil {
 				return err
 			}
 		}
 
+		if strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$") {
+			log.Debug("headcommentwas %v", mappedDoc.Node.HeadComment)
+			log.Debug("finished headcomment")
+			reader := bufio.NewReader(strings.NewReader(mappedDoc.Node.HeadComment))
+			mappedDoc.Node.HeadComment = ""
+
+			for {
+
+				readline, errReading := reader.ReadString('\n')
+				if errReading != nil && errReading != io.EOF {
+					return errReading
+				}
+				if strings.Contains(readline, "$yqLeadingContent$") {
+					// skip this
+
+				} else if strings.Contains(readline, "$yqDocSeperator$") {
+					if p.printDocSeparators {
+						if err := p.writeString(bufferedWriter, "---\n"); err != nil {
+							return err
+						}
+					}
+				} else if !p.outputToJSON {
+					if err := p.writeString(bufferedWriter, readline); err != nil {
+						return err
+					}
+				}
+
+				if errReading == io.EOF {
+					break
+				}
+			}
+
+		}
 		if err := p.printNode(mappedDoc.Node, bufferedWriter); err != nil {
 			return err
 		}
@@ -115,5 +164,14 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 		p.previousDocIndex = mappedDoc.Document
 	}
 
+	if p.appendixReader != nil && !p.outputToJSON {
+		log.Debug("Piping appendix reader...")
+		betterReader := bufio.NewReader(p.appendixReader)
+		_, err := io.Copy(bufferedWriter, betterReader)
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }

pkg/yqlib/printer_test.go

@@ -17,6 +17,16 @@ a: apple
 a: coconut
 `
 
+var multiDocSampleLeadingExpected = `# go cats
+---
+a: banana
+---
+a: apple
+---
+# cool
+a: coconut
+`
+
 func nodeToList(candidate *CandidateNode) *list.List {
 	elMap := list.New()
 	elMap.PushBack(candidate)
@@ -61,6 +71,48 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) {
 	test.AssertResult(t, multiDocSample, output.String())
 }
 
+func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
+	var output bytes.Buffer
+	var writer = bufio.NewWriter(&output)
+	printer := NewPrinter(writer, false, true, false, 2, true)
+
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
+	if err != nil {
+		panic(err)
+	}
+
+	el := inputs.Front()
+	el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
+	sample1 := nodeToList(el.Value.(*CandidateNode))
+
+	el = el.Next()
+	el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
+	sample2 := nodeToList(el.Value.(*CandidateNode))
+
+	el = el.Next()
+	el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n# cool\n"
+	sample3 := nodeToList(el.Value.(*CandidateNode))
+
+	err = printer.PrintResults(sample1)
+	if err != nil {
+		panic(err)
+	}
+
+	err = printer.PrintResults(sample2)
+	if err != nil {
+		panic(err)
+	}
+
+	err = printer.PrintResults(sample3)
+	if err != nil {
+		panic(err)
+	}
+
+	writer.Flush()
+
+	test.AssertResult(t, multiDocSampleLeadingExpected, output.String())
+}
+
 func TestPrinterMultipleFilesInSequence(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
@@ -108,6 +160,56 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) {
 	test.AssertResult(t, multiDocSample, output.String())
 }
 
+func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
+	var output bytes.Buffer
+	var writer = bufio.NewWriter(&output)
+	printer := NewPrinter(writer, false, true, false, 2, true)
+
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
+	if err != nil {
+		panic(err)
+	}
+
+	el := inputs.Front()
+	elNode := el.Value.(*CandidateNode)
+	elNode.Document = 0
+	elNode.FileIndex = 0
+	elNode.Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
+	sample1 := nodeToList(elNode)
+
+	el = el.Next()
+	elNode = el.Value.(*CandidateNode)
+	elNode.Document = 0
+	elNode.FileIndex = 1
+	elNode.Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
+	sample2 := nodeToList(elNode)
+
+	el = el.Next()
+	elNode = el.Value.(*CandidateNode)
+	elNode.Document = 0
+	elNode.FileIndex = 2
+	elNode.Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n# cool\n"
+	sample3 := nodeToList(elNode)
+
+	err = printer.PrintResults(sample1)
+	if err != nil {
+		panic(err)
+	}
+
+	err = printer.PrintResults(sample2)
+	if err != nil {
+		panic(err)
+	}
+
+	err = printer.PrintResults(sample3)
+	if err != nil {
+		panic(err)
+	}
+
+	writer.Flush()
+	test.AssertResult(t, multiDocSampleLeadingExpected, output.String())
+}
+
 func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
@@ -127,6 +229,86 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 	test.AssertResult(t, multiDocSample, output.String())
 }
 
+func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
+	var output bytes.Buffer
+	var writer = bufio.NewWriter(&output)
+	printer := NewPrinter(writer, false, true, false, 2, true)
+
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
+	if err != nil {
+		panic(err)
+	}
+
+	inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
+
+	err = printer.PrintResults(inputs)
+	if err != nil {
+		panic(err)
+	}
+
+	writer.Flush()
+	expected := `# go cats
+---
+a: banana
+---
+a: apple
+---
+a: coconut
+`
+	test.AssertResult(t, expected, output.String())
+}
+
+func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
+	var output bytes.Buffer
+	var writer = bufio.NewWriter(&output)
+	printer := NewPrinter(writer, false, true, false, 2, true)
+
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
+	if err != nil {
+		panic(err)
+	}
+	inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
+	err = printer.PrintResults(inputs)
+	if err != nil {
+		panic(err)
+	}
+
+	writer.Flush()
+	expected := `---
+a: banana
+---
+a: apple
+---
+a: coconut
+`
+	test.AssertResult(t, expected, output.String())
+}
+
+func TestPrinterScalarWithLeadingCont(t *testing.T) {
+	var output bytes.Buffer
+	var writer = bufio.NewWriter(&output)
+	printer := NewPrinter(writer, false, true, false, 2, true)
+
+	node, err := NewExpressionParser().ParseExpression(".a")
+	if err != nil {
+		panic(err)
+	}
+	streamEvaluator := NewStreamEvaluator()
+	_, err = streamEvaluator.Evaluate("sample", strings.NewReader(multiDocSample), node, printer, "# blah\n")
+	if err != nil {
+		panic(err)
+	}
+
+	writer.Flush()
+	expected := `banana
+---
+apple
+---
+coconut
+`
+	test.AssertResult(t, expected, output.String())
+}
+
 func TestPrinterMultipleDocsJson(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
@@ -139,6 +321,8 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 		panic(err)
 	}
 
+	inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# ignore this\n"
+
 	err = printer.PrintResults(inputs)
 	if err != nil {
 		panic(err)

pkg/yqlib/stream_evaluator.go

@@ -12,9 +12,9 @@ import (
 // Uses less memory than loading all documents and running the expression once, but this cannot process
 // cross document expressions.
 type StreamEvaluator interface {
-	Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer) error
-	EvaluateFiles(expression string, filenames []string, printer Printer) error
-	EvaluateNew(expression string, printer Printer) error
+	Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, leadingContent string) (uint, error)
+	EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error
+	EvaluateNew(expression string, printer Printer, leadingContent string) error
 }
 
 type streamEvaluator struct {
@@ -27,7 +27,7 @@ func NewStreamEvaluator() StreamEvaluator {
 	return &streamEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: NewExpressionParser()}
 }
 
-func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error {
+func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadingContent string) error {
 	node, err := s.treeCreator.ParseExpression(expression)
 	if err != nil {
 		return err
@@ -35,7 +35,7 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error
 	candidateNode := &CandidateNode{
 		Document:  0,
 		Filename:  "",
-		Node:      &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode},
+		Node:      &yaml.Node{Kind: yaml.DocumentNode, HeadComment: leadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
 		FileIndex: 0,
 	}
 	inputList := list.New()
@@ -48,35 +48,47 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error
 	return printer.PrintResults(result.MatchingNodes)
 }
 
-func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error {
-
+func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error {
+	var totalProcessDocs uint = 0
 	node, err := s.treeCreator.ParseExpression(expression)
 	if err != nil {
 		return err
 	}
 
-	for _, filename := range filenames {
-		reader, err := readStream(filename)
+	var firstFileLeadingContent string
+
+	for index, filename := range filenames {
+		reader, leadingContent, err := readStream(filename, leadingContentPreProcessing)
+
+		if index == 0 {
+			firstFileLeadingContent = leadingContent
+		}
+
 		if err != nil {
 			return err
 		}
-		err = s.Evaluate(filename, reader, node, printer)
+		processedDocs, err := s.Evaluate(filename, reader, node, printer, leadingContent)
 		if err != nil {
 			return err
 		}
+		totalProcessDocs = totalProcessDocs + processedDocs
 
 		switch reader := reader.(type) {
 		case *os.File:
 			safelyCloseFile(reader)
 		}
 	}
+
+	if totalProcessDocs == 0 {
+		return s.EvaluateNew(expression, printer, firstFileLeadingContent)
+	}
+
 	return nil
 }
 
-func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer) error {
+func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, leadingContent string) (uint, error) {
 
 	var currentIndex uint
-
 	decoder := yaml.NewDecoder(reader)
 	for {
 		var dataBucket yaml.Node
@@ -84,9 +96,12 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr
 
 		if errorReading == io.EOF {
 			s.fileIndex = s.fileIndex + 1
-			return nil
+			return currentIndex, nil
 		} else if errorReading != nil {
-			return errorReading
+			return currentIndex, errorReading
+		}
+		if currentIndex == 0 {
+			dataBucket.HeadComment = leadingContent
 		}
 		candidateNode := &CandidateNode{
 			Document:  currentIndex,
@@ -99,11 +114,12 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr
 
 		result, errorParsing := s.treeNavigator.GetMatchingNodes(Context{MatchingNodes: inputList}, node)
 		if errorParsing != nil {
-			return errorParsing
+			return currentIndex, errorParsing
 		}
 		err := printer.PrintResults(result.MatchingNodes)
+
 		if err != nil {
-			return err
+			return currentIndex, err
 		}
 		currentIndex = currentIndex + 1
 	}