Adding subtraction support for arrays

This reverts commit ffb6cbf769.

.dockerignore

@@ -1 +1 @@
-bin
+bin/*

.github/workflows/test-yq.yml

@@ -0,0 +1,19 @@
+name: Test Yq Action
+on: [push]
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get test
+        uses: mikefarah/yq@master
+        with:
+          cmd: yq eval '.a' examples/data1.yaml
+      - name: Write inplace test
+        id: lookupSdkVersion
+        uses: mikefarah/yq@master
+        with:
+          cmd: yq eval -i '.a.b = 5' examples/data1.yaml

.gitignore

@@ -38,3 +38,5 @@ parts/
 prime/
 .snapcraft/
 yq*.snap
+test.yml
+test2.yml

Dockerfile

@@ -12,7 +12,12 @@ RUN CGO_ENABLED=0 make local build
 
 # Choose alpine as a base image to make this useful for CI, as many
 # CI tools expect an interactive shell inside the container
-FROM alpine:3.12.3 as production
+FROM alpine:3.13.5 as production
+
+RUN mkdir /home/yq/
+RUN addgroup -g 1000 yq && \
+    adduser -u 1000 -G yq -s /bin/bash -h /home/yq -D yq
+RUN chown -R yq:yq /home/yq/
 
 COPY --from=builder /go/src/mikefarah/yq/yq /usr/bin/yq
 RUN chmod +x /usr/bin/yq
@@ -20,6 +25,8 @@ RUN chmod +x /usr/bin/yq
 ARG VERSION=none
 LABEL version=${VERSION}
 
+USER yq
+
 WORKDIR /workdir
 
 ENTRYPOINT ["/usr/bin/yq"]

Dockerfile.dev

@@ -6,26 +6,5 @@ RUN set -e -x \
     && /opt/devtools.sh
 ENV PATH=/go/bin:$PATH
 
-# install mkdocs
-RUN set -ex \
-    && buildDeps=' \
-        build-essential \
-        python3-dev \
-    ' \
-    && apt-get update && apt-get install -y --no-install-recommends \
-        $buildDeps \
-        python3 \
-        python3-setuptools \
-        python3-wheel \
-        python3-pip \
-    && pip3 install --upgrade \
-        pip \
-        'Markdown>=2.6.9' \
-        'mkdocs>=0.16.3' \
-        'mkdocs-material>=1.10.1' \
-        'markdown-include>=0.5.1' \
-    && apt-get purge -y --auto-remove $buildDeps \
-    && rm -rf /var/lib/apt/lists/*
-
 ENV CGO_ENABLED 0
 ENV GOPATH /go:/yq

Makefile

@@ -17,6 +17,7 @@ help:
 	@echo '    make vendor          Install dependencies to vendor directory.'
 	@echo '    make format          Run code formatter.'
 	@echo '    make check           Run static code analysis (lint).'
+	@echo '    make secure          Run gosec.'
 	@echo '    make test            Run tests on project.'
 	@echo '    make cover           Run tests and capture code coverage metrics on project.'
 	@echo '    make clean           Clean the directory tree of produced artifacts.'
@@ -84,6 +85,10 @@ format: vendor
 check: format
 	${DOCKRUN} bash ./scripts/check.sh
 
+.PHONY: secure
+secure: 
+	${DOCKRUN} bash ./scripts/secure.sh
+
 .PHONY: test
 test: check
 	${DOCKRUN} bash ./scripts/test.sh
@@ -96,11 +101,6 @@ cover: check
 	@find cover -type d -exec chmod 755 {} \; || :
 	@find cover -type f -exec chmod 644 {} \; || :
 
-.PHONY: build-docs
-build-docs: prepare mkdocs.yml mkdocs/*
-	${DOCKRUN} mkdocs build
-	@find docs -type d -exec chmod 755 {} \; || :
-	@find docs -type f -exec chmod 644 {} \; || :
 
 .PHONY: release
 release: xcompile

README.md

@@ -7,12 +7,39 @@ a lightweight and portable command-line YAML processor. `yq` uses [jq](https://g
 
 yq is written in go - so you can download a dependency free binary for your platform and you are good to go! If you prefer there are a variety of package managers that can be used as well as docker, all listed below.
 
-## V4 released!
-V4 is now officially released, it's quite different from V3 (sorry for the migration), however it is much more similar to ```jq```, using a similar expression syntax and therefore support much more complex functionality! 
+## Quick Usage Guide
 
-If you've been using v3 and want/need to upgrade, checkout the [upgrade guide](https://mikefarah.gitbook.io/yq/v/v4.x/upgrading-from-v3).
+Read a value:
 
-Support for v3 will cease August 2021, until then, critical bug and security fixes will still get applied if required.
+```bash
+yq e '.a.b[0].c' file.yaml
+```
+
+Update a yaml file, inplace
+```bash
+yq e -i '.a.b[0].c = "cool"' file.yaml
+```
+
+Update using environment variables
+```bash
+NAME=mike yq e -i '.a.b[0].c = strenv(NAME)' file.yaml
+```
+
+Merge multiple files
+```
+yq ea '. as $item ireduce ({}; . * $item )' path/to/*.yml
+```
+
+Multiple updates to a yaml file
+```bash
+yq e -i '
+  .a.b[0].c = "cool" |
+  .x.y.z = "foobar" |
+  .person.name = strenv(NAME)
+' file.yaml
+```
+
+See the [documentation](https://mikefarah.gitbook.io/yq/) for more.
 
 ## Install
 
@@ -87,6 +114,14 @@ rm /etc/myfile.tmp
 docker run --rm -v "${PWD}":/workdir mikefarah/yq <command> [flags] [expression ]FILE...
 ```
 
+#### Pipe in via STDIN:
+
+You'll need to pass the `-i\--interactive` flag to docker:
+
+```bash
+cat myfile.yml | docker run -i --rm mikefarah/yq e . -
+```
+
 #### Run commands interactively:
 
 ```bash
@@ -101,6 +136,36 @@ yq() {
 }
 ```
 
+
+#### Running as root:
+
+`yq`'s docker image no longer runs under root (https://github.com/mikefarah/yq/pull/860). If you'd like to install more things in the docker image, or you're having permissions issues when attempting to read/write files you'll need to either:
+
+
+```
+docker run --user="root" -it --entrypoint sh mikefarah/yq
+```
+
+Or, in your docker file:
+
+```
+FROM mikefarah/yq
+
+USER root
+RUN apk add bash
+USER yq
+```
+
+### GitHub Action
+```
+  - name: Set foobar to cool
+    uses: mikefarah/yq@master
+    with:
+      cmd: yq eval -i '.foo.bar = "cool"' 'config.yml'
+```
+
+See https://mikefarah.gitbook.io/yq/usage/github-action for more.
+
 ### Go Get:
 ```
 GO111MODULE=on go get github.com/mikefarah/yq/v4
@@ -119,6 +184,8 @@ See [webi](https://webinstall.dev/)
 Supported by @adithyasunil26 (https://github.com/webinstall/webi-installers/tree/master/yq)
 
 ### Windows:
+[![Chocolatey](https://img.shields.io/chocolatey/v/yq.svg)](https://chocolatey.org/packages/yq)
+[![Chocolatey](https://img.shields.io/chocolatey/dt/yq.svg)](https://chocolatey.org/packages/yq)
 ```
 choco install yq
 ```
@@ -151,19 +218,24 @@ sudo apt install yq -y
 Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
 
 ## Features
+- [Detailed documentation with many examples](https://mikefarah.gitbook.io/yq/)
 - Written in portable go, so you can download a lovely dependency free binary
 - Uses similar syntax as `jq` but works with YAML and JSON files
 - Fully supports multi document yaml files
+- Supports yaml [front matter](https://mikefarah.gitbook.io/yq/usage/front-matter) blocks (e.g. jekyll/assemble)
 - Colorized yaml output
-- [Deeply traverse yaml](https://mikefarah.gitbook.io/yq/v/v4.x/traverse)
-- [Sort yaml by keys](https://mikefarah.gitbook.io/yq/v/v4.x/sort-keys)
-- Manipulate yaml [comments](https://mikefarah.gitbook.io/yq/comment-operators), [styling](https://mikefarah.gitbook.io/yq/style), [tags](https://mikefarah.gitbook.io/yq/tag) and [anchors and aliases](https://mikefarah.gitbook.io/yq/anchor-and-alias-operators).
+- [Deeply traverse yaml](https://mikefarah.gitbook.io/yq/operators/traverse-read)
+- [Sort yaml by keys](https://mikefarah.gitbook.io/yq/operators/sort-keys)
+- Manipulate yaml [comments](https://mikefarah.gitbook.io/yq/operators/comment-operators), [styling](https://mikefarah.gitbook.io/yq/operators/style), [tags](https://mikefarah.gitbook.io/yq/operators/tag) and [anchors and aliases](https://mikefarah.gitbook.io/yq/operators/anchor-and-alias-operators).
 - [Update yaml inplace](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags)
-- [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/v/v4.x/select#select-and-update-matching-values-in-map)
+- [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/operators/select#select-and-update-matching-values-in-map)
 - Keeps yaml formatting and comments when updating (though there are issues with whitespace)
-- [Convert to/from json to yaml](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
+- [Convert to/from json](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
+- [Convert to properties](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties)
 - [Pipe data in by using '-'](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate)
 - [General shell completion scripts (bash/zsh/fish/powershell)](https://mikefarah.gitbook.io/yq/v/v4.x/commands/shell-completion)
+- [Reduce](https://mikefarah.gitbook.io/yq/operators/reduce) to merge multiple files or sum an array or other fancy things.
+- [Github Action](https://mikefarah.gitbook.io/yq/usage/github-action) to use in your automated pipeline (thanks @devorbitus)
 
 ## [Usage](https://mikefarah.gitbook.io/yq/)
 
@@ -175,33 +247,31 @@ Usage:
   yq [command]
 
 Available Commands:
-  eval             Apply expression to each document in each yaml file given in sequence
+  eval             Apply the expression to each document in each yaml file in sequence
   eval-all         Loads _all_ yaml documents of _all_ yaml files and runs expression once
   help             Help about any command
   shell-completion Generate completion script
 
 Flags:
-  -C, --colors        force print with colors
-  -e, --exit-status   set exit status if there are no matches or null or false is returned
-  -h, --help          help for yq
-  -I, --indent int    sets indent level for output (default 2)
-  -i, --inplace       update the yaml file inplace of first yaml file given.
-  -M, --no-colors     force print with no colors
-  -N, --no-doc        Don't print document separators (---)
-  -n, --null-input    Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.
-  -P, --prettyPrint    pretty print, shorthand for '... style = ""' 
-  -j, --tojson        output as json. Set indent to 0 to print json in one line.
-  -v, --verbose       verbose mode
-  -V, --version       Print version information and quit
+  -C, --colors                force print with colors
+  -e, --exit-status           set exit status if there are no matches or null or false is returned
+  -f, --front-matter string   (extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact
+  -h, --help                  help for yq
+  -I, --indent int            sets indent level for output (default 2)
+  -i, --inplace               update the yaml file inplace of first yaml file given.
+  -M, --no-colors             force print with no colors
+  -N, --no-doc                Don't print document separators (---)
+  -n, --null-input            Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.
+  -P, --prettyPrint           pretty print, shorthand for '... style = ""'
+  -j, --tojson                output as json. Set indent to 0 to print json in one line.
+      --unwrapScalar          unwrap scalar, print the value with no quotes, colors or comments (default true)
+  -v, --verbose               verbose mode
+  -V, --version               Print version information and quit
 
 Use "yq [command] --help" for more information about a command.
 ```
-
-Simple Example:
-
-```bash
-yq e '.a.b | length' f1.yml f2.yml 
-```
-
 ## Known Issues / Missing Features
 - `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details)
+- Powershell has its own...opinions: https://mikefarah.gitbook.io/yq/usage/tips-and-tricks#quotes-in-windows-powershell
+
+See [tips and tricks](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks) for more common problems and solutions.

acceptance_tests/basic.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+setUp() {
+  rm -f test.yml
+}
+
+testBasicEvalRoundTrip() {
+  ./yq e -n ".a = 123" > test.yml
+  X=$(./yq e '.a' test.yml)
+  assertEquals 123 "$X"
+}
+
+testBasicUpdateInPlaceSequence() {
+  cat >test.yml <<EOL
+a: 0
+EOL
+  ./yq e -i ".a = 10" test.yml
+  X=$(./yq e '.a' test.yml)
+  assertEquals "10" "$X"
+}
+
+testBasicUpdateInPlaceSequenceEvalAll() {
+  cat >test.yml <<EOL
+a: 0
+EOL
+  ./yq ea -i ".a = 10" test.yml
+  X=$(./yq e '.a' test.yml)
+  assertEquals "10" "$X"
+}
+
+testBasicNoExitStatus() {
+  echo "a: cat" > test.yml
+  X=$(./yq e '.z' test.yml)
+  assertEquals "null" "$X"
+}
+
+testBasicExitStatus() {
+  echo "a: cat" > test.yml
+  X=$(./yq e -e '.z' test.yml 2&>/dev/null)
+  assertEquals 1 "$?"
+}
+
+testBasicExtractFieldWithSeperator() {
+    cat >test.yml <<EOL
+---
+name: chart-name
+version: 1.2.3
+EOL
+  X=$(./yq e '.name' test.yml)
+  assertEquals "chart-name" "$X"
+}
+
+testBasicExtractMultipleFieldWithSeperator() {
+    cat >test.yml <<EOL
+---
+name: chart-name
+version: 1.2.3
+---
+name: thing
+version: 1.2.3
+EOL
+
+read -r -d '' expected << EOM
+chart-name
+---
+thing
+EOM
+  X=$(./yq e '.name' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+source ./scripts/shunit2

acceptance_tests/empty.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+setUp() {
+  cat >test.yml <<EOL
+# comment
+EOL
+}
+
+testEmptyEval() {
+  X=$(./yq e test.yml)
+  expected=$(cat test.yml)
+  assertEquals 0 $?
+  assertEquals "$expected" "$X"
+}
+
+testEmptyEvalNoNewLine() {
+  echo -n "#comment" >test.yml
+  X=$(./yq e test.yml)
+  expected=$(cat test.yml)
+  assertEquals 0 $?
+  assertEquals "$expected" "$X"
+}
+
+testEmptyEvalNoNewLineWithExpression() {
+  echo -n "# comment" >test.yml
+  X=$(./yq e '.apple = "tree"' test.yml)
+  read -r -d '' expected << EOM
+# comment
+apple: tree
+EOM
+  assertEquals "$expected" "$X"
+}
+
+testEmptyEvalPipe() {
+  X=$(./yq e - < test.yml)
+  assertEquals 0 $?
+}
+
+testEmptyCommentsWithExpressionEval() {
+  read -r -d '' expected << EOM
+# comment
+apple: tree
+EOM
+
+  X=$(./yq e '.apple="tree"' test.yml)
+
+  assertEquals "$expected" "$X"
+}
+
+testEmptyCommentsWithExpressionEvalAll() {
+  read -r -d '' expected << EOM
+# comment
+apple: tree
+EOM
+
+  X=$(./yq ea '.apple="tree"' test.yml)
+
+  assertEquals "$expected" "$X"
+}
+
+testEmptyWithExpressionEval() {
+  rm test.yml
+  touch test.yml
+  expected="apple: tree"
+
+  X=$(./yq e '.apple="tree"' test.yml)
+
+  assertEquals "$expected" "$X"
+}
+
+testEmptyWithExpressionEvalAll() {
+  rm test.yml
+  touch test.yml
+  expected="apple: tree"
+
+  X=$(./yq ea '.apple="tree"' test.yml)
+
+  assertEquals "$expected" "$X"
+}
+
+
+source ./scripts/shunit2

acceptance_tests/front-matter.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+setUp() {
+  cat >test.yml <<EOL
+---
+a: apple
+b: cat
+---
+not yaml
+c: at
+EOL
+}
+
+testFrontMatterProcessEval() {
+  read -r -d '' expected << EOM
+---
+a: apple
+b: dog
+---
+not yaml
+c: at
+EOM
+  ./yq e --front-matter="process" '.b = "dog"' test.yml -i
+  assertEquals "$expected" "$(cat test.yml)"
+}
+
+testFrontMatterProcessEvalAll() {
+  read -r -d '' expected << EOM
+---
+a: apple
+b: dog
+---
+not yaml
+c: at
+EOM
+  ./yq ea --front-matter="process" '.b = "dog"' test.yml -i
+  assertEquals "$expected" "$(cat test.yml)"
+}
+
+testFrontMatterExtractEval() {
+    cat >test.yml <<EOL
+a: apple
+b: cat
+---
+not yaml
+c: at
+EOL
+
+  read -r -d '' expected << EOM
+a: apple
+b: dog
+EOM
+  ./yq e --front-matter="extract" '.b = "dog"' test.yml -i
+  assertEquals "$expected" "$(cat test.yml)"
+}
+
+testFrontMatterExtractEvalAll() {
+  cat >test.yml <<EOL
+a: apple
+b: cat
+---
+not yaml
+c: at
+EOL
+
+  read -r -d '' expected << EOM
+a: apple
+b: dog
+EOM
+  ./yq ea --front-matter="extract" '.b = "dog"' test.yml -i
+  assertEquals "$expected" "$(cat test.yml)"
+}
+
+
+source ./scripts/shunit2

acceptance_tests/leading-seperator.sh

@@ -0,0 +1,449 @@
+#!/bin/bash
+
+setUp() {
+  cat >test.yml <<EOL
+---
+a: test
+EOL
+}
+
+testLeadingSeperatorWithDoc() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+---
+a: test
+---
+b: cool
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+---
+a: thing
+---
+b: cool
+EOM
+
+  X=$(./yq e '(select(di == 0) | .a) = "thing"' - < test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorPipeIntoEvalSeq() {
+  X=$(./yq e - < test.yml)
+  expected=$(cat test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorExtractField() {
+  X=$(./yq e '.a' - < test.yml)
+  assertEquals "test" "$X"
+}
+
+testLeadingSeperatorExtractFieldWithCommentsAfterSep() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+EOL
+  X=$(./yq e '.a' test.yml)
+  assertEquals "test" "$X"
+}
+
+testLeadingSeperatorExtractFieldWithCommentsBeforeSep() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+---
+a: test
+EOL
+  X=$(./yq e '.a' test.yml)
+  assertEquals "test" "$X"
+}
+
+
+testLeadingSeperatorExtractFieldMultiDoc() {
+  cat >test.yml <<EOL
+---
+a: test
+---
+a: test2
+EOL
+
+  read -r -d '' expected << EOM
+test
+---
+test2
+EOM
+  X=$(./yq e '.a' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorExtractFieldMultiDocWithComments() {
+  cat >test.yml <<EOL
+# here
+---
+# there
+a: test
+# whereever
+---
+# you are
+a: test2
+# woop
+EOL
+
+  read -r -d '' expected << EOM
+test
+---
+test2
+EOM
+  X=$(./yq e '.a' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+
+testLeadingSeperatorEvalSeq() {
+  X=$(./yq e test.yml)
+  expected=$(cat test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorPipeIntoEvalAll() {
+  X=$(./yq ea - < test.yml)
+  expected=$(cat test.yml)
+  assertEquals "$expected" "$X"
+}
+
+
+testLeadingSeperatorEvalAll() {
+  X=$(./yq ea test.yml)
+  expected=$(cat test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalSimple() {
+  read -r -d '' expected << EOM
+---
+a: test
+---
+version: 3
+application: MyApp
+EOM
+
+
+  X=$(./yq e '.' test.yml examples/order.yaml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocInOneFile() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+---
+b: things
+EOL
+  expected=$(cat test.yml)
+  X=$(./yq e '.' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocInOneFileEvalAll() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+---
+b: things
+EOL
+  expected=$(cat test.yml)
+  X=$(./yq ea '.' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalComments() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+a: test
+EOL
+
+cat >test2.yml <<EOL
+# this is another doc
+# great
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+a: test
+---
+# this is another doc
+# great
+b: sane
+EOM
+
+
+  X=$(./yq e '.' test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalCommentsTrailingSep() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+---
+a: test
+EOL
+
+cat >test2.yml <<EOL
+# this is another doc
+# great
+---
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+---
+a: test
+---
+# this is another doc
+# great
+---
+b: sane
+EOM
+
+
+  X=$(./yq e '.' test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiMultiDocEvalCommentsTrailingSep() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+---
+a: test
+---
+a1: test2
+EOL
+
+cat >test2.yml <<EOL
+# this is another doc
+# great
+---
+b: sane
+---
+b2: cool
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+---
+a: test
+---
+a1: test2
+---
+# this is another doc
+# great
+---
+b: sane
+---
+b2: cool
+EOM
+
+
+  X=$(./yq e '.' test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalCommentsLeadingSep() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+EOL
+
+cat >test2.yml <<EOL
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+---
+# hi peeps
+# cool
+a: test
+---
+# this is another doc
+# great
+b: sane
+EOM
+
+
+  X=$(./yq e '.' test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+# https://github.com/mikefarah/yq/issues/919
+testLeadingSeparatorDoesNotBreakCommentsOnOtherFiles() {
+  cat >test.yml <<EOL
+# a1
+a: 1
+# a2
+EOL
+
+cat >test2.yml <<EOL
+# b1
+b: 2
+# b2
+EOL
+
+  read -r -d '' expected << EOM
+# a1
+a: 1
+# a2
+
+# b1
+b: 2
+# b2
+EOM
+
+
+  X=$(./yq ea 'select(fi == 0) * select(fi == 1)' test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalCommentsStripComments() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  # it will be hard to remove that top level separator
+  read -r -d '' expected << EOM
+a: test
+---
+b: sane
+EOM
+
+  X=$(./yq e '... comments=""'  test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalCommentsLeadingSepNoDocFlag() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+a: test
+# this is another doc
+# great
+b: sane
+EOM
+
+
+  X=$(./yq e '.' --no-doc test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalJsonFlag() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+EOL
+
+cat >test2.yml <<EOL
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+{
+  "a": "test"
+}
+{
+  "b": "sane"
+}
+EOM
+
+
+  X=$(./yq e '.' -j test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalAllJsonFlag() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+EOL
+
+cat >test2.yml <<EOL
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+{
+  "a": "test"
+}
+{
+  "b": "sane"
+}
+EOM
+
+
+  X=$(./yq ea '.' -j test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalAll() {
+  read -r -d '' expected << EOM
+---
+a: test
+---
+version: 3
+application: MyApp
+EOM
+
+
+  X=$(./yq ea '.' test.yml examples/order.yaml)
+  assertEquals "$expected" "$X"
+}
+
+source ./scripts/shunit2

acceptance_tests/output-format.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+
+testOutputJsonDeprecated() {
+  cat >test.yml <<EOL
+a: {b: ["cat"]}
+EOL
+
+  read -r -d '' expected << EOM
+{
+  "a": {
+    "b": [
+      "cat"
+    ]
+  }
+}
+EOM
+
+  X=$(./yq e -j test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -j test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testOutputJson() {
+  cat >test.yml <<EOL
+a: {b: ["cat"]}
+EOL
+
+  read -r -d '' expected << EOM
+{
+  "a": {
+    "b": [
+      "cat"
+    ]
+  }
+}
+EOM
+
+  X=$(./yq e --output-format=json test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --output-format=json test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testOutputJsonShort() {
+  cat >test.yml <<EOL
+a: {b: ["cat"]}
+EOL
+
+  read -r -d '' expected << EOM
+{
+  "a": {
+    "b": [
+      "cat"
+    ]
+  }
+}
+EOM
+
+  X=$(./yq e -o=j test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -o=j test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testOutputProperties() {
+  cat >test.yml <<EOL
+a: {b: {c: ["cat"]}}
+EOL
+
+  read -r -d '' expected << EOM
+a.b.c.0 = cat
+EOM
+
+  X=$(./yq e --output-format=props test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --output-format=props test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testOutputPropertiesShort() {
+  cat >test.yml <<EOL
+a: {b: {c: ["cat"]}}
+EOL
+
+  read -r -d '' expected << EOM
+a.b.c.0 = cat
+EOM
+
+  X=$(./yq e -o=p test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -o=p test.yml)
+  assertEquals "$expected" "$X"
+}
+
+source ./scripts/shunit2

acceptance_tests/pretty-print.sh

@@ -0,0 +1,173 @@
+#!/bin/bash
+
+
+testPrettyPrintWithBooleans() {
+  cat >test.yml <<EOL
+leaveUnquoted: [yes, no, on, off, y, n, true, false]
+leaveQuoted: ["yes", "no", "on", "off", "y", "n", "true", "false"]
+
+EOL
+
+  read -r -d '' expected << EOM
+leaveUnquoted:
+  - yes
+  - no
+  - on
+  - off
+  - y
+  - n
+  - true
+  - false
+leaveQuoted:
+  - "yes"
+  - "no"
+  - "on"
+  - "off"
+  - "y"
+  - "n"
+  - "true"
+  - "false"
+EOM
+
+  X=$(./yq e --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testPrettyPrintWithBooleansCapitals() {
+  cat >test.yml <<EOL
+leaveUnquoted: [YES, NO, ON, OFF, Y, N, TRUE, FALSE]
+leaveQuoted: ["YES", "NO", "ON", "OFF", "Y", "N", "TRUE", "FALSE"]
+EOL
+
+  read -r -d '' expected << EOM
+leaveUnquoted:
+  - YES
+  - NO
+  - ON
+  - OFF
+  - Y
+  - N
+  - TRUE
+  - FALSE
+leaveQuoted:
+  - "YES"
+  - "NO"
+  - "ON"
+  - "OFF"
+  - "Y"
+  - "N"
+  - "TRUE"
+  - "FALSE"
+EOM
+
+  X=$(./yq e --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testPrettyPrintOtherStringValues() {
+  cat >test.yml <<EOL
+leaveUnquoted: [yesSir, hellno, bonapite]
+makeUnquoted: ["yesSir", "hellno", "bonapite"]
+EOL
+
+  read -r -d '' expected << EOM
+leaveUnquoted:
+  - yesSir
+  - hellno
+  - bonapite
+makeUnquoted:
+  - yesSir
+  - hellno
+  - bonapite
+EOM
+
+  X=$(./yq e --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testPrettyPrintKeys() {
+  cat >test.yml <<EOL
+"removeQuotes": "please"
+EOL
+
+  read -r -d '' expected << EOM
+removeQuotes: please
+EOM
+
+  X=$(./yq e --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testPrettyPrintOtherStringValues() {
+  cat >test.yml <<EOL
+leaveUnquoted: [yesSir, hellno, bonapite]
+makeUnquoted: ["yesSir", "hellno", "bonapite"]
+EOL
+
+  read -r -d '' expected << EOM
+leaveUnquoted:
+  - yesSir
+  - hellno
+  - bonapite
+makeUnquoted:
+  - yesSir
+  - hellno
+  - bonapite
+EOM
+
+  X=$(./yq e --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testPrettyPrintStringBlocks() {
+  cat >test.yml <<EOL
+"removeQuotes": | 
+  "please"
+EOL
+
+  read -r -d '' expected << EOM
+removeQuotes: |
+  "please"
+EOM
+
+  X=$(./yq e --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testPrettyPrintWithExpression() {
+  cat >test.yml <<EOL
+a: {b: {c: ["cat"]}}
+EOL
+
+  read -r -d '' expected << EOM
+b:
+  c:
+    - cat
+EOM
+
+  X=$(./yq e '.a' --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea '.a' --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+}
+
+source ./scripts/shunit2

action.yml

@@ -1,11 +1,15 @@
 name: 'yq - portable yaml processor'
 description: 'create, read, update, delete, merge, validate and do more with yaml'
-icon: command
-color: gray-dark
+branding:
+  icon: command
+  color: gray-dark
 inputs:
   cmd:
     description: 'The Command which should be run'
     required: true
+outputs:
+  result:
+    description: "The complete result from the yq command being run"
 runs:
   using: 'docker'
   image: 'github-action/Dockerfile'

cmd/constant.go

@@ -1,9 +1,11 @@
 package cmd
 
+var leadingContentPreProcessing = true
 var unwrapScalar = true
 
 var writeInplace = false
 var outputToJSON = false
+var outputFormat = "yaml"
 
 var exitStatus = false
 var forceColor = false
@@ -16,4 +18,7 @@ var verbose = false
 var version = false
 var prettyPrint = false
 
+// can be either "" (off), "extract" or "process"
+var frontMatter = ""
+
 var completedSuccessfully = false

cmd/evaluate_all_command.go

@@ -15,10 +15,27 @@ func createEvaluateAllCommand() *cobra.Command {
 		Aliases: []string{"ea"},
 		Short:   "Loads _all_ yaml documents of _all_ yaml files and runs expression once",
 		Example: `
-# merges f2.yml into f1.yml (inplace)
-yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' f1.yml f2.yml		
+# Merge f2.yml into f1.yml (inplace)
+yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' f1.yml f2.yml
+## the same command and expression using shortened names:
+yq ea -i 'select(fi == 0) * select(fi == 1)' f1.yml f2.yml
+
+
+# Merge all given files
+yq ea '. as $item ireduce ({}; . * $item )' file1.yml file2.yml ...
+
+# Read from STDIN
+## use '-' as a filename to read from STDIN
+cat file2.yml | yq ea '.a.b' file1.yml - file3.yml
+`,
+		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
+See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.
+
+## Evaluate All ##
+This command loads _all_ yaml documents of _all_ yaml files and runs expression once
+Useful when you need to run an expression across several yaml documents or files (like merge).
+Note that it consumes more memory than eval.
 `,
-		Long: "Evaluate All:\nUseful when you need to run an expression across several yaml documents or files. Consumes more memory than eval",
 		RunE: evaluateAll,
 	}
 	return cmdEvalAll
@@ -69,26 +86,51 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 	if nullInput && len(args) > 1 {
 		return errors.New("Cannot pass files in when using null-input flag")
 	}
+	// backwards compatibilty
+	if outputToJSON {
+		outputFormat = "json"
+	}
 
-	printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+	format, err := yqlib.OutputFormatFromString(outputFormat)
+	if err != nil {
+		return err
+	}
+
+	printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+
+	if frontMatter != "" {
+		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])
+		err = frontMatterHandler.Split()
+		if err != nil {
+			return err
+		}
+		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
+
+		if frontMatter == "process" {
+			reader := frontMatterHandler.GetContentReader()
+			printer.SetAppendix(reader)
+			defer yqlib.SafelyCloseReader(reader)
+		}
+		defer frontMatterHandler.CleanUp()
+	}
 
 	allAtOnceEvaluator := yqlib.NewAllAtOnceEvaluator()
 	switch len(args) {
 	case 0:
 		if pipingStdIn {
-			err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer)
+			err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer, leadingContentPreProcessing)
 		} else {
 			cmd.Println(cmd.UsageString())
 			return nil
 		}
 	case 1:
 		if nullInput {
-			err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(args[0]), printer)
+			err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(args[0]), printer, "")
 		} else {
-			err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer)
+			err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer, leadingContentPreProcessing)
 		}
 	default:
-		err = allAtOnceEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer)
+		err = allAtOnceEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing)
 	}
 
 	completedSuccessfully = err == nil

cmd/evalute_sequence_command.go

@@ -13,32 +13,42 @@ func createEvaluateSequenceCommand() *cobra.Command {
 	var cmdEvalSequence = &cobra.Command{
 		Use:     "eval [expression] [yaml_file1]...",
 		Aliases: []string{"e"},
-		Short:   "Apply expression to each document in each yaml file given in sequence",
+		Short:   "Apply the expression to each document in each yaml file in sequence",
 		Example: `
-# runs the expression against each file, in series
-yq e '.a.b | length' f1.yml f2.yml 
+# Reads field under the given path for each file
+yq e '.a.b' f1.yml f2.yml 
 
-# prints out the file
+# Prints out the file
 yq e sample.yaml 
 
-# prints a new yaml document
+# Read from STDIN
+## use '-' as a filename to read from STDIN
+cat file2.yml | yq e '.a.b' file1.yml - file3.yml
+
+# Creates a new yaml document
+## Note that editing an empty file does not work.
 yq e -n '.a.b.c = "cat"' 
 
-
-# updates file.yaml directly
+# Update a file inplace
 yq e '.a.b = "cool"' -i file.yaml 
 `,
-		Long: "Evaluate Sequence:\nIterate over each yaml document, apply the expression and print the results, in sequence.",
+		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
+See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.
+
+## Evaluate Sequence ##
+This command iterates over each yaml document from each given file, applies the 
+expression and prints the result in sequence.`,
 		RunE: evaluateSequence,
 	}
 	return cmdEvalSequence
 }
 
 func processExpression(expression string) string {
+	var prettyPrintExp = `(... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not))  ) style=""`
 	if prettyPrint && expression == "" {
-		return `... style=""`
+		return prettyPrintExp
 	} else if prettyPrint {
-		return fmt.Sprintf("%v | ... style= \"\"", expression)
+		return fmt.Sprintf("%v | %v", expression, prettyPrintExp)
 	}
 	return expression
 }
@@ -86,7 +96,17 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 		defer func() { writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully) }()
 	}
 
-	printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+	// backwards compatibilty
+	if outputToJSON {
+		outputFormat = "json"
+	}
+
+	format, err := yqlib.OutputFormatFromString(outputFormat)
+	if err != nil {
+		return err
+	}
+
+	printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
 
 	streamEvaluator := yqlib.NewStreamEvaluator()
 
@@ -94,22 +114,38 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 		return errors.New("Cannot pass files in when using null-input flag")
 	}
 
+	if frontMatter != "" {
+		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])
+		err = frontMatterHandler.Split()
+		if err != nil {
+			return err
+		}
+		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
+
+		if frontMatter == "process" {
+			reader := frontMatterHandler.GetContentReader()
+			printer.SetAppendix(reader)
+			defer yqlib.SafelyCloseReader(reader)
+		}
+		defer frontMatterHandler.CleanUp()
+	}
+
 	switch len(args) {
 	case 0:
 		if pipingStdIn {
-			err = streamEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer)
+			err = streamEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer, leadingContentPreProcessing)
 		} else {
 			cmd.Println(cmd.UsageString())
 			return nil
 		}
 	case 1:
 		if nullInput {
-			err = streamEvaluator.EvaluateNew(processExpression(args[0]), printer)
+			err = streamEvaluator.EvaluateNew(processExpression(args[0]), printer, "")
 		} else {
-			err = streamEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer)
+			err = streamEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer, leadingContentPreProcessing)
 		}
 	default:
-		err = streamEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer)
+		err = streamEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing)
 	}
 	completedSuccessfully = err == nil
 

cmd/root.go

@@ -11,7 +11,8 @@ func New() *cobra.Command {
 	var rootCmd = &cobra.Command{
 		Use:   "yq",
 		Short: "yq is a lightweight and portable command-line YAML processor.",
-		Long:  `yq is a lightweight and portable command-line YAML processor. It aims to be the jq or sed of yaml files.`,
+		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
+See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if version {
 				cmd.Print(GetVersionDisplay())
@@ -40,7 +41,14 @@ func New() *cobra.Command {
 	}
 
 	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "verbose mode")
-	rootCmd.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "output as json. Set indent to 0 to print json in one line.")
+
+	rootCmd.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "(deprecated) output as json. Set indent to 0 to print json in one line.")
+	err := rootCmd.PersistentFlags().MarkDeprecated("tojson", "please use -o=json instead")
+	if err != nil {
+		panic(err)
+	}
+
+	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "yaml", "[yaml|y|json|j|props|p] output format type.")
 	rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.")
 	rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)")
 
@@ -53,6 +61,8 @@ func New() *cobra.Command {
 
 	rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")
 	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
+	rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
+	rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and seperators before processing expression. This is a workaround for go-yaml to persist header content. This flag will be removed once this feature has been out in the wild for a while.")
 	rootCmd.AddCommand(
 		createEvaluateSequenceCommand(),
 		createEvaluateAllCommand(),

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.5.1"
+	Version = "4.12.2"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release
@@ -25,7 +25,7 @@ const ProductName = "yq"
 // GetVersionDisplay composes the parts of the version in a way that's suitable
 // for displaying to humans.
 func GetVersionDisplay() string {
-	return fmt.Sprintf("%s version %s\n", ProductName, getHumanVersion())
+	return fmt.Sprintf("yq (https://github.com/mikefarah/yq/) version %s\n", getHumanVersion())
 }
 
 func getHumanVersion() string {

cmd/version_test.go

@@ -3,7 +3,7 @@ package cmd
 import "testing"
 
 func TestGetVersionDisplay(t *testing.T) {
-	var expectedVersion = ProductName + " version " + Version
+	var expectedVersion = ProductName + " (https://github.com/mikefarah/yq/) version " + Version
 	if VersionPrerelease != "" {
 		expectedVersion = expectedVersion + "-" + VersionPrerelease
 	}

debian/changelog

@@ -1,3 +1,12 @@
+yq (4.9.6) focal; urgency=medium
+
+  * Added darwin/arm64 build, thanks @alecthomas
+  * Incremented docker alpine base version, thanks @da6d6i7-bronga
+  * Bug fix: multine expression
+  * Bug fix: special character
+
+ -- Roberto Mier Escandon <rmescandon@gmail.com>  Tue, 29 Jun 2021 21:32:14 +0200
+
 yq (3.3.2) focal; urgency=medium
 
   * Bug fix: existStatus bug (#459)

debian/control

@@ -3,8 +3,7 @@ Section: devel
 Priority: optional
 Maintainer: Roberto Mier Escandón <rmescandon@gmail.com>
 Build-Depends: debhelper (>=10),
-               dh-golang (>=1.34),
-               golang-1.13,
+               golang-1.15,
                rsync
 Standards-Version: 4.1.4
 Homepage: https://github.com/mikefarah/yq.git

debian/rules

@@ -17,7 +17,7 @@
 PROJECT := yq
 OWNER := mikefarah
 REPO := github.com
-GOVERSION := 1.13
+GOVERSION := 1.15
 
 export DH_OPTIONS
 export DH_GOPKG := ${REPO}/${OWNER}/${PROJECT}
@@ -34,7 +34,7 @@ BINDIR := /usr/bin
 ASSETSDIR := /usr/share/${PROJECT}
 
 %:
-	dh $@ --builddirectory=${GOPATH} --buildsystem=golang
+	dh $@
 
 override_dh_auto_build:
 	mkdir -p ${SRCDIR}

examples/data1.yaml

@@ -1,6 +1,3 @@
-a:
-  key1: "value1"
-  key2: 2.6
-  ab:
-    key1: 6
-    key2: "h"
+# a1
+a: 1
+# a2

examples/data2.yaml

@@ -1,7 +1,3 @@
-a: other # better than the original
-b: [3, 4]
-c:
-  toast: leave
-  test: 1
-  tell: 1
-  tasty.taco: cool
+#b1 
+b: 2
+#b2

examples/empty.yaml

@@ -1,2 +1 @@
-# a: apple
-# b: cat
+# comment

examples/front-matter.yaml

@@ -0,0 +1,6 @@
+---
+a: apple
+b: bannana
+---
+hello there
+apples: great

examples/leading-seperator.yaml

@@ -0,0 +1,2 @@
+---
+a: test

github-action/Dockerfile

@@ -1,5 +1,8 @@
-FROM mikefarah/yq:4.5.1
+FROM mikefarah/yq:4.12.2
 
 COPY entrypoint.sh /entrypoint.sh
 
+# this seems to be the default user in github actions
+USER 1001
+
 ENTRYPOINT ["/entrypoint.sh"]

github-action/entrypoint.sh

@@ -1,4 +1,6 @@
 #!/bin/sh -l
-
-echo "$1"
-eval $1
+set -e
+echo "::debug::\$cmd: $1"
+RESULT=$(eval "$1")
+echo "::debug::\$RESULT: $RESULT"
+echo ::set-output name=result::"$RESULT"

go.mod

@@ -3,12 +3,13 @@ module github.com/mikefarah/yq/v4
 require (
 	github.com/elliotchance/orderedmap v1.4.0
 	github.com/fatih/color v1.10.0
-	github.com/goccy/go-yaml v1.8.8
-	github.com/jinzhu/copier v0.2.3
-	github.com/spf13/cobra v1.1.1
+	github.com/goccy/go-yaml v1.8.9
+	github.com/jinzhu/copier v0.2.8
+	github.com/magiconair/properties v1.8.5
+	github.com/spf13/cobra v1.1.3
 	github.com/timtadh/data-structures v0.5.3 // indirect
 	github.com/timtadh/lexmachine v0.2.2
-	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect
+	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )

go.sum

@@ -52,8 +52,8 @@ github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTM
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/goccy/go-yaml v1.8.8 h1:MGfRB1GeSn/hWXYWS2Pt67iC2GJNnebdIro01ddyucA=
-github.com/goccy/go-yaml v1.8.8/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
+github.com/goccy/go-yaml v1.8.9 h1:4AEXg2qx+/w29jXnXpMY6mTckmYu1TMoHteKuMf0HFg=
+github.com/goccy/go-yaml v1.8.9/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -101,8 +101,8 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p
 github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/jinzhu/copier v0.2.3 h1:Oe09ju+9qft7TffZ7l/04AB2f8u1+V4ZMxmp/nnqeOs=
-github.com/jinzhu/copier v0.2.3/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro=
+github.com/jinzhu/copier v0.2.8 h1:N8MbL5niMwE3P4dOwurJixz5rMkKfujmMRFmAanSzWE=
+github.com/jinzhu/copier v0.2.8/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -119,6 +119,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls=
+github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
@@ -168,10 +170,9 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4=
-github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
+github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
+github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
@@ -180,7 +181,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -257,10 +257,9 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=
-golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
@@ -270,7 +269,6 @@ golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd h1:/e+gpKk9r3dJobndpTytxS2gOy6m5uvpg+ISQoEcusQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
@@ -283,7 +281,6 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc h1:NCy3Ohtk6Iny5V/reW2Ktypo4zIpWBdRJ1uFMjBxdg8=
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
@@ -310,7 +307,6 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -321,11 +317,9 @@ gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
-gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/yqlib/all_at_once_evaluator.go

@@ -8,7 +8,7 @@ import (
 
 // A yaml expression evaluator that runs the expression once against all files/nodes in memory.
 type Evaluator interface {
-	EvaluateFiles(expression string, filenames []string, printer Printer) error
+	EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error
 
 	// EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes
 	EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error)
@@ -46,15 +46,21 @@ func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCand
 	return context.MatchingNodes, nil
 }
 
-func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error {
+func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error {
 	fileIndex := 0
+	firstFileLeadingContent := ""
 
 	var allDocuments *list.List = list.New()
 	for _, filename := range filenames {
-		reader, err := readStream(filename)
+		reader, leadingContent, err := readStream(filename, fileIndex == 0 && leadingContentPreProcessing)
 		if err != nil {
 			return err
 		}
+
+		if fileIndex == 0 {
+			firstFileLeadingContent = leadingContent
+		}
+
 		fileDocuments, err := readDocuments(reader, filename, fileIndex)
 		if err != nil {
 			return err
@@ -62,6 +68,19 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
 		allDocuments.PushBackList(fileDocuments)
 		fileIndex = fileIndex + 1
 	}
+
+	if allDocuments.Len() == 0 {
+		candidateNode := &CandidateNode{
+			Document:  0,
+			Filename:  "",
+			Node:      &yaml.Node{Kind: yaml.DocumentNode, HeadComment: firstFileLeadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
+			FileIndex: 0,
+		}
+		allDocuments.PushBack(candidateNode)
+	} else {
+		allDocuments.Front().Value.(*CandidateNode).Node.HeadComment = firstFileLeadingContent
+	}
+
 	matches, err := e.EvaluateCandidateNodes(expression, allDocuments)
 	if err != nil {
 		return err

pkg/yqlib/candidate_node.go

@@ -8,9 +8,10 @@ import (
 )
 
 type CandidateNode struct {
-	Node      *yaml.Node    // the actual node
-	Path      []interface{} /// the path we took to get to this node
-	Document  uint          // the document index of this node
+	Node      *yaml.Node     // the actual node
+	Parent    *CandidateNode // parent node
+	Path      []interface{}  /// the path we took to get to this node
+	Document  uint           // the document index of this node
 	Filename  string
 	FileIndex int
 	// when performing op against all nodes given, this will treat all the nodes as one
@@ -31,6 +32,7 @@ func (n *CandidateNode) CreateChild(path interface{}, node *yaml.Node) *Candidat
 	return &CandidateNode{
 		Node:      node,
 		Path:      n.createChildPath(path),
+		Parent:    n,
 		Document:  n.Document,
 		Filename:  n.Filename,
 		FileIndex: n.FileIndex,
@@ -86,7 +88,14 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode) {
 	if n.Node.Style == 0 {
 		n.Node.Style = other.Node.Style
 	}
-	n.Node.FootComment = n.Node.FootComment + other.Node.FootComment
-	n.Node.HeadComment = n.Node.HeadComment + other.Node.HeadComment
-	n.Node.LineComment = n.Node.LineComment + other.Node.LineComment
+
+	if other.Node.FootComment != "" {
+		n.Node.FootComment = other.Node.FootComment
+	}
+	if other.Node.HeadComment != "" {
+		n.Node.HeadComment = other.Node.HeadComment
+	}
+	if other.Node.LineComment != "" {
+		n.Node.LineComment = other.Node.LineComment
+	}
 }

pkg/yqlib/context.go

@@ -2,8 +2,10 @@ package yqlib
 
 import (
 	"container/list"
+	"fmt"
 
 	"github.com/jinzhu/copier"
+	logging "gopkg.in/op/go-logging.v1"
 )
 
 type Context struct {
@@ -12,6 +14,14 @@ type Context struct {
 	DontAutoCreate bool
 }
 
+func (n *Context) SingleReadonlyChildContext(candidate *CandidateNode) Context {
+	list := list.New()
+	list.PushBack(candidate)
+	newContext := n.ChildContext(list)
+	newContext.DontAutoCreate = true
+	return newContext
+}
+
 func (n *Context) SingleChildContext(candidate *CandidateNode) Context {
 	list := list.New()
 	list.PushBack(candidate)
@@ -43,6 +53,14 @@ func (n *Context) ChildContext(results *list.List) Context {
 	return clone
 }
 
+func (n *Context) ToString() string {
+	if !log.IsEnabledFor(logging.DEBUG) {
+		return ""
+	}
+	result := fmt.Sprintf("Context\nDontAutoCreate: %v\n", n.DontAutoCreate)
+	return result + NodesToString(n.MatchingNodes)
+}
+
 func (n *Context) Clone() Context {
 	clone := Context{}
 	err := copier.Copy(&clone, n)
@@ -52,3 +70,15 @@ func (n *Context) Clone() Context {
 	}
 	return clone
 }
+
+func (n *Context) ReadOnlyClone() Context {
+	clone := n.Clone()
+	clone.DontAutoCreate = true
+	return clone
+}
+
+func (n *Context) WritableClone() Context {
+	clone := n.Clone()
+	clone.DontAutoCreate = false
+	return clone
+}

pkg/yqlib/doc/Add.md

@@ -101,7 +101,7 @@ will output
 - hello
 ```
 
-## Update array (append)
+## Append to array
 Given a sample.yml file of:
 ```yaml
 a:
@@ -127,6 +127,62 @@ b:
   - 4
 ```
 
+## Append another array using +=
+Given a sample.yml file of:
+```yaml
+a:
+  - 1
+  - 2
+b:
+  - 3
+  - 4
+```
+then
+```bash
+yq eval '.a += .b' sample.yml
+```
+will output
+```yaml
+a:
+  - 1
+  - 2
+  - 3
+  - 4
+b:
+  - 3
+  - 4
+```
+
+## Relative append
+Given a sample.yml file of:
+```yaml
+a:
+  a1:
+    b:
+      - cat
+  a2:
+    b:
+      - dog
+  a3: {}
+```
+then
+```bash
+yq eval '.a[].b += ["mouse"]' sample.yml
+```
+will output
+```yaml
+a:
+  a1:
+    b:
+      - cat
+      - mouse
+  a2:
+    b:
+      - dog
+      - mouse
+  a3: {b: [mouse]}
+```
+
 ## String concatenation
 Given a sample.yml file of:
 ```yaml
@@ -143,22 +199,6 @@ a: catmeow
 b: meow
 ```
 
-## Relative string concatenation
-Given a sample.yml file of:
-```yaml
-a: cat
-b: meow
-```
-then
-```bash
-yq eval '.a += .b' sample.yml
-```
-will output
-```yaml
-a: catmeow
-b: meow
-```
-
 ## Number addition - float
 If the lhs or rhs are floats then the expression will be calculated with floats.
 
@@ -195,18 +235,20 @@ a: 7
 b: 4
 ```
 
-## Increment number
+## Increment numbers
 Given a sample.yml file of:
 ```yaml
 a: 3
+b: 5
 ```
 then
 ```bash
-yq eval '.a += 1' sample.yml
+yq eval '.[] += 1' sample.yml
 ```
 will output
 ```yaml
 a: 4
+b: 6
 ```
 
 ## Add to null

pkg/yqlib/doc/Anchor and Alias Operators.md

@@ -1,8 +1,99 @@
-Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference aliases and remove anchor names).
+Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names).
 
 `yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.
 
 
+## Merge one map
+see https://yaml.org/type/merge.html
+
+Given a sample.yml file of:
+```yaml
+- &CENTER
+  x: 1
+  y: 2
+- &LEFT
+  x: 0
+  y: 2
+- &BIG
+  r: 10
+- &SMALL
+  r: 1
+- !!merge <<: *CENTER
+  r: 10
+```
+then
+```bash
+yq eval '.[4] | explode(.)' sample.yml
+```
+will output
+```yaml
+x: 1
+y: 2
+r: 10
+```
+
+## Merge multiple maps
+see https://yaml.org/type/merge.html
+
+Given a sample.yml file of:
+```yaml
+- &CENTER
+  x: 1
+  y: 2
+- &LEFT
+  x: 0
+  y: 2
+- &BIG
+  r: 10
+- &SMALL
+  r: 1
+- !!merge <<:
+    - *CENTER
+    - *BIG
+```
+then
+```bash
+yq eval '.[4] | explode(.)' sample.yml
+```
+will output
+```yaml
+r: 10
+x: 1
+y: 2
+```
+
+## Override
+see https://yaml.org/type/merge.html
+
+Given a sample.yml file of:
+```yaml
+- &CENTER
+  x: 1
+  y: 2
+- &LEFT
+  x: 0
+  y: 2
+- &BIG
+  r: 10
+- &SMALL
+  r: 1
+- !!merge <<:
+    - *BIG
+    - *LEFT
+    - *SMALL
+  x: 1
+```
+then
+```bash
+yq eval '.[4] | explode(.)' sample.yml
+```
+will output
+```yaml
+r: 10
+x: 1
+y: 2
+```
+
 ## Get anchor
 Given a sample.yml file of:
 ```yaml
@@ -78,6 +169,22 @@ b: &meow purr
 a: *meow
 ```
 
+## Set alias to blank does nothing
+Given a sample.yml file of:
+```yaml
+b: &meow purr
+a: cat
+```
+then
+```bash
+yq eval '.a alias = ""' sample.yml
+```
+will output
+```yaml
+b: &meow purr
+a: cat
+```
+
 ## Set alias relatively using assign-update
 Given a sample.yml file of:
 ```yaml
@@ -183,12 +290,42 @@ bar:
   c: bar_c
 foobarList:
   b: bar_b
-  a: foo_a
-  thing: bar_thing
+  thing: foo_thing
   c: foobarList_c
+  a: foo_a
 foobar:
   c: foo_c
   a: foo_a
   thing: foobar_thing
 ```
 
+## Dereference and update a field
+`Use explode with multiply to dereference an object
+
+Given a sample.yml file of:
+```yaml
+item_value: &item_value
+  value: true
+thingOne:
+  name: item_1
+  !!merge <<: *item_value
+thingTwo:
+  name: item_2
+  !!merge <<: *item_value
+```
+then
+```bash
+yq eval '.thingOne |= explode(.) * {"value": false}' sample.yml
+```
+will output
+```yaml
+item_value: &item_value
+  value: true
+thingOne:
+  name: item_1
+  value: false
+thingTwo:
+  name: item_2
+  !!merge <<: *item_value
+```
+

pkg/yqlib/doc/Assign (Update).md

@@ -1,10 +1,4 @@
-This operator is used to update node values. It can be used in either the:
 
-### plain form: `=`
-Which will assign the LHS node values to the RHS node values. The RHS expression is run against the matching nodes in the pipeline.
-
-### relative form: `|=`
-This will do a similar thing to the plain form, however, the RHS expression is run against _the LHS nodes_. This is useful for updating values based on old values, e.g. increment.
 ## Create yaml file
 Running
 ```bash
@@ -34,6 +28,24 @@ a:
   g: foof
 ```
 
+## Double elements in an array
+Given a sample.yml file of:
+```yaml
+- 1
+- 2
+- 3
+```
+then
+```bash
+yq eval '.[] |= . * 2' sample.yml
+```
+will output
+```yaml
+- 2
+- 4
+- 6
+```
+
 ## Update node from another file
 Note this will also work when the second file is a scalar (string/number)
 
@@ -81,7 +93,7 @@ c: fieldC
 ```
 then
 ```bash
-yq eval '(.a, .c) |= "potatoe"' sample.yml
+yq eval '(.a, .c) = "potatoe"' sample.yml
 ```
 will output
 ```yaml

pkg/yqlib/doc/Boolean Operators.md

@@ -1,5 +1,14 @@
-The `or` and `and` operators take two parameters and return a boolean result. `not` flips a boolean from true to false, or vice versa. These are most commonly used with the `select` operator to filter particular nodes.
-## OR example
+The `or` and `and` operators take two parameters and return a boolean result. 
+
+`not` flips a boolean from true to false, or vice versa. 
+
+`any` will return `true` if there are any `true` values in a array sequence, and `all` will return true if _all_ elements in an array are true.
+
+`any_c(condition)` and `all_c(condition)` are like `any` and `all` but they take a condition expression that is used against each element to determine if it's `true`. Note: in `jq` you can simply pass a condition to `any` or `all` and it simply works - `yq` isn't that clever..yet
+
+These are most commonly used with the `select` operator to filter particular nodes.
+
+## `or` example
 Running
 ```bash
 yq eval --null-input 'true or false'
@@ -9,7 +18,7 @@ will output
 true
 ```
 
-## AND example
+## `and` example
 Running
 ```bash
 yq eval --null-input 'true and false'
@@ -41,6 +50,104 @@ will output
   b: fly
 ```
 
+## `any` returns true if any boolean in a given array is true
+Given a sample.yml file of:
+```yaml
+- false
+- true
+```
+then
+```bash
+yq eval 'any' sample.yml
+```
+will output
+```yaml
+true
+```
+
+## `any` returns false for an empty array
+Given a sample.yml file of:
+```yaml
+[]
+```
+then
+```bash
+yq eval 'any' sample.yml
+```
+will output
+```yaml
+false
+```
+
+## `any_c` returns true if any element in the array is true for the given condition.
+Given a sample.yml file of:
+```yaml
+a:
+  - rad
+  - awesome
+b:
+  - meh
+  - whatever
+```
+then
+```bash
+yq eval '.[] |= any_c(. == "awesome")' sample.yml
+```
+will output
+```yaml
+a: true
+b: false
+```
+
+## `all` returns true if all booleans in a given array are true
+Given a sample.yml file of:
+```yaml
+- true
+- true
+```
+then
+```bash
+yq eval 'all' sample.yml
+```
+will output
+```yaml
+true
+```
+
+## `all` returns true for an empty array
+Given a sample.yml file of:
+```yaml
+[]
+```
+then
+```bash
+yq eval 'all' sample.yml
+```
+will output
+```yaml
+true
+```
+
+## `all_c` returns true if all elements in the array are true for the given condition.
+Given a sample.yml file of:
+```yaml
+a:
+  - rad
+  - awesome
+b:
+  - meh
+  - 12
+```
+then
+```bash
+yq eval '.[] |= all_c(tag == "!!str")' sample.yml
+```
+will output
+```yaml
+a: true
+b: false
+```
+
 ## Not true is false
 Running
 ```bash

pkg/yqlib/doc/Comment Operators.md

@@ -1,4 +1,15 @@
 Use these comment operators to set or retrieve comments.
+
+Like the `=` and `|=` assign operators, the same syntax applies when updating comments:
+
+
+### plain form: `=`
+This will assign the LHS nodes comments to the expression on the RHS. The RHS is run against the matching nodes in the pipeline
+
+### relative form: `|=` 
+Similar to the plain form, however the RHS evaluates against each matching LHS node! This is useful if you want to set the comments as a relative expression of the node, for instance its value or path.
+
+
 ## Set line comment
 Given a sample.yml file of:
 ```yaml
@@ -77,7 +88,7 @@ a: cat
 b: dog # leave this
 ```
 
-## Remove all comments
+## Remove (strip) all comments
 Note the use of `...` to ensure key nodes are included.
 
 Given a sample.yml file of:

pkg/yqlib/doc/Entries.md

@@ -0,0 +1,118 @@
+Similar to the same named functions in `jq` these functions convert to/from an object and an array of key-value pairs. This is most useful for performing operations on keys of maps.
+## to_entries Map
+Given a sample.yml file of:
+```yaml
+a: 1
+b: 2
+```
+then
+```bash
+yq eval 'to_entries' sample.yml
+```
+will output
+```yaml
+- key: a
+  value: 1
+- key: b
+  value: 2
+```
+
+## to_entries Array
+Given a sample.yml file of:
+```yaml
+- a
+- b
+```
+then
+```bash
+yq eval 'to_entries' sample.yml
+```
+will output
+```yaml
+- key: 0
+  value: a
+- key: 1
+  value: b
+```
+
+## to_entries null
+Given a sample.yml file of:
+```yaml
+null
+```
+then
+```bash
+yq eval 'to_entries' sample.yml
+```
+will output
+```yaml
+```
+
+## from_entries map
+Given a sample.yml file of:
+```yaml
+a: 1
+b: 2
+```
+then
+```bash
+yq eval 'to_entries | from_entries' sample.yml
+```
+will output
+```yaml
+a: 1
+b: 2
+```
+
+## from_entries with numeric key indexes
+from_entries always creates a map, even for numeric keys
+
+Given a sample.yml file of:
+```yaml
+- a
+- b
+```
+then
+```bash
+yq eval 'to_entries | from_entries' sample.yml
+```
+will output
+```yaml
+0: a
+1: b
+```
+
+## Use with_entries to update keys
+Given a sample.yml file of:
+```yaml
+a: 1
+b: 2
+```
+then
+```bash
+yq eval 'with_entries(.key |= "KEY_" + .)' sample.yml
+```
+will output
+```yaml
+KEY_a: 1
+KEY_b: 2
+```
+
+## Use with_entries to filter the map
+Given a sample.yml file of:
+```yaml
+a:
+  b: bird
+c:
+  d: dog
+```
+then
+```bash
+yq eval 'with_entries(select(.value | has("b")))' sample.yml
+```
+will output
+```yaml
+a:
+  b: bird
+```
+

pkg/yqlib/doc/Equals.md

@@ -93,3 +93,31 @@ will output
 true
 ```
 
+## Non exisitant key doesn't equal a value
+Given a sample.yml file of:
+```yaml
+a: frog
+```
+then
+```bash
+yq eval 'select(.b != "thing")' sample.yml
+```
+will output
+```yaml
+a: frog
+```
+
+## Two non existant keys are equal
+Given a sample.yml file of:
+```yaml
+a: frog
+```
+then
+```bash
+yq eval 'select(.b == .c)' sample.yml
+```
+will output
+```yaml
+a: frog
+```
+

pkg/yqlib/doc/File Operators.md

@@ -35,6 +35,26 @@ will output
 0
 ```
 
+## Get file indices of multiple documents
+Given a sample.yml file of:
+```yaml
+a: cat
+```
+And another sample another.yml file of:
+```yaml
+a: cat
+```
+then
+```bash
+yq eval-all 'fileIndex' sample.yml another.yml
+```
+will output
+```yaml
+0
+---
+1
+```
+
 ## Get file index alias
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Has.md

@@ -19,6 +19,29 @@ true
 false
 ```
 
+## Select, checking for existence of deep paths
+Simply pipe in parent expressions into `has`
+
+Given a sample.yml file of:
+```yaml
+- a:
+    b:
+      c: cat
+- a:
+    b:
+      d: dog
+```
+then
+```bash
+yq eval '.[] | select(.a.b | has("c"))' sample.yml
+```
+will output
+```yaml
+a:
+  b:
+    c: cat
+```
+
 ## Has array index
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Multiply (Merge).md

@@ -1,19 +1,34 @@
-Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently only objects are supported, which have the effect of merging the RHS into the LHS.
+Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported.
 
-To concatenate arrays when merging objects, use the *+ form (see examples below). This will recursively merge objects, appending arrays when it encounters them.
+## Objects and arrays - merging
+Objects are merged deeply matching on matching keys. By default, array values override and are not deeply merged.
 
-To merge only existing fields, use the *? form. Note that this can be used with the concatenate arrays too *+?.
 Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
 
-Multiplication of strings and numbers are not yet supported.
+### Merge Flags
+You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`.  See examples below
 
-## Merging files
+- `+` to append arrays
+- `?` to only merge existing fields
+- `d` to deeply merge arrays
+
+### Merging files
 Note the use of `eval-all` to ensure all documents are loaded into memory.
 
 ```bash
 yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' file1.yaml file2.yaml
 ```
 
+## Multiply integers
+Running
+```bash
+yq eval --null-input '3 * 4'
+```
+will output
+```yaml
+12
+```
+
 ## Merge objects together, returning merged result only
 Given a sample.yml file of:
 ```yaml
@@ -190,6 +205,72 @@ thing:
   - 4
 ```
 
+## Merge, deeply merging arrays
+Merging arrays deeply means arrays are merge like objects, with indexes as their key. In this case, we merge the first item in the array, and do nothing with the second.
+
+Given a sample.yml file of:
+```yaml
+a:
+  - name: fred
+    age: 12
+  - name: bob
+    age: 32
+b:
+  - name: fred
+    age: 34
+```
+then
+```bash
+yq eval '.a *d .b' sample.yml
+```
+will output
+```yaml
+- name: fred
+  age: 34
+- name: bob
+  age: 32
+```
+
+## Merge arrays of objects together, matching on a key
+It's a complex command, the trickyness comes from needing to have the right context in the expressions.
+First we save the second array into a variable '$two' which lets us reference it later.
+We then need to update the first array. We will use the relative update (|=) because we need to update relative to the current element of the array in the LHS in the RHS expression. 
+We set the current element of the first array as $cur. Now we multiply (merge) $cur with the matching entry in $two, by passing $two through a select filter.
+
+
+Given a sample.yml file of:
+```yaml
+- a: apple
+  b: appleB
+- a: kiwi
+  b: kiwiB
+- a: banana
+  b: bananaB
+```
+And another sample another.yml file of:
+```yaml
+- a: banana
+  c: bananaC
+- a: apple
+  b: appleB2
+- a: dingo
+  c: dingoC
+```
+then
+```bash
+yq eval-all '(select(fi==1) | .[]) as $two | select(fi==0) | .[] |= (. as $cur |  $cur * ($two | select(.a == $cur.a)))' sample.yml another.yml
+```
+will output
+```yaml
+- a: apple
+  b: appleB2
+- a: kiwi
+  b: kiwiB
+- a: banana
+  b: bananaB
+  c: bananaC
+```
+
 ## Merge to prefix an element
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Reduce.md

@@ -0,0 +1,76 @@
+Reduce is a powerful way to process a collection of data into a new form.
+
+```
+<exp> as $<name> ireduce (<init>; <block>)
+```
+
+e.g.
+
+```
+.[] as $item ireduce (0; . + $item)
+```
+
+On the LHS we are configuring the collection of items that will be reduced `<exp>` as well as what each element will be called `$<name>`. Note that the array has been splatted into its individual elements.
+
+On the RHS there is `<init>`, the starting value of the accumulator and `<block>`, the expression that will update the accumulator for each element in the collection. Note that within the block expression, `.` will evaluate to the current value of the accumulator. 
+
+## yq vs jq syntax
+Reduce syntax in `yq` is a little different from `jq` - as `yq` (currently) isn't as sophisticated as `jq` and its only supports infix notation (e.g. a + b, where the operator is in the middle of the two parameters) - where as `jq` uses a mix of infix notation with _prefix_ notation (e.g. `reduce a b` is like writing `+ a b`).
+
+To that end, the reduce operator is called `ireduce` for backwards compatability if a `jq` like prefix version of `reduce` is ever added.
+
+
+## Sum numbers
+Given a sample.yml file of:
+```yaml
+- 10
+- 2
+- 5
+- 3
+```
+then
+```bash
+yq eval '.[] as $item ireduce (0; . + $item)' sample.yml
+```
+will output
+```yaml
+20
+```
+
+## Merge all yaml files together
+Given a sample.yml file of:
+```yaml
+a: cat
+```
+And another sample another.yml file of:
+```yaml
+b: dog
+```
+then
+```bash
+yq eval-all '. as $item ireduce ({}; . * $item )' sample.yml another.yml
+```
+will output
+```yaml
+a: cat
+b: dog
+```
+
+## Convert an array to an object
+Given a sample.yml file of:
+```yaml
+- name: Cathy
+  has: apples
+- name: Bob
+  has: bananas
+```
+then
+```bash
+yq eval '.[] as $item ireduce ({}; .[$item | .name] = ($item | .has) )' sample.yml
+```
+will output
+```yaml
+Cathy: apples
+Bob: bananas
+```
+

pkg/yqlib/doc/String Operators.md

@@ -1,5 +1,47 @@
 # String Operators
 
+## RegEx
+This uses golangs native regex functions under the hood - See https://github.com/google/re2/wiki/Syntax for the supported syntax.
+
+
+# String blocks, bash and newlines
+Bash is notorious for chomping on precious trailing newline characters, making it tricky to set strings with newlines properly. In particular, the `$( exp )` _will trim trailing newlines_.
+
+For instance to get this yaml:
+
+```
+a: |
+  cat
+```
+
+Using `$( exp )` wont work, as it will trim the trailing new line.
+
+```
+m=$(echo "cat\n") yq e -n '.a = strenv(m)'
+a: cat
+```
+
+However, using printf works:
+```
+printf -v m "cat\n" ; m="$m" yq e -n '.a = strenv(m)'
+a: |
+  cat
+```
+
+As well as having multiline expressions:
+```
+m="cat
+"  yq e -n '.a = strenv(m)'
+a: |
+  cat
+```
+
+Similarly, if you're trying to set the content from a file, and want a trailing new line:
+
+```
+IFS= read -rd '' output < <(cat my_file)
+output=$output ./yq e '.data.values = strenv(output)' first.yml
+```
 ## Join strings
 Given a sample.yml file of:
 ```yaml
@@ -18,6 +60,207 @@ will output
 cat; meow; 1; ; true
 ```
 
+## Match string
+Given a sample.yml file of:
+```yaml
+foo bar foo
+```
+then
+```bash
+yq eval 'match("foo")' sample.yml
+```
+will output
+```yaml
+string: foo
+offset: 0
+length: 3
+captures: []
+```
+
+## Match string, case insensitive
+Given a sample.yml file of:
+```yaml
+foo bar FOO
+```
+then
+```bash
+yq eval '[match("(?i)foo"; "g")]' sample.yml
+```
+will output
+```yaml
+- string: foo
+  offset: 0
+  length: 3
+  captures: []
+- string: FOO
+  offset: 8
+  length: 3
+  captures: []
+```
+
+## Match with capture groups
+Given a sample.yml file of:
+```yaml
+abc abc
+```
+then
+```bash
+yq eval '[match("(abc)+"; "g")]' sample.yml
+```
+will output
+```yaml
+- string: abc
+  offset: 0
+  length: 3
+  captures:
+    - string: abc
+      offset: 0
+      length: 3
+- string: abc
+  offset: 4
+  length: 3
+  captures:
+    - string: abc
+      offset: 4
+      length: 3
+```
+
+## Match with named capture groups
+Given a sample.yml file of:
+```yaml
+foo bar foo foo  foo
+```
+then
+```bash
+yq eval '[match("foo (?P<bar123>bar)? foo"; "g")]' sample.yml
+```
+will output
+```yaml
+- string: foo bar foo
+  offset: 0
+  length: 11
+  captures:
+    - string: bar
+      offset: 4
+      length: 3
+      name: bar123
+- string: foo  foo
+  offset: 12
+  length: 8
+  captures:
+    - string: null
+      offset: -1
+      length: 0
+      name: bar123
+```
+
+## Capture named groups into a map
+Given a sample.yml file of:
+```yaml
+xyzzy-14
+```
+then
+```bash
+yq eval 'capture("(?P<a>[a-z]+)-(?P<n>[0-9]+)")' sample.yml
+```
+will output
+```yaml
+a: xyzzy
+n: "14"
+```
+
+## Match without global flag
+Given a sample.yml file of:
+```yaml
+cat cat
+```
+then
+```bash
+yq eval 'match("cat")' sample.yml
+```
+will output
+```yaml
+string: cat
+offset: 0
+length: 3
+captures: []
+```
+
+## Match with global flag
+Given a sample.yml file of:
+```yaml
+cat cat
+```
+then
+```bash
+yq eval '[match("cat"; "g")]' sample.yml
+```
+will output
+```yaml
+- string: cat
+  offset: 0
+  length: 3
+  captures: []
+- string: cat
+  offset: 4
+  length: 3
+  captures: []
+```
+
+## Test using regex
+Like jq'q equivalant, this works like match but only returns true/false instead of full match details
+
+Given a sample.yml file of:
+```yaml
+- cat
+- dog
+```
+then
+```bash
+yq eval '.[] | test("at")' sample.yml
+```
+will output
+```yaml
+true
+false
+```
+
+## Substitute / Replace string
+This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)
+Note the use of `|=` to run in context of the current string value.
+
+Given a sample.yml file of:
+```yaml
+a: dogs are great
+```
+then
+```bash
+yq eval '.a |= sub("dogs", "cats")' sample.yml
+```
+will output
+```yaml
+a: cats are great
+```
+
+## Substitute / Replace string with regex
+This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)
+Note the use of `|=` to run in context of the current string value.
+
+Given a sample.yml file of:
+```yaml
+a: cat
+b: heat
+```
+then
+```bash
+yq eval '.[] |= sub("(a)", "${1}r")' sample.yml
+```
+will output
+```yaml
+a: cart
+b: heart
+```
+
 ## Split strings
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Style.md

@@ -1,4 +1,42 @@
 The style operator can be used to get or set the style of nodes (e.g. string style, yaml style)
+## Update and set style of a particular node (simple)
+Given a sample.yml file of:
+```yaml
+a:
+  b: thing
+  c: something
+```
+then
+```bash
+yq eval '.a.b = "new" | .a.b style="double"' sample.yml
+```
+will output
+```yaml
+a:
+  b: "new"
+  c: something
+```
+
+## Update and set style of a particular node using path variables
+You can use a variable to re-use a path
+
+Given a sample.yml file of:
+```yaml
+a:
+  b: thing
+  c: something
+```
+then
+```bash
+yq eval '.a.b as $x | $x = "new" | $x style="double"' sample.yml
+```
+will output
+```yaml
+a:
+  b: "new"
+  c: something
+```
+
 ## Set tagged style
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Subtract.md

@@ -0,0 +1,110 @@
+
+## Array subtraction
+Running
+```bash
+yq eval --null-input '[1,2] - [2,3]'
+```
+will output
+```yaml
+- 1
+```
+
+## Array subtraction with nested array
+Running
+```bash
+yq eval --null-input '[[1], 1, 2] - [[1], 3]'
+```
+will output
+```yaml
+- 1
+- 2
+```
+
+## Array subtraction with nested object
+Note that order of the keys does not matter
+
+Given a sample.yml file of:
+```yaml
+- a: b
+  c: d
+- a: b
+```
+then
+```bash
+yq eval '. - [{"c": "d", "a": "b"}]' sample.yml
+```
+will output
+```yaml
+- a: b
+```
+
+## Number subtraction - float
+If the lhs or rhs are floats then the expression will be calculated with floats.
+
+Given a sample.yml file of:
+```yaml
+a: 3
+b: 4.5
+```
+then
+```bash
+yq eval '.a = .a - .b' sample.yml
+```
+will output
+```yaml
+a: -1.5
+b: 4.5
+```
+
+## Number subtraction - float
+If the lhs or rhs are floats then the expression will be calculated with floats.
+
+Given a sample.yml file of:
+```yaml
+a: 3
+b: 4.5
+```
+then
+```bash
+yq eval '.a = .a - .b' sample.yml
+```
+will output
+```yaml
+a: -1.5
+b: 4.5
+```
+
+## Number subtraction - int
+If both the lhs and rhs are ints then the expression will be calculated with ints.
+
+Given a sample.yml file of:
+```yaml
+a: 3
+b: 4
+```
+then
+```bash
+yq eval '.a = .a - .b' sample.yml
+```
+will output
+```yaml
+a: -1
+b: 4
+```
+
+## Decrement numbers
+Given a sample.yml file of:
+```yaml
+a: 3
+b: 5
+```
+then
+```bash
+yq eval '.[] -= 1' sample.yml
+```
+will output
+```yaml
+a: 2
+b: 4
+```
+

pkg/yqlib/doc/Traverse (Read).md

@@ -1,4 +1,4 @@
-This is the simplest (and perhaps most used) operator, it is used to navigate deeply into yaml structurse.
+This is the simplest (and perhaps most used) operator, it is used to navigate deeply into yaml structures.
 ## Simple map navigation
 Given a sample.yml file of:
 ```yaml
@@ -32,6 +32,21 @@ b: apple
 c: banana
 ```
 
+## Optional Splat
+Just like splat, but won't error if you run it against scalars
+
+Given a sample.yml file of:
+```yaml
+cat
+```
+then
+```bash
+yq eval '.[]' sample.yml
+```
+will output
+```yaml
+```
+
 ## Special characters
 Use quotes with brackets around path elements with special characters
 
@@ -48,6 +63,22 @@ will output
 frog
 ```
 
+## Nested special characters
+Given a sample.yml file of:
+```yaml
+a:
+  "key.withdots":
+    "another.key": apple
+```
+then
+```bash
+yq eval '.a["key.withdots"]["another.key"]' sample.yml
+```
+will output
+```yaml
+apple
+```
+
 ## Keys with spaces
 Use quotes with brackets around path elements with special characters
 
@@ -98,6 +129,23 @@ will output
 null
 ```
 
+## Optional identifier
+Like jq, does not output an error when the yaml is not an array or object as expected
+
+Given a sample.yml file of:
+```yaml
+- 1
+- 2
+- 3
+```
+then
+```bash
+yq eval '.a?' sample.yml
+```
+will output
+```yaml
+```
+
 ## Wildcard matching
 Given a sample.yml file of:
 ```yaml
@@ -179,6 +227,20 @@ will output
 1
 ```
 
+## Traversing nested arrays by index
+Given a sample.yml file of:
+```yaml
+[[], [cat]]
+```
+then
+```bash
+yq eval '.[1][0]' sample.yml
+```
+will output
+```yaml
+cat
+```
+
 ## Maps with numeric keys
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Unique.md

@@ -0,0 +1,82 @@
+This is used to filter out duplicated items in an array.
+
+## Unique array of scalars (string/numbers)
+Given a sample.yml file of:
+```yaml
+- 1
+- 2
+- 3
+- 2
+```
+then
+```bash
+yq eval 'unique' sample.yml
+```
+will output
+```yaml
+- 1
+- 2
+- 3
+```
+
+## Unique nulls
+Unique works on the node value, so it considers different representations of nulls to be different
+
+Given a sample.yml file of:
+```yaml
+- ~
+- null
+- ~
+- null
+```
+then
+```bash
+yq eval 'unique' sample.yml
+```
+will output
+```yaml
+- ~
+- null
+```
+
+## Unique all nulls
+Run against the node tag to unique all the nulls
+
+Given a sample.yml file of:
+```yaml
+- ~
+- null
+- ~
+- null
+```
+then
+```bash
+yq eval 'unique_by(tag)' sample.yml
+```
+will output
+```yaml
+- ~
+```
+
+## Unique array object fields
+Given a sample.yml file of:
+```yaml
+- name: harry
+  pet: cat
+- name: billy
+  pet: dog
+- name: harry
+  pet: dog
+```
+then
+```bash
+yq eval 'unique_by(.name)' sample.yml
+```
+will output
+```yaml
+- name: harry
+  pet: cat
+- name: billy
+  pet: dog
+```
+

pkg/yqlib/doc/aa.md

@@ -1,12 +1,116 @@
 # Operators
 
-In `yq` expressions are made up of operators. Operators have 0-2 arguments and run against the current 'matching' nodes in the expression tree.
+In `yq` expressions are made up of operators and pipes. A context of nodes is passed through the expression and each operation takes the context as input and returns a new context as output. That output is piped in as input for the next operation in the expression. To begin with, the context is set to the first yaml document of the first yaml file (if processing in sequence using eval).
 
 Lets look at a couple of examples.
 
-The `length` operator take no arguments, and will simply return the length of _each_ matching node. So if there were 2 nodes, one string and one array, length will update the 'matching' nodes context to be two new numeric scalar nodes representing the lengths of the orignal 'matching' nodes.
+## Example with a simple operator
 
-The `=` operator takes two arguments, a `lhs` expression and `rhs` expression. It runs the 'matching' nodes context against the `lhs` expression to find the nodes to update, lets call it `lhsNodes`, and then runs the matching nodes against the `rhs` to find the new values, lets call that `rhsNodes`. It updates the `lhsNodes` values with the `rhsNodes` values and _returns the original matching nodes_. This is important, where length changed the matching nodes to be new nodes with the length values, `=` returns the original matching nodes, albeit with some of the nodes values updated. So `.a = 3` will still return the parent matching node, but with the matching child updated.
+Given a document like:
 
-Please see the individual operator docs for more information and examples.
+```yaml
+- [a]
+- "cat"
+```
 
+with an expression:
+
+```
+.[] | length
+```
+
+`yq` will initially set the context as single node of the entire yaml document, an array of two elements.
+
+```yaml
+- [a]
+- "cat"
+```
+
+This gets piped into the splat operator `.[]` which will split out the context into a collection of two nodes `[a]` and `"cat"`. Note that this is _not_ a yaml array.
+
+The `length` operator take no arguments, and will simply return the length of _each_ matching node in the context. So for the context of `[a]` and `"cat"`, it will return a new context of `1` and `3`.
+
+This being the last operation in the expression, the results will be printed out:
+
+```
+1
+3
+```
+
+# Example with an operator that takes arguments.
+
+Given a document like:
+
+```yaml
+a: cat
+b: dog
+```
+
+with an expression:
+
+```
+.a = .b
+```
+
+The `=` operator takes two arguments, a `lhs` expression, which in this case is `.a` and `rhs` expression which is `.b`. 
+
+It pipes the current, lets call it 'root' context through the `lhs` expression of `.a` to return the node 
+
+```yaml
+cat
+```
+
+Note that this node holds not only its value 'cat', but comments and metadata too, including path and parent information.
+
+The `=` operator then pipes the 'root' context through the `rhs` expression of `.b` to return the node
+
+```yaml
+dog
+```
+
+Both sides have now been evaluated, so now the operator copies across the value from the RHS to the value on the LHS, and it returns the now updated context:
+
+```yaml
+a: dog
+b: dog
+```
+
+# Relative update (e.g. `|=`)
+There is another form of the `=` operator which we call the relative form. It's very similar to `=` but with one key difference when evaluating the RHS expression.
+
+In the plain form, we pass in the 'root' level context to the RHS expression. In relative form, we pass in _each result of the LHS_ to the RHS expression. Let's go through an example.
+
+Given a document like:
+
+```yaml
+a: 1
+b: thing
+```
+
+with an expression:
+
+```
+.a |= . + 1
+```
+
+Similar to the `=` operator, `|=` takes two operands, the LHS and RHS.
+
+It pipes the current context (the whole document) through the LHS expression  of `.a` to get the node value:
+
+```
+1
+```
+
+Now it pipes _that LHS context_ into the RHS expression `. + 1` (whereas in the `=` plain form it piped the original document context into the RHS) to yield:
+
+
+```
+2
+```
+
+The assignment operator then copies across the value from the RHS to the value on the LHS, and it returns the now updated 'root' context:
+
+```yaml
+a: 2
+b: thing
+```

pkg/yqlib/doc/headers/Anchor and Alias Operators.md

@@ -1,4 +1,4 @@
-Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference aliases and remove anchor names).
+Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names).
 
 `yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.
 

pkg/yqlib/doc/headers/Boolean Operators.md

@@ -1 +1,9 @@
-The `or` and `and` operators take two parameters and return a boolean result. `not` flips a boolean from true to false, or vice versa. These are most commonly used with the `select` operator to filter particular nodes.
+The `or` and `and` operators take two parameters and return a boolean result. 
+
+`not` flips a boolean from true to false, or vice versa. 
+
+`any` will return `true` if there are any `true` values in a array sequence, and `all` will return true if _all_ elements in an array are true.
+
+`any_c(condition)` and `all_c(condition)` are like `any` and `all` but they take a condition expression that is used against each element to determine if it's `true`. Note: in `jq` you can simply pass a condition to `any` or `all` and it simply works - `yq` isn't that clever..yet
+
+These are most commonly used with the `select` operator to filter particular nodes.

pkg/yqlib/doc/headers/Comment Operators.md

@@ -1 +1,11 @@
-Use these comment operators to set or retrieve comments.
+Use these comment operators to set or retrieve comments.
+
+Like the `=` and `|=` assign operators, the same syntax applies when updating comments:
+
+
+### plain form: `=`
+This will assign the LHS nodes comments to the expression on the RHS. The RHS is run against the matching nodes in the pipeline
+
+### relative form: `|=` 
+Similar to the plain form, however the RHS evaluates against each matching LHS node! This is useful if you want to set the comments as a relative expression of the node, for instance its value or path.
+

pkg/yqlib/doc/headers/Entries.md

@@ -0,0 +1 @@
+Similar to the same named functions in `jq` these functions convert to/from an object and an array of key-value pairs. This is most useful for performing operations on keys of maps.

pkg/yqlib/doc/headers/Multiply (Merge).md

@@ -0,0 +1,20 @@
+Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently numbers, arrays and objects are supported.
+
+## Objects and arrays - merging
+Objects are merged deeply matching on matching keys. By default, array values override and are not deeply merged.
+
+Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
+
+### Merge Flags
+You can control how objects are merged by using one or more of the following flags. Multiple flags can be used together, e.g. `.a *+? .b`.  See examples below
+
+- `+` to append arrays
+- `?` to only merge existing fields
+- `d` to deeply merge arrays
+
+### Merging files
+Note the use of `eval-all` to ensure all documents are loaded into memory.
+
+```bash
+yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' file1.yaml file2.yaml
+```

pkg/yqlib/doc/headers/Multiply.md

@@ -1,15 +0,0 @@
-Like the multiple operator in jq, depending on the operands, this multiply operator will do different things. Currently only objects are supported, which have the effect of merging the RHS into the LHS.
-
-To concatenate arrays when merging objects, use the *+ form (see examples below). This will recursively merge objects, appending arrays when it encounters them.
-
-To merge only existing fields, use the *? form. Note that this can be used with the concatenate arrays too *+?.
-Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below.
-
-Multiplication of strings and numbers are not yet supported.
-
-## Merging files
-Note the use of `eval-all` to ensure all documents are loaded into memory.
-
-```bash
-yq eval-all 'select(fileIndex == 0) * select(fileIndex == 1)' file1.yaml file2.yaml
-```

pkg/yqlib/doc/headers/Reduce.md

@@ -0,0 +1,21 @@
+Reduce is a powerful way to process a collection of data into a new form.
+
+```
+<exp> as $<name> ireduce (<init>; <block>)
+```
+
+e.g.
+
+```
+.[] as $item ireduce (0; . + $item)
+```
+
+On the LHS we are configuring the collection of items that will be reduced `<exp>` as well as what each element will be called `$<name>`. Note that the array has been splatted into its individual elements.
+
+On the RHS there is `<init>`, the starting value of the accumulator and `<block>`, the expression that will update the accumulator for each element in the collection. Note that within the block expression, `.` will evaluate to the current value of the accumulator. 
+
+## yq vs jq syntax
+Reduce syntax in `yq` is a little different from `jq` - as `yq` (currently) isn't as sophisticated as `jq` and its only supports infix notation (e.g. a + b, where the operator is in the middle of the two parameters) - where as `jq` uses a mix of infix notation with _prefix_ notation (e.g. `reduce a b` is like writing `+ a b`).
+
+To that end, the reduce operator is called `ireduce` for backwards compatability if a `jq` like prefix version of `reduce` is ever added.
+

pkg/yqlib/doc/headers/String Operators.md

@@ -1 +1,44 @@
 # String Operators
+
+## RegEx
+This uses golangs native regex functions under the hood - See https://github.com/google/re2/wiki/Syntax for the supported syntax.
+
+
+# String blocks, bash and newlines
+Bash is notorious for chomping on precious trailing newline characters, making it tricky to set strings with newlines properly. In particular, the `$( exp )` _will trim trailing newlines_.
+
+For instance to get this yaml:
+
+```
+a: |
+  cat
+```
+
+Using `$( exp )` wont work, as it will trim the trailing new line.
+
+```
+m=$(echo "cat\n") yq e -n '.a = strenv(m)'
+a: cat
+```
+
+However, using printf works:
+```
+printf -v m "cat\n" ; m="$m" yq e -n '.a = strenv(m)'
+a: |
+  cat
+```
+
+As well as having multiline expressions:
+```
+m="cat
+"  yq e -n '.a = strenv(m)'
+a: |
+  cat
+```
+
+Similarly, if you're trying to set the content from a file, and want a trailing new line:
+
+```
+IFS= read -rd '' output < <(cat my_file)
+output=$output ./yq e '.data.values = strenv(output)' first.yml
+```

pkg/yqlib/doc/headers/Traverse (Read).md

@@ -1 +1 @@
-This is the simplest (and perhaps most used) operator, it is used to navigate deeply into yaml structurse.
+This is the simplest (and perhaps most used) operator, it is used to navigate deeply into yaml structures.

pkg/yqlib/doc/headers/Unique.md

@@ -0,0 +1 @@
+This is used to filter out duplicated items in an array.

pkg/yqlib/encoder_properties.go

@@ -0,0 +1,80 @@
+package yqlib
+
+import (
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/magiconair/properties"
+	yaml "gopkg.in/yaml.v3"
+)
+
+type propertiesEncoder struct {
+	destination io.Writer
+}
+
+func NewPropertiesEncoder(destination io.Writer) Encoder {
+	return &propertiesEncoder{destination}
+}
+
+func (pe *propertiesEncoder) Encode(node *yaml.Node) error {
+	mapKeysToStrings(node)
+	p := properties.NewProperties()
+	err := pe.doEncode(p, node, "")
+	if err != nil {
+		return err
+	}
+
+	_, err = p.WriteComment(pe.destination, "#", properties.UTF8)
+	return err
+}
+
+func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yaml.Node, path string) error {
+	p.SetComment(path,
+		strings.Replace(node.HeadComment, "#", "", 1)+
+			strings.Replace(node.LineComment, "#", "", 1))
+	switch node.Kind {
+	case yaml.ScalarNode:
+		_, _, err := p.Set(path, node.Value)
+		return err
+	case yaml.DocumentNode:
+		return pe.doEncode(p, node.Content[0], path)
+	case yaml.SequenceNode:
+		return pe.encodeArray(p, node.Content, path)
+	case yaml.MappingNode:
+		return pe.encodeMap(p, node.Content, path)
+	case yaml.AliasNode:
+		return pe.doEncode(p, node.Alias, path)
+	default:
+		return fmt.Errorf("Unsupported node %v", node.Tag)
+	}
+}
+
+func (pe *propertiesEncoder) appendPath(path string, key interface{}) string {
+	if path == "" {
+		return fmt.Sprintf("%v", key)
+	}
+	return fmt.Sprintf("%v.%v", path, key)
+}
+
+func (pe *propertiesEncoder) encodeArray(p *properties.Properties, kids []*yaml.Node, path string) error {
+	for index, child := range kids {
+		err := pe.doEncode(p, child, pe.appendPath(path, index))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (pe *propertiesEncoder) encodeMap(p *properties.Properties, kids []*yaml.Node, path string) error {
+	for index := 0; index < len(kids); index = index + 2 {
+		key := kids[index]
+		value := kids[index+1]
+		err := pe.doEncode(p, value, pe.appendPath(path, key.Value))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/yqlib/encoder_properties_test.go

@@ -0,0 +1,78 @@
+package yqlib
+
+import (
+	"bufio"
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+func yamlToProps(sampleYaml string) string {
+	var output bytes.Buffer
+	writer := bufio.NewWriter(&output)
+
+	var propsEncoder = NewPropertiesEncoder(writer)
+	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0)
+	if err != nil {
+		panic(err)
+	}
+	node := inputs.Front().Value.(*CandidateNode).Node
+	err = propsEncoder.Encode(node)
+	if err != nil {
+		panic(err)
+	}
+	writer.Flush()
+
+	return strings.TrimSuffix(output.String(), "\n")
+}
+
+func TestPropertiesEncoderSimple(t *testing.T) {
+	var sampleYaml = `a: 'bob cool'`
+
+	var expectedJson = `a = bob cool`
+	var actualProps = yamlToProps(sampleYaml)
+	test.AssertResult(t, expectedJson, actualProps)
+}
+
+func TestPropertiesEncoderSimpleWithComments(t *testing.T) {
+	var sampleYaml = `a: 'bob cool' # line`
+
+	var expectedJson = `# line
+a = bob cool`
+	var actualProps = yamlToProps(sampleYaml)
+	test.AssertResult(t, expectedJson, actualProps)
+}
+
+func TestPropertiesEncoderDeep(t *testing.T) {
+	var sampleYaml = `a: 
+  b: "bob cool"
+`
+
+	var expectedJson = `a.b = bob cool`
+	var actualProps = yamlToProps(sampleYaml)
+	test.AssertResult(t, expectedJson, actualProps)
+}
+
+func TestPropertiesEncoderDeepWithComments(t *testing.T) {
+	var sampleYaml = `a:  # a thing
+  b: "bob cool" # b thing
+`
+
+	var expectedJson = `# b thing
+a.b = bob cool`
+	var actualProps = yamlToProps(sampleYaml)
+	test.AssertResult(t, expectedJson, actualProps)
+}
+
+func TestPropertiesEncoderArray(t *testing.T) {
+	var sampleYaml = `a: 
+  b: [{c: dog}, {c: cat}]
+`
+
+	var expectedJson = `a.b.0.c = dog
+a.b.1.c = cat`
+	var actualProps = yamlToProps(sampleYaml)
+	test.AssertResult(t, expectedJson, actualProps)
+}

pkg/yqlib/expression_parser.go

@@ -5,9 +5,6 @@ import (
 	"strings"
 )
 
-var myPathTokeniser = newExpressionTokeniser()
-var myPathPostfixer = newExpressionPostFixer()
-
 type ExpressionNode struct {
 	Operation *Operation
 	Lhs       *ExpressionNode
@@ -19,19 +16,21 @@ type ExpressionParser interface {
 }
 
 type expressionParserImpl struct {
+	pathTokeniser expressionTokeniser
+	pathPostFixer expressionPostFixer
 }
 
 func NewExpressionParser() ExpressionParser {
-	return &expressionParserImpl{}
+	return &expressionParserImpl{newExpressionTokeniser(), newExpressionPostFixer()}
 }
 
 func (p *expressionParserImpl) ParseExpression(expression string) (*ExpressionNode, error) {
-	tokens, err := myPathTokeniser.Tokenise(expression)
+	tokens, err := p.pathTokeniser.Tokenise(expression)
 	if err != nil {
 		return nil, err
 	}
 	var Operations []*Operation
-	Operations, err = myPathPostfixer.ConvertToPostfix(tokens)
+	Operations, err = p.pathPostFixer.ConvertToPostfix(tokens)
 	if err != nil {
 		return nil, err
 	}

pkg/yqlib/expression_postfix.go

@@ -54,12 +54,26 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
 			opStack = opStack[0 : len(opStack)-1]
 			log.Debugf("deleteing open bracket from opstack")
 
-			//and append a collect to the opStack
-			result = append(result, &Operation{OperationType: collectOperator})
+			//and append a collect to the result
+			// hack - see if there's the optional traverse flag
+			// on the close op - move it to the collect op.
+			// allows for .["cat"]?
+			prefs := traversePreferences{}
+			closeTokenMatch := string(currentToken.Match.Bytes)
+			if closeTokenMatch[len(closeTokenMatch)-1:] == "?" {
+				prefs.OptionalTraverse = true
+			}
+			result = append(result, &Operation{OperationType: collectOperator, Preferences: prefs})
 			log.Debugf("put collect onto the result")
 			result = append(result, &Operation{OperationType: shortPipeOpType})
 			log.Debugf("put shortpipe onto the result")
 
+			//traverseArrayCollect is a sneaky op that needs to be included too
+			//when closing a []
+			if len(opStack) > 0 && opStack[len(opStack)-1].Operation != nil && opStack[len(opStack)-1].Operation.OperationType == traverseArrayOpType {
+				opStack, result = popOpToResult(opStack, result)
+			}
+
 		case closeBracket:
 			for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != openBracket {
 				opStack, result = popOpToResult(opStack, result)

pkg/yqlib/expression_processing_test.go

@@ -7,11 +7,64 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
+var variableWithNewLine = `"cat
+"`
+
 var pathTests = []struct {
 	path            string
 	expectedTokens  []interface{}
 	expectedPostFix []interface{}
 }{
+	{
+		"0x12",
+		append(make([]interface{}, 0), "18 (int64)"),
+		append(make([]interface{}, 0), "18 (int64)"),
+	},
+	{
+		"0X12",
+		append(make([]interface{}, 0), "18 (int64)"),
+		append(make([]interface{}, 0), "18 (int64)"),
+	},
+	{
+		".a\n",
+		append(make([]interface{}, 0), "a"),
+		append(make([]interface{}, 0), "a"),
+	},
+	{
+		variableWithNewLine,
+		append(make([]interface{}, 0), "cat\n (string)"),
+		append(make([]interface{}, 0), "cat\n (string)"),
+	},
+	{
+		`.[0]`,
+		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
+		append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
+	{
+		`.[0][1]`,
+		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]", "TRAVERSE_ARRAY", "[", "1 (int64)", "]"),
+		append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "1 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
+	{
+		`"\""`,
+		append(make([]interface{}, 0), "\" (string)"),
+		append(make([]interface{}, 0), "\" (string)"),
+	},
+	{
+		`[]|join(".")`,
+		append(make([]interface{}, 0), "[", "EMPTY", "]", "PIPE", "JOIN", "(", ". (string)", ")"),
+		append(make([]interface{}, 0), "EMPTY", "COLLECT", "SHORT_PIPE", ". (string)", "JOIN", "PIPE"),
+	},
+	{
+		`{"cool": .b or .c}`,
+		append(make([]interface{}, 0), "{", "cool (string)", "CREATE_MAP", "b", "OR", "c", "}"),
+		append(make([]interface{}, 0), "cool (string)", "b", "c", "OR", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"),
+	},
+	{
+		`{"cool": []|join(".")}`,
+		append(make([]interface{}, 0), "{", "cool (string)", "CREATE_MAP", "[", "EMPTY", "]", "PIPE", "JOIN", "(", ". (string)", ")", "}"),
+		append(make([]interface{}, 0), "cool (string)", "EMPTY", "COLLECT", "SHORT_PIPE", ". (string)", "JOIN", "PIPE", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"),
+	},
 	{
 		`.a as $item ireduce (0; . + $item)`, // note - add code to shuffle reduce to this position for postfix
 		append(make([]interface{}, 0), "a", "ASSIGN_VARIABLE", "GET_VARIABLE", "REDUCE", "(", "0 (int64)", "BLOCK", "SELF", "ADD", "GET_VARIABLE", ")"),
@@ -47,6 +100,11 @@ var pathTests = []struct {
 		append(make([]interface{}, 0), "b", "TRAVERSE_ARRAY", "[", "a", "]"),
 		append(make([]interface{}, 0), "b", "a", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
 	},
+	{
+		`.b[.a]?`,
+		append(make([]interface{}, 0), "b", "TRAVERSE_ARRAY", "[", "a", "]"),
+		append(make([]interface{}, 0), "b", "a", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
 	{
 		`.[]`,
 		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
@@ -57,6 +115,11 @@ var pathTests = []struct {
 		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
 		append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
 	},
+	{
+		`.a[]?`,
+		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
+		append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
 	{
 		`.a.[]`,
 		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
@@ -67,6 +130,11 @@ var pathTests = []struct {
 		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
 		append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
 	},
+	{
+		`.a[0]?`,
+		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
+		append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
 	{
 		`.a.[0]`,
 		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),

pkg/yqlib/expression_tokeniser.go

@@ -29,8 +29,9 @@ const (
 type token struct {
 	TokenType            tokenType
 	Operation            *Operation
-	AssignOperation      *Operation // e.g. tag (GetTag) op becomes AssignTag if '=' follows it
-	CheckForPostTraverse bool       // e.g. [1]cat should really be [1].cat
+	AssignOperation      *Operation      // e.g. tag (GetTag) op becomes AssignTag if '=' follows it
+	CheckForPostTraverse bool            // e.g. [1]cat should really be [1].cat
+	Match                *machines.Match // match that created this token
 
 }
 
@@ -63,12 +64,19 @@ func (t *token) toString(detail bool) string {
 func pathToken(wrapped bool) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		value := string(m.Bytes)
+		prefs := traversePreferences{}
+
+		if value[len(value)-1:] == "?" {
+			prefs.OptionalTraverse = true
+			value = value[:len(value)-1]
+		}
+
 		value = value[1:]
 		if wrapped {
 			value = unwrap(value)
 		}
 		log.Debug("PathToken %v", value)
-		op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: traversePreferences{}}
+		op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: prefs}
 		return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
 	}
 }
@@ -100,6 +108,9 @@ func multiplyWithPrefs() lex.Action {
 		if strings.Contains(options, "?") {
 			prefs.TraversePrefs = traversePreferences{DontAutoCreate: true}
 		}
+		if strings.Contains(options, "d") {
+			prefs.DeepMergeArrays = true
+		}
 		op := &Operation{OperationType: multiplyOpType, Value: multiplyOpType.Type, StringValue: options, Preferences: prefs}
 		return &token{TokenType: operationToken, Operation: op}, nil
 	}
@@ -135,7 +146,7 @@ func assignAllCommentsOp(updateAssign bool) lex.Action {
 
 func literalToken(pType tokenType, checkForPost bool) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &token{TokenType: pType, CheckForPostTraverse: checkForPost}, nil
+		return &token{TokenType: pType, CheckForPostTraverse: checkForPost, Match: m}, nil
 	}
 }
 
@@ -155,6 +166,20 @@ func numberValue() lex.Action {
 	}
 }
 
+func hexValue() lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		var originalString = string(m.Bytes)
+		var numberString = originalString[2:]
+		log.Debugf("numberString: %v", numberString)
+		var number, errParsingInt = strconv.ParseInt(numberString, 16, 64) // nolint
+		if errParsingInt != nil {
+			return nil, errParsingInt
+		}
+
+		return &token{TokenType: operationToken, Operation: createValueOperation(number, originalString)}, nil
+	}
+}
+
 func floatValue() lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		var numberString = string(m.Bytes)
@@ -178,6 +203,7 @@ func stringValue(wrapped bool) lex.Action {
 		if wrapped {
 			value = unwrap(value)
 		}
+		value = strings.ReplaceAll(value, "\\\"", "\"")
 		return &token{TokenType: operationToken, Operation: createValueOperation(value, value)}, nil
 	}
 }
@@ -186,7 +212,7 @@ func getVariableOpToken() lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		value := string(m.Bytes)
 
-		value = value[1 : len(value)-1]
+		value = value[1:]
 
 		getVarOperation := createValueOperation(value, value)
 		getVarOperation.OperationType = getVariableOpType
@@ -248,6 +274,8 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`sortKeys`), opToken(sortKeysOpType))
 	lexer.Add([]byte(`select`), opToken(selectOpType))
 	lexer.Add([]byte(`has`), opToken(hasOpType))
+	lexer.Add([]byte(`unique`), opToken(uniqueOpType))
+	lexer.Add([]byte(`unique_by`), opToken(uniqueByOpType))
 	lexer.Add([]byte(`explode`), opToken(explodeOpType))
 	lexer.Add([]byte(`or`), opToken(orOpType))
 	lexer.Add([]byte(`and`), opToken(andOpType))
@@ -261,6 +289,16 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`splitDoc`), opToken(splitDocumentOpType))
 
 	lexer.Add([]byte(`join`), opToken(joinStringOpType))
+	lexer.Add([]byte(`sub`), opToken(subStringOpType))
+	lexer.Add([]byte(`match`), opToken(matchOpType))
+	lexer.Add([]byte(`capture`), opToken(captureOpType))
+	lexer.Add([]byte(`test`), opToken(testOpType))
+
+	lexer.Add([]byte(`any`), opToken(anyOpType))
+	lexer.Add([]byte(`any_c`), opToken(anyConditionOpType))
+	lexer.Add([]byte(`all`), opToken(allOpType))
+	lexer.Add([]byte(`all_c`), opToken(allConditionOpType))
+
 	lexer.Add([]byte(`split`), opToken(splitStringOpType))
 	lexer.Add([]byte(`keys`), opToken(keysOpType))
 
@@ -273,6 +311,9 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`fileIndex`), opToken(getFileIndexOpType))
 	lexer.Add([]byte(`fi`), opToken(getFileIndexOpType))
 	lexer.Add([]byte(`path`), opToken(getPathOpType))
+	lexer.Add([]byte(`to_entries`), opToken(toEntriesOpType))
+	lexer.Add([]byte(`from_entries`), opToken(fromEntriesOpType))
+	lexer.Add([]byte(`with_entries`), opToken(withEntriesOpType))
 
 	lexer.Add([]byte(`lineComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}))
 
@@ -295,12 +336,13 @@ func initLexer() (*lex.Lexer, error) {
 
 	lexer.Add([]byte("( |\t|\n|\r)+"), skip)
 
-	lexer.Add([]byte(`\."[^ "]+"`), pathToken(true))
-	lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=]+`), pathToken(false))
+	lexer.Add([]byte(`\."[^ "]+"\??`), pathToken(true))
+	lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=\n]+\??`), pathToken(false))
 	lexer.Add([]byte(`\.`), selfToken())
 
 	lexer.Add([]byte(`\|`), opToken(pipeOpType))
 
+	lexer.Add([]byte(`0[xX][0-9A-Fa-f]+`), hexValue())
 	lexer.Add([]byte(`-?\d+(\.\d+)`), floatValue())
 	lexer.Add([]byte(`-?[1-9](\.\d+)?[Ee][-+]?\d+`), floatValue())
 	lexer.Add([]byte(`-?\d+`), numberValue())
@@ -311,21 +353,23 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`[Nn][Uu][Ll][Ll]`), nullValue())
 	lexer.Add([]byte(`~`), nullValue())
 
-	lexer.Add([]byte(`"[^"]*"`), stringValue(true))
+	lexer.Add([]byte(`"([^"\\]*(\\.[^"\\]*)*)"`), stringValue(true))
 	lexer.Add([]byte(`strenv\([^\)]+\)`), envOp(true))
 	lexer.Add([]byte(`env\([^\)]+\)`), envOp(false))
 
 	lexer.Add([]byte(`\[`), literalToken(openCollect, false))
-	lexer.Add([]byte(`\]`), literalToken(closeCollect, true))
+	lexer.Add([]byte(`\]\??`), literalToken(closeCollect, true))
 	lexer.Add([]byte(`\{`), literalToken(openCollectObject, false))
 	lexer.Add([]byte(`\}`), literalToken(closeCollectObject, true))
-	lexer.Add([]byte(`\*[\+|\?]*`), multiplyWithPrefs())
+	lexer.Add([]byte(`\*[\+|\?d]*`), multiplyWithPrefs())
 	lexer.Add([]byte(`\+`), opToken(addOpType))
 	lexer.Add([]byte(`\+=`), opToken(addAssignOpType))
+	lexer.Add([]byte(`\-`), opToken(subtractOpType))
+	lexer.Add([]byte(`\-=`), opToken(subtractAssignOpType))
 	lexer.Add([]byte(`\$[a-zA-Z_-0-9]+`), getVariableOpToken())
 	lexer.Add([]byte(`as`), opToken(assignVariableOpType))
 
-	err := lexer.Compile()
+	err := lexer.CompileNFA()
 	if err != nil {
 		return nil, err
 	}
@@ -385,16 +429,19 @@ func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postPr
 	skipNextToken = false
 	currentToken := tokens[index]
 
+	log.Debug("processing %v", currentToken.toString(true))
+
 	if currentToken.TokenType == traverseArrayCollect {
 		//need to put a traverse array then a collect currentToken
 		// do this by adding traverse then converting currentToken to collect
 
 		if index == 0 || tokens[index-1].TokenType != operationToken ||
 			tokens[index-1].Operation.OperationType != traversePathOpType {
+			log.Debug("  adding self")
 			op := &Operation{OperationType: selfReferenceOpType, StringValue: "SELF"}
 			postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 		}
-
+		log.Debug("  adding traverse array")
 		op := &Operation{OperationType: traverseArrayOpType, StringValue: "TRAVERSE_ARRAY"}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 
@@ -405,17 +452,19 @@ func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postPr
 	if index != len(tokens)-1 && currentToken.AssignOperation != nil &&
 		tokens[index+1].TokenType == operationToken &&
 		tokens[index+1].Operation.OperationType == assignOpType {
+		log.Debug("  its an update assign")
 		currentToken.Operation = currentToken.AssignOperation
 		currentToken.Operation.UpdateAssign = tokens[index+1].Operation.UpdateAssign
 		skipNextToken = true
 	}
 
+	log.Debug("  adding token to the fixed list")
 	postProcessedTokens = append(postProcessedTokens, currentToken)
 
 	if index != len(tokens)-1 &&
 		((currentToken.TokenType == openCollect && tokens[index+1].TokenType == closeCollect) ||
 			(currentToken.TokenType == openCollectObject && tokens[index+1].TokenType == closeCollectObject)) {
-
+		log.Debug("  adding empty")
 		op := &Operation{OperationType: emptyOpType, StringValue: "EMPTY"}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 	}
@@ -423,12 +472,19 @@ func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postPr
 	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
 		tokens[index+1].TokenType == operationToken &&
 		tokens[index+1].Operation.OperationType == traversePathOpType {
+		log.Debug("  adding pipe because the next thing is traverse")
 		op := &Operation{OperationType: shortPipeOpType, Value: "PIPE"}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 	}
 	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
 		tokens[index+1].TokenType == openCollect {
 
+		// if tokens[index].TokenType == closeCollect {
+		// 	log.Debug("  adding pipe because next is opencollect")
+		// 	op := &Operation{OperationType: shortPipeOpType, Value: "PIPE"}
+		// 	postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
+		// }
+		log.Debug("  adding traverArray because next is opencollect")
 		op := &Operation{OperationType: traverseArrayOpType}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 	}

pkg/yqlib/file_utils.go

@@ -2,6 +2,7 @@ package yqlib
 
 import (
 	"io"
+	"io/ioutil"
 	"os"
 )
 
@@ -23,9 +24,20 @@ func safelyRenameFile(from string, to string) {
 	}
 }
 
+func tryRemoveFile(filename string) {
+	log.Debug("Removing temp file: %v", filename)
+	removeErr := os.Remove(filename)
+	if removeErr != nil {
+		log.Errorf("Failed to remove temp file: %v", filename)
+	}
+}
+
 // thanks https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang
 func copyFileContents(src, dst string) (err error) {
-	in, err := os.Open(src) // nolint gosec
+	// ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory,
+	// and ensuring that it's not possible to give a path to a file outside thar directory.
+
+	in, err := os.Open(src) // #nosec
 	if err != nil {
 		return err
 	}
@@ -41,6 +53,13 @@ func copyFileContents(src, dst string) (err error) {
 	return out.Sync()
 }
 
+func SafelyCloseReader(reader io.Reader) {
+	switch reader := reader.(type) {
+	case *os.File:
+		safelyCloseFile(reader)
+	}
+}
+
 func safelyCloseFile(file *os.File) {
 	err := file.Close()
 	if err != nil {
@@ -48,3 +67,22 @@ func safelyCloseFile(file *os.File) {
 		log.Error(err.Error())
 	}
 }
+
+func createTempFile() (*os.File, error) {
+	_, err := os.Stat(os.TempDir())
+	if os.IsNotExist(err) {
+		err = os.Mkdir(os.TempDir(), 0700)
+		if err != nil {
+			return nil, err
+		}
+	} else if err != nil {
+		return nil, err
+	}
+
+	file, err := ioutil.TempFile("", "temp")
+	if err != nil {
+		return nil, err
+	}
+
+	return file, err
+}

pkg/yqlib/front_matter.go

@@ -0,0 +1,93 @@
+package yqlib
+
+import (
+	"bufio"
+	"io"
+	"os"
+)
+
+type frontMatterHandler interface {
+	Split() error
+	GetYamlFrontMatterFilename() string
+	GetContentReader() io.Reader
+	CleanUp()
+}
+
+type frontMatterHandlerImpl struct {
+	originalFilename        string
+	yamlFrontMatterFilename string
+	contentReader           io.Reader
+}
+
+func NewFrontMatterHandler(originalFilename string) frontMatterHandler {
+	return &frontMatterHandlerImpl{originalFilename, "", nil}
+}
+
+func (f *frontMatterHandlerImpl) GetYamlFrontMatterFilename() string {
+	return f.yamlFrontMatterFilename
+}
+
+func (f *frontMatterHandlerImpl) GetContentReader() io.Reader {
+	return f.contentReader
+}
+
+func (f *frontMatterHandlerImpl) CleanUp() {
+	tryRemoveFile(f.yamlFrontMatterFilename)
+}
+
+// Splits the given file by yaml front matter
+// yaml content will be saved to first temporary file
+// remaining content will be saved to second temporary file
+func (f *frontMatterHandlerImpl) Split() error {
+	var reader *bufio.Reader
+	var err error
+	if f.originalFilename == "-" {
+		reader = bufio.NewReader(os.Stdin)
+	} else {
+		file, err := os.Open(f.originalFilename) // #nosec
+		if err != nil {
+			return err
+		}
+		reader = bufio.NewReader(file)
+	}
+	f.contentReader = reader
+
+	yamlTempFile, err := createTempFile()
+	if err != nil {
+		return err
+	}
+	f.yamlFrontMatterFilename = yamlTempFile.Name()
+	log.Debug("yamlTempFile: %v", yamlTempFile.Name())
+
+	lineCount := 0
+
+	for {
+		peekBytes, err := reader.Peek(3)
+		if err == io.EOF {
+			// we've finished reading the yaml content..I guess
+			break
+		} else if err != nil {
+			return err
+		}
+		if lineCount > 0 && string(peekBytes) == "---" {
+			// we've finished reading the yaml content..
+			break
+		}
+		line, errReading := reader.ReadString('\n')
+		lineCount = lineCount + 1
+		if errReading != nil && errReading != io.EOF {
+			return errReading
+		}
+
+		_, errWriting := yamlTempFile.Write([]byte(line))
+
+		if errWriting != nil {
+			return errWriting
+		}
+	}
+
+	safelyCloseFile(yamlTempFile)
+
+	return nil
+
+}

pkg/yqlib/front_matter_test.go

@@ -0,0 +1,142 @@
+package yqlib
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+func createTestFile(content string) string {
+	tempFile, err := createTempFile()
+	if err != nil {
+		panic(err)
+	}
+
+	_, err = tempFile.Write([]byte(content))
+	if err != nil {
+		panic(err)
+	}
+
+	safelyCloseFile(tempFile)
+
+	return tempFile.Name()
+}
+
+func readFile(filename string) string {
+	bytes, err := ioutil.ReadFile(filename)
+	if err != nil {
+		panic(err)
+	}
+	return string(bytes)
+}
+
+func TestFrontMatterSplitWithLeadingSep(t *testing.T) {
+	file := createTestFile(`---
+a: apple
+b: banana
+---
+not a 
+yaml: doc
+`)
+
+	expectedYamlFm := `---
+a: apple
+b: banana
+`
+
+	expectedContent := `---
+not a 
+yaml: doc
+`
+
+	fmHandler := NewFrontMatterHandler(file)
+	err := fmHandler.Split()
+	if err != nil {
+		panic(err)
+	}
+
+	yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename())
+
+	test.AssertResult(t, expectedYamlFm, yamlFm)
+
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
+	if err != nil {
+		panic(err)
+	}
+	test.AssertResult(t, expectedContent, string(contentBytes))
+
+	tryRemoveFile(file)
+	fmHandler.CleanUp()
+}
+
+func TestFrontMatterSplitWithNoLeadingSep(t *testing.T) {
+	file := createTestFile(`a: apple
+b: banana
+---
+not a 
+yaml: doc
+`)
+
+	expectedYamlFm := `a: apple
+b: banana
+`
+
+	expectedContent := `---
+not a 
+yaml: doc
+`
+
+	fmHandler := NewFrontMatterHandler(file)
+	err := fmHandler.Split()
+	if err != nil {
+		panic(err)
+	}
+
+	yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename())
+
+	test.AssertResult(t, expectedYamlFm, yamlFm)
+
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
+	if err != nil {
+		panic(err)
+	}
+	test.AssertResult(t, expectedContent, string(contentBytes))
+
+	tryRemoveFile(file)
+	fmHandler.CleanUp()
+}
+
+func TestFrontMatterSplitWithArray(t *testing.T) {
+	file := createTestFile(`[1,2,3]
+---
+not a 
+yaml: doc
+`)
+
+	expectedYamlFm := "[1,2,3]\n"
+
+	expectedContent := `---
+not a 
+yaml: doc
+`
+
+	fmHandler := NewFrontMatterHandler(file)
+	err := fmHandler.Split()
+	if err != nil {
+		panic(err)
+	}
+
+	yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename())
+
+	test.AssertResult(t, expectedYamlFm, yamlFm)
+
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
+	if err != nil {
+		panic(err)
+	}
+	test.AssertResult(t, expectedContent, string(contentBytes))
+
+	tryRemoveFile(file)
+	fmHandler.CleanUp()
+}

pkg/yqlib/lib.go

@@ -6,6 +6,8 @@ import (
 	"bytes"
 	"container/list"
 	"fmt"
+	"strconv"
+	"strings"
 
 	logging "gopkg.in/op/go-logging.v1"
 	yaml "gopkg.in/yaml.v3"
@@ -20,12 +22,9 @@ type operationType struct {
 	Handler    operatorHandler
 }
 
-// operators TODO:
-// - mergeEmpty (sets only if the document is empty, do I do that now?)
-
 var orOpType = &operationType{Type: "OR", NumArgs: 2, Precedence: 20, Handler: orOperator}
 var andOpType = &operationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: andOperator}
-var reduceOpType = &operationType{Type: "REDUCE", NumArgs: 2, Precedence: 5, Handler: reduceOperator}
+var reduceOpType = &operationType{Type: "REDUCE", NumArgs: 2, Precedence: 35, Handler: reduceOperator}
 
 var blockOpType = &operationType{Type: "BLOCK", Precedence: 10, NumArgs: 2, Handler: emptyOperator}
 
@@ -35,6 +34,7 @@ var pipeOpType = &operationType{Type: "PIPE", NumArgs: 2, Precedence: 30, Handle
 
 var assignOpType = &operationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: assignUpdateOperator}
 var addAssignOpType = &operationType{Type: "ADD_ASSIGN", NumArgs: 2, Precedence: 40, Handler: addAssignOperator}
+var subtractAssignOpType = &operationType{Type: "SUBTRACT_ASSIGN", NumArgs: 2, Precedence: 40, Handler: subtractAssignOperator}
 
 var assignAttributesOpType = &operationType{Type: "ASSIGN_ATTRIBUTES", NumArgs: 2, Precedence: 40, Handler: assignAttributesOperator}
 var assignStyleOpType = &operationType{Type: "ASSIGN_STYLE", NumArgs: 2, Precedence: 40, Handler: assignStyleOperator}
@@ -46,16 +46,29 @@ var assignAliasOpType = &operationType{Type: "ASSIGN_ALIAS", NumArgs: 2, Precede
 
 var multiplyOpType = &operationType{Type: "MULTIPLY", NumArgs: 2, Precedence: 42, Handler: multiplyOperator}
 var addOpType = &operationType{Type: "ADD", NumArgs: 2, Precedence: 42, Handler: addOperator}
+var subtractOpType = &operationType{Type: "SUBTRACT", NumArgs: 2, Precedence: 42, Handler: subtractOperator}
 var alternativeOpType = &operationType{Type: "ALTERNATIVE", NumArgs: 2, Precedence: 42, Handler: alternativeOperator}
 
 var equalsOpType = &operationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: equalsOperator}
 var notEqualsOpType = &operationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: notEqualsOperator}
-var createMapOpType = &operationType{Type: "CREATE_MAP", NumArgs: 2, Precedence: 40, Handler: createMapOperator}
+
+//createmap needs to be above union, as we use union to build the components of the objects
+var createMapOpType = &operationType{Type: "CREATE_MAP", NumArgs: 2, Precedence: 15, Handler: createMapOperator}
 
 var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence: 45, Handler: pipeOperator}
 
 var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator}
 var collectOpType = &operationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: collectOperator}
+
+var anyOpType = &operationType{Type: "ANY", NumArgs: 0, Precedence: 50, Handler: anyOperator}
+var allOpType = &operationType{Type: "ALL", NumArgs: 0, Precedence: 50, Handler: allOperator}
+var anyConditionOpType = &operationType{Type: "ANY_CONDITION", NumArgs: 1, Precedence: 50, Handler: anyOperator}
+var allConditionOpType = &operationType{Type: "ALL_CONDITION", NumArgs: 1, Precedence: 50, Handler: allOperator}
+
+var toEntriesOpType = &operationType{Type: "TO_ENTRIES", NumArgs: 0, Precedence: 50, Handler: toEntriesOperator}
+var fromEntriesOpType = &operationType{Type: "FROM_ENTRIES", NumArgs: 0, Precedence: 50, Handler: fromEntriesOperator}
+var withEntriesOpType = &operationType{Type: "WITH_ENTRIES", NumArgs: 1, Precedence: 50, Handler: withEntriesOperator}
+
 var splitDocumentOpType = &operationType{Type: "SPLIT_DOC", NumArgs: 0, Precedence: 50, Handler: splitDocumentOperator}
 var getVariableOpType = &operationType{Type: "GET_VARIABLE", NumArgs: 0, Precedence: 55, Handler: getVariableOperator}
 var getStyleOpType = &operationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: getStyleOperator}
@@ -71,6 +84,10 @@ var getPathOpType = &operationType{Type: "GET_PATH", NumArgs: 0, Precedence: 50,
 var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: explodeOperator}
 var sortKeysOpType = &operationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: sortKeysOperator}
 var joinStringOpType = &operationType{Type: "JOIN", NumArgs: 1, Precedence: 50, Handler: joinStringOperator}
+var subStringOpType = &operationType{Type: "SUBSTR", NumArgs: 1, Precedence: 50, Handler: substituteStringOperator}
+var matchOpType = &operationType{Type: "MATCH", NumArgs: 1, Precedence: 50, Handler: matchOperator}
+var captureOpType = &operationType{Type: "CAPTURE", NumArgs: 1, Precedence: 50, Handler: captureOperator}
+var testOpType = &operationType{Type: "TEST", NumArgs: 1, Precedence: 50, Handler: testOperator}
 var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 50, Handler: splitStringOperator}
 
 var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 50, Handler: keysOperator}
@@ -89,8 +106,9 @@ var recursiveDescentOpType = &operationType{Type: "RECURSIVE_DESCENT", NumArgs:
 
 var selectOpType = &operationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Handler: selectOperator}
 var hasOpType = &operationType{Type: "HAS", NumArgs: 1, Precedence: 50, Handler: hasOperator}
+var uniqueOpType = &operationType{Type: "UNIQUE", NumArgs: 0, Precedence: 50, Handler: unique}
+var uniqueByOpType = &operationType{Type: "UNIQUE_BY", NumArgs: 1, Precedence: 50, Handler: uniqueBy}
 var deleteChildOpType = &operationType{Type: "DELETE", NumArgs: 1, Precedence: 40, Handler: deleteChildOperator}
-var deleteImmediateChildOpType = &operationType{Type: "DELETE_IMMEDIATE_CHILD", NumArgs: 1, Precedence: 40, Handler: deleteImmediateChildOperator}
 
 type Operation struct {
 	OperationType *operationType
@@ -101,8 +119,77 @@ type Operation struct {
 	UpdateAssign  bool // used for assign ops, when true it means we evaluate the rhs given the lhs
 }
 
-func createValueOperation(value interface{}, stringValue string) *Operation {
-	var node yaml.Node = yaml.Node{Kind: yaml.ScalarNode}
+func recurseNodeArrayEqual(lhs *yaml.Node, rhs *yaml.Node) bool {
+	if len(lhs.Content) != len(rhs.Content) {
+		return false
+	}
+
+	for index := 0; index < len(lhs.Content); index = index + 1 {
+		if !recursiveNodeEqual(lhs.Content[index], rhs.Content[index]) {
+			return false
+		}
+	}
+	return true
+}
+
+func findInArray(array *yaml.Node, item *yaml.Node) int {
+
+	for index := 0; index < len(array.Content); index = index + 1 {
+		if recursiveNodeEqual(array.Content[index], item) {
+			return index
+		}
+	}
+	return -1
+}
+
+func recurseNodeObjectEqual(lhs *yaml.Node, rhs *yaml.Node) bool {
+	if len(lhs.Content) != len(rhs.Content) {
+		return false
+	}
+
+	for index := 0; index < len(lhs.Content); index = index + 2 {
+		key := lhs.Content[index]
+		value := lhs.Content[index+1]
+
+		indexInRhs := findInArray(rhs, key)
+
+		if indexInRhs == -1 || !recursiveNodeEqual(value, rhs.Content[indexInRhs+1]) {
+			return false
+		}
+	}
+	return true
+}
+
+func recursiveNodeEqual(lhs *yaml.Node, rhs *yaml.Node) bool {
+	if lhs.Kind != rhs.Kind || lhs.Tag != rhs.Tag {
+		return false
+	} else if lhs.Tag == "!!null" {
+		return true
+
+	} else if lhs.Kind == yaml.ScalarNode {
+		return lhs.Value == rhs.Value
+	} else if lhs.Kind == yaml.SequenceNode {
+		return recurseNodeArrayEqual(lhs, rhs)
+	} else if lhs.Kind == yaml.MappingNode {
+		return recurseNodeObjectEqual(lhs, rhs)
+	}
+	return false
+
+}
+
+// yaml numbers can be hex encoded...
+func parseInt(numberString string) (string, int64, error) {
+	if strings.HasPrefix(numberString, "0x") ||
+		strings.HasPrefix(numberString, "0X") {
+		num, err := strconv.ParseInt(numberString[2:], 16, 64) // nolint
+		return "0x%X", num, err
+	}
+	num, err := strconv.ParseInt(numberString, 10, 64) // nolint
+	return "%v", num, err
+}
+
+func createScalarNode(value interface{}, stringValue string) *yaml.Node {
+	var node = &yaml.Node{Kind: yaml.ScalarNode}
 	node.Value = stringValue
 
 	switch value.(type) {
@@ -117,12 +204,17 @@ func createValueOperation(value interface{}, stringValue string) *Operation {
 	case nil:
 		node.Tag = "!!null"
 	}
+	return node
+}
+
+func createValueOperation(value interface{}, stringValue string) *Operation {
+	var node *yaml.Node = createScalarNode(value, stringValue)
 
 	return &Operation{
 		OperationType: valueOpType,
 		Value:         value,
 		StringValue:   stringValue,
-		CandidateNode: &CandidateNode{Node: &node},
+		CandidateNode: &CandidateNode{Node: node},
 	}
 }
 
@@ -145,7 +237,7 @@ func NodesToString(collection *list.List) string {
 		return ""
 	}
 
-	result := ""
+	result := fmt.Sprintf("%v results\n", collection.Len())
 	for el := collection.Front(); el != nil; el = el.Next() {
 		result = result + "\n" + NodeToString(el.Value.(*CandidateNode))
 	}
@@ -166,7 +258,10 @@ func NodeToString(node *CandidateNode) string {
 	if errorEncoding != nil {
 		log.Error("Error debugging node, %v", errorEncoding.Error())
 	}
-	encoder.Close()
+	errorClosingEncoder := encoder.Close()
+	if errorClosingEncoder != nil {
+		log.Error("Error closing encoder: ", errorClosingEncoder.Error())
+	}
 	tag := value.Tag
 	if value.Kind == yaml.DocumentNode {
 		tag = "doc"

pkg/yqlib/operator_add.go

@@ -15,11 +15,7 @@ func createAddOp(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode {
 }
 
 func addAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	assignmentOp := &Operation{OperationType: assignOpType}
-	assignmentOp.UpdateAssign = false
-
-	assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: expressionNode.Lhs, Rhs: createAddOp(expressionNode.Lhs, expressionNode.Rhs)}
-	return d.GetMatchingNodes(context, assignmentOpNode)
+	return compoundAssignFunction(d, context, expressionNode, createAddOp)
 }
 
 func toNodes(candidate *CandidateNode) []*yaml.Node {
@@ -39,7 +35,7 @@ func toNodes(candidate *CandidateNode) []*yaml.Node {
 func addOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("Add operator")
 
-	return crossFunction(d, context, expressionNode, add)
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, add, false)
 }
 
 func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
@@ -80,17 +76,17 @@ func addScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*Candida
 		target.Node.Tag = "!!str"
 		target.Node.Value = lhs.Value + rhs.Value
 	} else if lhs.Tag == "!!int" && rhs.Tag == "!!int" {
-		lhsNum, err := strconv.Atoi(lhs.Value)
+		format, lhsNum, err := parseInt(lhs.Value)
 		if err != nil {
 			return nil, err
 		}
-		rhsNum, err := strconv.Atoi(rhs.Value)
+		_, rhsNum, err := parseInt(rhs.Value)
 		if err != nil {
 			return nil, err
 		}
 		sum := lhsNum + rhsNum
 		target.Node.Tag = "!!int"
-		target.Node.Value = fmt.Sprintf("%v", sum)
+		target.Node.Value = fmt.Sprintf(format, sum)
 	} else if (lhs.Tag == "!!int" || lhs.Tag == "!!float") && (rhs.Tag == "!!int" || rhs.Tag == "!!float") {
 		lhsNum, err := strconv.ParseFloat(lhs.Value, 64)
 		if err != nil {

pkg/yqlib/operator_add_test.go

@@ -14,6 +14,22 @@ var addOperatorScenarios = []expressionScenario{
 			"D0, P[1 a], (!!int)::3\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `{}`,
+		expression: "(.a + .b) as $x",
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: 0`,
+		expression: ".a += .b.c",
+		expected: []string{
+			"D0, P[], (doc)::a: 0\n",
+		},
+	},
 	{
 		description: "Concatenate and assign arrays",
 		document:    `{a: {val: thing, b: [cat,dog]}}`,
@@ -63,7 +79,7 @@ var addOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description: "Update array (append)",
+		description: "Append to array",
 		document:    `{a: [1,2], b: [3,4]}`,
 		expression:  `.a = .a + .b`,
 		expected: []string{
@@ -71,17 +87,25 @@ var addOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description: "String concatenation",
-		document:    `{a: cat, b: meow}`,
-		expression:  `.a = .a + .b`,
+		description: "Append another array using +=",
+		document:    `{a: [1,2], b: [3,4]}`,
+		expression:  `.a += .b`,
 		expected: []string{
-			"D0, P[], (doc)::{a: catmeow, b: meow}\n",
+			"D0, P[], (doc)::{a: [1, 2, 3, 4], b: [3, 4]}\n",
 		},
 	},
 	{
-		description: "Relative string concatenation",
+		description: "Relative append",
+		document:    `a: { a1: {b: [cat]}, a2: {b: [dog]}, a3: {} }`,
+		expression:  `.a[].b += ["mouse"]`,
+		expected: []string{
+			"D0, P[], (doc)::a: {a1: {b: [cat, mouse]}, a2: {b: [dog, mouse]}, a3: {b: [mouse]}}\n",
+		},
+	},
+	{
+		description: "String concatenation",
 		document:    `{a: cat, b: meow}`,
-		expression:  `.a += .b`,
+		expression:  `.a = .a + .b`,
 		expected: []string{
 			"D0, P[], (doc)::{a: catmeow, b: meow}\n",
 		},
@@ -105,11 +129,11 @@ var addOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description: "Increment number",
-		document:    `{a: 3}`,
-		expression:  `.a += 1`,
+		description: "Increment numbers",
+		document:    `{a: 3, b: 5}`,
+		expression:  `.[] += 1`,
 		expected: []string{
-			"D0, P[], (doc)::{a: 4}\n",
+			"D0, P[], (doc)::{a: 4, b: 6}\n",
 		},
 	},
 	{

pkg/yqlib/operator_alternative.go

@@ -1,14 +1,17 @@
 package yqlib
 
-// corssFunction no matches
-// can boolean use crossfunction
-
 func alternativeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- alternative")
-	return crossFunction(d, context, expressionNode, alternativeFunc)
+	return crossFunction(d, context, expressionNode, alternativeFunc, true)
 }
 
 func alternativeFunc(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	if lhs == nil {
+		return rhs, nil
+	}
+	if rhs == nil {
+		return lhs, nil
+	}
 	lhs.Node = unwrapDoc(lhs.Node)
 	rhs.Node = unwrapDoc(rhs.Node)
 	log.Debugf("Alternative LHS: %v", lhs.Node.Tag)

pkg/yqlib/operator_alternative_test.go

@@ -5,6 +5,31 @@ import (
 )
 
 var alternativeOperatorScenarios = []expressionScenario{
+	{
+		// to match jq - we do not use a readonly clone context on the LHS.
+		skipDoc:    true,
+		expression: `.b // .c`,
+		document:   `a: bridge`,
+		expected: []string{
+			"D0, P[c], (!!null)::null\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `(.b // "hello") as $x`,
+		document:   `a: bridge`,
+		expected: []string{
+			"D0, P[], (doc)::a: bridge\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `.a // .b`,
+		document:   `a: 2`,
+		expected: []string{
+			"D0, P[a], (!!int)::2\n",
+		},
+	},
 	{
 		description: "LHS is defined",
 		expression:  `.a // "hello"`,
@@ -13,6 +38,14 @@ var alternativeOperatorScenarios = []expressionScenario{
 			"D0, P[a], (!!str)::bridge\n",
 		},
 	},
+	{
+		expression: `select(tag == "seq") // "cat"`,
+		skipDoc:    true,
+		document:   `a: frog`,
+		expected: []string{
+			"D0, P[], (!!str)::cat\n",
+		},
+	},
 	{
 		description: "LHS is not defined",
 		expression:  `.a // "hello"`,

pkg/yqlib/operator_anchors_aliases.go

@@ -12,7 +12,7 @@ func assignAliasOperator(d *dataTreeNavigator, context Context, expressionNode *
 
 	aliasName := ""
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 		if err != nil {
 			return Context{}, err
 		}
@@ -32,7 +32,7 @@ func assignAliasOperator(d *dataTreeNavigator, context Context, expressionNode *
 		log.Debugf("Setting aliasName : %v", candidate.GetKey())
 
 		if expressionNode.Operation.UpdateAssign {
-			rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 			if err != nil {
 				return Context{}, err
 			}
@@ -41,8 +41,10 @@ func assignAliasOperator(d *dataTreeNavigator, context Context, expressionNode *
 			}
 		}
 
-		candidate.Node.Kind = yaml.AliasNode
-		candidate.Node.Value = aliasName
+		if aliasName != "" {
+			candidate.Node.Kind = yaml.AliasNode
+			candidate.Node.Value = aliasName
+		}
 	}
 	return context, nil
 }
@@ -66,7 +68,7 @@ func assignAnchorOperator(d *dataTreeNavigator, context Context, expressionNode
 
 	anchorName := ""
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 		if err != nil {
 			return Context{}, err
 		}
@@ -87,7 +89,7 @@ func assignAnchorOperator(d *dataTreeNavigator, context Context, expressionNode
 		log.Debugf("Setting anchorName of : %v", candidate.GetKey())
 
 		if expressionNode.Operation.UpdateAssign {
-			rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 			if err != nil {
 				return Context{}, err
 			}
@@ -176,7 +178,7 @@ func explodeNode(node *yaml.Node, context Context) error {
 			} else {
 				if valueNode.Kind == yaml.SequenceNode {
 					log.Debugf("an alias merge list!")
-					for index := 0; index < len(valueNode.Content); index = index + 1 {
+					for index := len(valueNode.Content) - 1; index >= 0; index = index - 1 {
 						aliasNode := valueNode.Content[index]
 						err := applyAlias(node, aliasNode.Alias, index, context.ChildContext(newContent))
 						if err != nil {

pkg/yqlib/operator_anchors_aliases_test.go

@@ -4,7 +4,59 @@ import (
 	"testing"
 )
 
+var specDocument = `- &CENTER { x: 1, y: 2 }
+- &LEFT { x: 0, y: 2 }
+- &BIG { r: 10 }
+- &SMALL { r: 1 }
+`
+
+var expectedSpecResult = "D0, P[4], (!!map)::x: 1\ny: 2\nr: 10\n"
+
+var simpleArrayRef = `
+item_value: &item_value
+  value: true
+
+thingOne:
+  name: item_1
+  <<: *item_value
+
+thingTwo:
+  name: item_2
+  <<: *item_value
+`
+
+var expectedUpdatedArrayRef = `D0, P[], (doc)::item_value: &item_value
+    value: true
+thingOne:
+    name: item_1
+    value: false
+thingTwo:
+    name: item_2
+    !!merge <<: *item_value
+`
+
 var anchorOperatorScenarios = []expressionScenario{
+	{
+		description:    "Merge one map",
+		subdescription: "see https://yaml.org/type/merge.html",
+		document:       specDocument + "- << : *CENTER\n  r: 10\n",
+		expression:     ".[4] | explode(.)",
+		expected:       []string{expectedSpecResult},
+	},
+	{
+		description:    "Merge multiple maps",
+		subdescription: "see https://yaml.org/type/merge.html",
+		document:       specDocument + "- << : [ *CENTER, *BIG ]\n",
+		expression:     ".[4] | explode(.)",
+		expected:       []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"},
+	},
+	{
+		description:    "Override",
+		subdescription: "see https://yaml.org/type/merge.html",
+		document:       specDocument + "- << : [ *BIG, *LEFT, *SMALL ]\n  x: 1\n",
+		expression:     ".[4] | explode(.)",
+		expected:       []string{"D0, P[4], (!!map)::r: 10\nx: 1\ny: 2\n"},
+	},
 	{
 		description: "Get anchor",
 		document:    `a: &billyBob cat`,
@@ -29,6 +81,22 @@ var anchorOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: &cat {b: cat}\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `a: {c: cat}`,
+		expression: `.a anchor |= .b`,
+		expected: []string{
+			"D0, P[], (doc)::a: {c: cat}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: {c: cat}`,
+		expression: `.a anchor = .b`,
+		expected: []string{
+			"D0, P[], (doc)::a: {c: cat}\n",
+		},
+	},
 	{
 		description: "Get alias",
 		document:    `{b: &billyBob meow, a: *billyBob}`,
@@ -45,6 +113,30 @@ var anchorOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{b: &meow purr, a: *meow}\n",
 		},
 	},
+	{
+		description: "Set alias to blank does nothing",
+		document:    `{b: &meow purr, a: cat}`,
+		expression:  `.a alias = ""`,
+		expected: []string{
+			"D0, P[], (doc)::{b: &meow purr, a: cat}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{b: &meow purr, a: cat}`,
+		expression: `.a alias = .c`,
+		expected: []string{
+			"D0, P[], (doc)::{b: &meow purr, a: cat}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{b: &meow purr, a: cat}`,
+		expression: `.a alias |= .c`,
+		expected: []string{
+			"D0, P[], (doc)::{b: &meow purr, a: cat}\n",
+		},
+	},
 	{
 		description: "Set alias relatively using assign-update",
 		document:    `{b: &meow purr, a: {f: meow}}`,
@@ -91,9 +183,9 @@ bar:
     c: bar_c
 foobarList:
     b: bar_b
-    a: foo_a
-    thing: bar_thing
+    thing: foo_thing
     c: foobarList_c
+    a: foo_a
 foobar:
     c: foo_c
     a: foo_a
@@ -106,7 +198,7 @@ foobar:
 		expression: `.foo* | explode(.) | (. style="flow")`,
 		expected: []string{
 			"D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n",
-			"D0, P[foobarList], (!!map)::{b: bar_b, a: foo_a, thing: bar_thing, c: foobarList_c}\n",
+			"D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n",
 			"D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n",
 		},
 	},
@@ -116,7 +208,7 @@ foobar:
 		expression: `.foo* | explode(explode(.)) | (. style="flow")`,
 		expected: []string{
 			"D0, P[foo], (!!map)::{a: foo_a, thing: foo_thing, c: foo_c}\n",
-			"D0, P[foobarList], (!!map)::{b: bar_b, a: foo_a, thing: bar_thing, c: foobarList_c}\n",
+			"D0, P[foobarList], (!!map)::{b: bar_b, thing: foo_thing, c: foobarList_c, a: foo_a}\n",
 			"D0, P[foobar], (!!map)::{c: foo_c, a: foo_a, thing: foobar_thing}\n",
 		},
 	},
@@ -128,9 +220,16 @@ foobar:
 			"D0, P[], (doc)::{f: {a: cat, b: {f: cat}, cat: {f: cat}}}\n",
 		},
 	},
+	{
+		description:    "Dereference and update a field",
+		subdescription: "`Use explode with multiply to dereference an object",
+		document:       simpleArrayRef,
+		expression:     `.thingOne |= explode(.) * {"value": false}`,
+		expected:       []string{expectedUpdatedArrayRef},
+	},
 }
 
-func TestAnchorAliaseOperatorScenarios(t *testing.T) {
+func TestAnchorAliasOperatorScenarios(t *testing.T) {
 	for _, tt := range anchorOperatorScenarios {
 		testScenario(t, &tt)
 	}

pkg/yqlib/operator_assign.go

@@ -1,21 +1,27 @@
 package yqlib
 
+func assignUpdateFunc(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	rhs.Node = unwrapDoc(rhs.Node)
+	lhs.UpdateFrom(rhs)
+	return lhs, nil
+}
+
 func assignUpdateOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	lhs, err := d.GetMatchingNodes(context, expressionNode.Lhs)
 	if err != nil {
 		return Context{}, err
 	}
-	var rhs Context
+
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err = d.GetMatchingNodes(context, expressionNode.Rhs)
+		// this works because we already ran against LHS with an editable context.
+		_, err := crossFunction(d, context.ReadOnlyClone(), expressionNode, assignUpdateFunc, false)
+		return context, err
 	}
 
 	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 
-		if expressionNode.Operation.UpdateAssign {
-			rhs, err = d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
-		}
+		rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
 
 		if err != nil {
 			return Context{}, err
@@ -44,7 +50,7 @@ func assignAttributesOperator(d *dataTreeNavigator, context Context, expressionN
 	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 
-		rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 
 		if err != nil {
 			return Context{}, err

pkg/yqlib/operator_assign_test.go

@@ -12,6 +12,22 @@ var assignOperatorScenarios = []expressionScenario{
 			"D0, P[], ()::a:\n    b: cat\nx: frog\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: `.a |= .b`,
+		expected: []string{
+			"D0, P[], (doc)::{a: null}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: `.a = .b`,
+		expected: []string{
+			"D0, P[], (doc)::{a: null}\n",
+		},
+	},
 	{
 		description: "Update node to be the child value",
 		document:    `{a: {b: {g: foof}}}`,
@@ -20,6 +36,14 @@ var assignOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: {g: foof}}\n",
 		},
 	},
+	{
+		description: "Double elements in an array",
+		document:    `[1,2,3]`,
+		expression:  `.[] |= . * 2`,
+		expected: []string{
+			"D0, P[], (doc)::[2, 4, 6]\n",
+		},
+	},
 	{
 		description:    "Update node from another file",
 		subdescription: "Note this will also work when the second file is a scalar (string/number)",
@@ -41,7 +65,7 @@ var assignOperatorScenarios = []expressionScenario{
 	{
 		description: "Updated multiple paths",
 		document:    `{a: fieldA, b: fieldB, c: fieldC}`,
-		expression:  `(.a, .c) |= "potatoe"`,
+		expression:  `(.a, .c) = "potatoe"`,
 		expected: []string{
 			"D0, P[], (doc)::{a: potatoe, b: fieldB, c: potatoe}\n",
 		},
@@ -143,5 +167,5 @@ func TestAssignOperatorScenarios(t *testing.T) {
 	for _, tt := range assignOperatorScenarios {
 		testScenario(t, &tt)
 	}
-	documentScenarios(t, "Assign", assignOperatorScenarios)
+	documentScenarios(t, "Assign (Update)", assignOperatorScenarios)
 }

pkg/yqlib/operator_booleans.go

@@ -2,14 +2,13 @@ package yqlib
 
 import (
 	"container/list"
+	"fmt"
 
 	yaml "gopkg.in/yaml.v3"
 )
 
-func isTruthy(c *CandidateNode) (bool, error) {
-	node := unwrapDoc(c.Node)
+func isTruthyNode(node *yaml.Node) (bool, error) {
 	value := true
-
 	if node.Tag == "!!null" {
 		return false, nil
 	}
@@ -23,41 +22,131 @@ func isTruthy(c *CandidateNode) (bool, error) {
 	return value, nil
 }
 
+func isTruthy(c *CandidateNode) (bool, error) {
+	node := unwrapDoc(c.Node)
+	return isTruthyNode(node)
+}
+
 type boolOp func(bool, bool) bool
 
 func performBoolOp(op boolOp) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-		lhs.Node = unwrapDoc(lhs.Node)
-		rhs.Node = unwrapDoc(rhs.Node)
+		owner := lhs
 
-		lhsTrue, errDecoding := isTruthy(lhs)
-		if errDecoding != nil {
-			return nil, errDecoding
+		if lhs == nil && rhs == nil {
+			owner = &CandidateNode{}
+		} else if lhs == nil {
+			owner = rhs
 		}
 
-		rhsTrue, errDecoding := isTruthy(rhs)
-		if errDecoding != nil {
-			return nil, errDecoding
-		}
+		var errDecoding error
+		lhsTrue := false
+		if lhs != nil {
+			lhs.Node = unwrapDoc(lhs.Node)
+			lhsTrue, errDecoding = isTruthy(lhs)
 
-		return createBooleanCandidate(lhs, op(lhsTrue, rhsTrue)), nil
+			if errDecoding != nil {
+				return nil, errDecoding
+			}
+		}
+		log.Debugf("-- lhsTrue", lhsTrue)
+
+		rhsTrue := false
+		if rhs != nil {
+			rhs.Node = unwrapDoc(rhs.Node)
+			rhsTrue, errDecoding = isTruthy(rhs)
+			if errDecoding != nil {
+				return nil, errDecoding
+			}
+		}
+		log.Debugf("-- rhsTrue", rhsTrue)
+
+		return createBooleanCandidate(owner, op(lhsTrue, rhsTrue)), nil
 	}
 }
 
+func findBoolean(wantBool bool, d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, sequenceNode *yaml.Node) (bool, error) {
+	for _, node := range sequenceNode.Content {
+
+		if expressionNode != nil {
+			//need to evaluate the expression against the node
+			candidate := &CandidateNode{Node: node}
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode)
+			if err != nil {
+				return false, err
+			}
+			if rhs.MatchingNodes.Len() > 0 {
+				node = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node
+			} else {
+				// no results found, ignore this entry
+				continue
+			}
+		}
+
+		truthy, err := isTruthyNode(node)
+		if err != nil {
+			return false, err
+		}
+		if truthy == wantBool {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func allOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		candidateNode := unwrapDoc(candidate.Node)
+		if candidateNode.Kind != yaml.SequenceNode {
+			return Context{}, fmt.Errorf("any only supports arrays, was %v", candidateNode.Tag)
+		}
+		booleanResult, err := findBoolean(false, d, context, expressionNode.Rhs, candidateNode)
+		if err != nil {
+			return Context{}, err
+		}
+		result := createBooleanCandidate(candidate, !booleanResult)
+		results.PushBack(result)
+	}
+	return context.ChildContext(results), nil
+}
+
+func anyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		candidateNode := unwrapDoc(candidate.Node)
+		if candidateNode.Kind != yaml.SequenceNode {
+			return Context{}, fmt.Errorf("any only supports arrays, was %v", candidateNode.Tag)
+		}
+		booleanResult, err := findBoolean(true, d, context, expressionNode.Rhs, candidateNode)
+		if err != nil {
+			return Context{}, err
+		}
+		result := createBooleanCandidate(candidate, booleanResult)
+		results.PushBack(result)
+	}
+	return context.ChildContext(results), nil
+}
+
 func orOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- orOp")
-	return crossFunction(d, context, expressionNode, performBoolOp(
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, performBoolOp(
 		func(b1 bool, b2 bool) bool {
+			log.Debugf("-- peformingOrOp with %v and %v", b1, b2)
 			return b1 || b2
-		}))
+		}), true)
 }
 
 func andOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- AndOp")
-	return crossFunction(d, context, expressionNode, performBoolOp(
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, performBoolOp(
 		func(b1 bool, b2 bool) bool {
 			return b1 && b2
-		}))
+		}), true)
 }
 
 func notOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {

pkg/yqlib/operator_booleans_test.go

@@ -6,14 +6,46 @@ import (
 
 var booleanOperatorScenarios = []expressionScenario{
 	{
-		description: "OR example",
+		description: "`or` example",
 		expression:  `true or false`,
 		expected: []string{
 			"D0, P[], (!!bool)::true\n",
 		},
 	},
 	{
-		description: "AND example",
+		skipDoc:    true,
+		document:   "b: hi",
+		expression: `.a or .c`,
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "b: false",
+		expression: `.b or .c`,
+		expected: []string{
+			"D0, P[b], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "b: hi",
+		expression: `select(.a or .b)`,
+		expected: []string{
+			"D0, P[], (doc)::b: hi\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "b: hi",
+		expression: `select((.a and .b) | not)`,
+		expected: []string{
+			"D0, P[], (doc)::b: hi\n",
+		},
+	},
+	{
+		description: "`and` example",
 		expression:  `true and false`,
 		expected: []string{
 			"D0, P[], (!!bool)::false\n",
@@ -27,6 +59,86 @@ var booleanOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::- {a: bird, b: dog}\n- {a: cat, b: fly}\n",
 		},
 	},
+	{
+		description: "`any` returns true if any boolean in a given array is true",
+		document:    `[false, true]`,
+		expression:  "any",
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		description: "`any` returns false for an empty array",
+		document:    `[]`,
+		expression:  "any",
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		description: "`any_c` returns true if any element in the array is true for the given condition.",
+		document:    "a: [rad, awesome]\nb: [meh, whatever]",
+		expression:  `.[] |= any_c(. == "awesome")`,
+		expected: []string{
+			"D0, P[], (doc)::a: true\nb: false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{pet: cat}]`,
+		expression: `any_c(.name == "harry") as $c`,
+		expected: []string{
+			"D0, P[], (doc)::[{pet: cat}]\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{pet: cat}]`,
+		expression: `all_c(.name == "harry") as $c`,
+		expected: []string{
+			"D0, P[], (doc)::[{pet: cat}]\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[false, false]`,
+		expression: "any",
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		description: "`all` returns true if all booleans in a given array are true",
+		document:    `[true, true]`,
+		expression:  "all",
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[false, true]`,
+		expression: "all",
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		description: "`all` returns true for an empty array",
+		document:    `[]`,
+		expression:  "all",
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		description: "`all_c` returns true if all elements in the array are true for the given condition.",
+		document:    "a: [rad, awesome]\nb: [meh, 12]",
+		expression:  `.[] |= all_c(tag == "!!str")`,
+		expected: []string{
+			"D0, P[], (doc)::a: true\nb: false\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		expression: `false or false`,
@@ -45,6 +157,22 @@ var booleanOperatorScenarios = []expressionScenario{
 			"D0, P[b], (!!bool)::true\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `{}`,
+		expression: `(.a.b or .c) as $x`,
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{}`,
+		expression: `(.a.b and .c) as $x`,
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
 	{
 		description: "Not true is false",
 		expression:  `true | not`,

pkg/yqlib/operator_collect_object.go

@@ -17,9 +17,12 @@ import (
 ...
 */
 
-func collectObjectOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+func collectObjectOperator(d *dataTreeNavigator, originalContext Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- collectObjectOperation")
 
+	context := originalContext.Clone()
+	context.DontAutoCreate = false
+
 	if context.MatchingNodes.Len() == 0 {
 		node := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map", Value: "{}"}
 		candidate := &CandidateNode{Node: node}

pkg/yqlib/operator_collect_object_test.go

@@ -5,6 +5,30 @@ import (
 )
 
 var collectObjectOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   "a: []",
+		expression: `.a += [{"key": "att2", "value": "val2"}]`,
+		expected: []string{
+			"D0, P[], (doc)::a: [{key: att2, value: val2}]\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "",
+		expression: `.a += {"key": "att2", "value": "val2"}`,
+		expected: []string{
+			"D0, P[], ()::a:\n    key: att2\n    value: val2\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "",
+		expression: `.a += [0]`,
+		expected: []string{
+			"D0, P[], ()::a:\n    - 0\n",
+		},
+	},
 	{
 		description: `Collect empty object`,
 		document:    ``,

pkg/yqlib/operator_collect_test.go

@@ -5,6 +5,14 @@ import (
 )
 
 var collectOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   ``,
+		expression: `.a += [0]`,
+		expected: []string{
+			"D0, P[], ()::a:\n    - 0\n",
+		},
+	},
 	{
 		description: "Collect empty",
 		document:    ``,

pkg/yqlib/operator_comments.go

@@ -27,7 +27,7 @@ func assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNod
 
 	comment := ""
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+		rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 		if err != nil {
 			return Context{}, err
 		}
@@ -41,7 +41,7 @@ func assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNod
 		candidate := el.Value.(*CandidateNode)
 
 		if expressionNode.Operation.UpdateAssign {
-			rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
+			rhs, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
 			if err != nil {
 				return Context{}, err
 			}

pkg/yqlib/operator_comments_test.go

@@ -62,6 +62,22 @@ var commentOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: cat\n\n# cat\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `a: cat`,
+		expression: `. footComment=.b.d`,
+		expected: []string{
+			"D0, P[], (doc)::a: cat\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: cat`,
+		expression: `. footComment|=.b.d`,
+		expected: []string{
+			"D0, P[], (doc)::a: cat\n",
+		},
+	},
 	{
 		description: "Remove comment",
 		document:    "a: cat # comment\nb: dog # leave this",
@@ -71,7 +87,7 @@ var commentOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description:    "Remove all comments",
+		description:    "Remove (strip) all comments",
 		subdescription: "Note the use of `...` to ensure key nodes are included.",
 		document:       "# hi\n\na: cat # comment\n\n# great\n\nb: # key comment",
 		expression:     `... comments=""`,

pkg/yqlib/operator_create_map.go

@@ -62,7 +62,7 @@ func sequenceFor(d *dataTreeNavigator, context Context, matchingNode *CandidateN
 			}
 
 			return &CandidateNode{Node: &node, Document: document, Path: path}, nil
-		})
+		}, false)
 
 	if err != nil {
 		return nil, err

pkg/yqlib/operator_delete.go

@@ -7,9 +7,7 @@ import (
 )
 
 func deleteChildOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	contextToUse := context.Clone()
-	contextToUse.DontAutoCreate = true
-	nodesToDelete, err := d.GetMatchingNodes(contextToUse, expressionNode.Rhs)
+	nodesToDelete, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 
 	if err != nil {
 		return Context{}, err
@@ -18,48 +16,23 @@ func deleteChildOperator(d *dataTreeNavigator, context Context, expressionNode *
 	for el := nodesToDelete.MatchingNodes.Back(); el != nil; el = el.Prev() {
 		candidate := el.Value.(*CandidateNode)
 
-		if len(candidate.Path) > 0 {
-			deleteImmediateChildOp := &Operation{
-				OperationType: deleteImmediateChildOpType,
-				Value:         candidate.Path[len(candidate.Path)-1],
-			}
-
-			deleteImmediateChildOpNode := &ExpressionNode{
-				Operation: deleteImmediateChildOp,
-				Rhs:       createTraversalTree(candidate.Path[0:len(candidate.Path)-1], traversePreferences{}, false),
-			}
-
-			_, err := d.GetMatchingNodes(contextToUse, deleteImmediateChildOpNode)
-			if err != nil {
-				return Context{}, err
-			}
+		//problem: context may already be '.a' and then I pass in '.a.a2'.
+		// should pass in .a2.
+		if candidate.Parent == nil {
+			log.Info("Could not find parent of %v", candidate.GetKey())
+			return context, nil
 		}
-	}
-	return context, nil
-}
 
-func deleteImmediateChildOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	parents, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+		parentNode := candidate.Parent.Node
+		childPath := candidate.Path[len(candidate.Path)-1]
 
-	if err != nil {
-		return Context{}, err
-	}
-
-	childPath := expressionNode.Operation.Value
-
-	log.Debug("childPath to remove %v", childPath)
-
-	for el := parents.MatchingNodes.Front(); el != nil; el = el.Next() {
-		parent := el.Value.(*CandidateNode)
-		parentNode := unwrapDoc(parent.Node)
 		if parentNode.Kind == yaml.MappingNode {
-			deleteFromMap(parent, childPath)
+			deleteFromMap(candidate.Parent, childPath)
 		} else if parentNode.Kind == yaml.SequenceNode {
-			deleteFromArray(parent, childPath)
+			deleteFromArray(candidate.Parent, childPath)
 		} else {
 			return Context{}, fmt.Errorf("Cannot delete nodes from parent of tag %v", parentNode.Tag)
 		}
-
 	}
 	return context, nil
 }

pkg/yqlib/operator_delete_test.go

@@ -21,6 +21,70 @@ var deleteOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: {a2: frood}}\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `{a: {a1: fred, a2: frood}}`,
+		expression: `.a | del(.a1)`,
+		expected: []string{
+			"D0, P[a], (!!map)::{a2: frood}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: [1,2,3]`,
+		expression: `.a | del(.[1])`,
+		expected: []string{
+			"D0, P[a], (!!seq)::[1, 3]\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[0, {a: cat, b: dog}]`,
+		expression: `.[1] | del(.a)`,
+		expected: []string{
+			"D0, P[1], (!!map)::{b: dog}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{a: cat, b: dog}]`,
+		expression: `.[0] | del(.a)`,
+		expected: []string{
+			"D0, P[0], (!!map)::{b: dog}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{a: {b: thing, c: frog}}]`,
+		expression: `.[0].a | del(.b)`,
+		expected: []string{
+			"D0, P[0 a], (!!map)::{c: frog}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{a: {b: thing, c: frog}}]`,
+		expression: `.[0] | del(.a.b)`,
+		expected: []string{
+			"D0, P[0], (!!map)::{a: {c: frog}}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: [0, {b: thing, c: frog}]}`,
+		expression: `.a[1] | del(.b)`,
+		expected: []string{
+			"D0, P[a 1], (!!map)::{c: frog}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: [0, {b: thing, c: frog}]}`,
+		expression: `.a | del(.[1].b)`,
+		expected: []string{
+			"D0, P[a], (!!seq)::[0, {c: frog}]\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `{a: {a1: fred, a2: frood}}`,

pkg/yqlib/operator_entries.go

@@ -0,0 +1,162 @@
+package yqlib
+
+import (
+	"container/list"
+	"fmt"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+func entrySeqFor(key *yaml.Node, value *yaml.Node) *yaml.Node {
+	var keyKey = &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "key"}
+	var valueKey = &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "value"}
+
+	return &yaml.Node{
+		Kind:    yaml.MappingNode,
+		Tag:     "!!map",
+		Content: []*yaml.Node{keyKey, key, valueKey, value},
+	}
+}
+
+func toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode {
+	var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+	var entriesNode = candidateNode.CreateChild(nil, sequence)
+
+	var contents = unwrapDoc(candidateNode.Node).Content
+	for index := 0; index < len(contents); index = index + 2 {
+		key := contents[index]
+		value := contents[index+1]
+
+		sequence.Content = append(sequence.Content, entrySeqFor(key, value))
+	}
+	return entriesNode
+}
+
+func toEntriesfromSeq(candidateNode *CandidateNode) *CandidateNode {
+	var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+	var entriesNode = candidateNode.CreateChild(nil, sequence)
+
+	var contents = unwrapDoc(candidateNode.Node).Content
+	for index := 0; index < len(contents); index = index + 1 {
+		key := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!int", Value: fmt.Sprintf("%v", index)}
+		value := contents[index]
+
+		sequence.Content = append(sequence.Content, entrySeqFor(key, value))
+	}
+	return entriesNode
+}
+
+func toEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	var results = list.New()
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		candidateNode := unwrapDoc(candidate.Node)
+
+		switch candidateNode.Kind {
+		case yaml.MappingNode:
+			results.PushBack(toEntriesFromMap(candidate))
+
+		case yaml.SequenceNode:
+			results.PushBack(toEntriesfromSeq(candidate))
+		default:
+			if candidateNode.Tag != "!!null" {
+				return Context{}, fmt.Errorf("%v has no keys", candidate.Node.Tag)
+			}
+		}
+	}
+
+	return context.ChildContext(results), nil
+}
+
+func parseEntry(d *dataTreeNavigator, entry *yaml.Node, position int) (*yaml.Node, *yaml.Node, error) {
+	prefs := traversePreferences{DontAutoCreate: true}
+	candidateNode := &CandidateNode{Node: entry}
+
+	keyResults, err := traverseMap(Context{}, candidateNode, "key", prefs, false)
+
+	if err != nil {
+		return nil, nil, err
+	} else if keyResults.Len() != 1 {
+		return nil, nil, fmt.Errorf("Expected to find one 'key' entry but found %v in position %v", keyResults.Len(), position)
+	}
+
+	valueResults, err := traverseMap(Context{}, candidateNode, "value", prefs, false)
+
+	if err != nil {
+		return nil, nil, err
+	} else if valueResults.Len() != 1 {
+		return nil, nil, fmt.Errorf("Expected to find one 'value' entry but found %v in position %v", valueResults.Len(), position)
+	}
+
+	return keyResults.Front().Value.(*CandidateNode).Node, valueResults.Front().Value.(*CandidateNode).Node, nil
+
+}
+
+func fromEntries(d *dataTreeNavigator, candidateNode *CandidateNode) (*CandidateNode, error) {
+	var node = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
+	var mapCandidateNode = candidateNode.CreateChild(nil, node)
+
+	var contents = unwrapDoc(candidateNode.Node).Content
+
+	for index := 0; index < len(contents); index = index + 1 {
+		key, value, err := parseEntry(d, contents[index], index)
+		if err != nil {
+			return nil, err
+		}
+
+		node.Content = append(node.Content, key, value)
+	}
+	return mapCandidateNode, nil
+}
+
+func fromEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	var results = list.New()
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		candidateNode := unwrapDoc(candidate.Node)
+
+		switch candidateNode.Kind {
+		case yaml.SequenceNode:
+			mapResult, err := fromEntries(d, candidate)
+			if err != nil {
+				return Context{}, err
+			}
+			results.PushBack(mapResult)
+		default:
+			return Context{}, fmt.Errorf("from entries only runs against arrays")
+		}
+	}
+
+	return context.ChildContext(results), nil
+}
+
+func withEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	//to_entries on the context
+	toEntries, err := toEntriesOperator(d, context, expressionNode)
+	if err != nil {
+		return Context{}, err
+	}
+
+	//run expression against entries
+	// splat toEntries and pipe it into Rhs
+	splatted, err := splat(d, toEntries, traversePreferences{})
+	if err != nil {
+		return Context{}, err
+	}
+
+	result, err := d.GetMatchingNodes(splatted, expressionNode.Rhs)
+	log.Debug("expressionNode.Rhs %v", expressionNode.Rhs.Operation.OperationType)
+	log.Debug("result %v", result)
+	if err != nil {
+		return Context{}, err
+	}
+
+	collected, err := collectOperator(d, result, expressionNode)
+	if err != nil {
+		return Context{}, err
+	}
+
+	//from_entries on the result
+	return fromEntriesOperator(d, collected, expressionNode)
+}

pkg/yqlib/operator_entries_test.go

@@ -0,0 +1,70 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var entriesOperatorScenarios = []expressionScenario{
+	{
+		description: "to_entries Map",
+		document:    `{a: 1, b: 2}`,
+		expression:  `to_entries`,
+		expected: []string{
+			"D0, P[], (!!seq)::- key: a\n  value: 1\n- key: b\n  value: 2\n",
+		},
+	},
+	{
+		description: "to_entries Array",
+		document:    `[a, b]`,
+		expression:  `to_entries`,
+		expected: []string{
+			"D0, P[], (!!seq)::- key: 0\n  value: a\n- key: 1\n  value: b\n",
+		},
+	},
+	{
+		description: "to_entries null",
+		document:    `null`,
+		expression:  `to_entries`,
+		expected:    []string{},
+	},
+	{
+		description: "from_entries map",
+		document:    `{a: 1, b: 2}`,
+		expression:  `to_entries | from_entries`,
+		expected: []string{
+			"D0, P[], (!!map)::a: 1\nb: 2\n",
+		},
+	},
+	{
+		description:    "from_entries with numeric key indexes",
+		subdescription: "from_entries always creates a map, even for numeric keys",
+		document:       `[a,b]`,
+		expression:     `to_entries | from_entries`,
+		expected: []string{
+			"D0, P[], (!!map)::0: a\n1: b\n",
+		},
+	},
+	{
+		description: "Use with_entries to update keys",
+		document:    `{a: 1, b: 2}`,
+		expression:  `with_entries(.key |= "KEY_" + .)`,
+		expected: []string{
+			"D0, P[], (!!map)::KEY_a: 1\nKEY_b: 2\n",
+		},
+	},
+	{
+		description: "Use with_entries to filter the map",
+		document:    `{a: { b: bird }, c: { d: dog }}`,
+		expression:  `with_entries(select(.value | has("b")))`,
+		expected: []string{
+			"D0, P[], (!!map)::a: {b: bird}\n",
+		},
+	},
+}
+
+func TestEntriesOperatorScenarios(t *testing.T) {
+	for _, tt := range entriesOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "Entries", entriesOperatorScenarios)
+}

pkg/yqlib/operator_equals.go

@@ -4,12 +4,33 @@ import "gopkg.in/yaml.v3"
 
 func equalsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- equalsOperation")
-	return crossFunction(d, context, expressionNode, isEquals(false))
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, isEquals(false), true)
 }
 
 func isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 		value := false
+		log.Debugf("-- isEquals cross function")
+		if lhs == nil && rhs == nil {
+			owner := &CandidateNode{}
+			return createBooleanCandidate(owner, !flip), nil
+		} else if lhs == nil {
+			log.Debugf("lhs nil, but rhs is not")
+			rhsNode := unwrapDoc(rhs.Node)
+			value := rhsNode.Tag == "!!null"
+			if flip {
+				value = !value
+			}
+			return createBooleanCandidate(rhs, value), nil
+		} else if rhs == nil {
+			log.Debugf("lhs not nil, but rhs is")
+			lhsNode := unwrapDoc(lhs.Node)
+			value := lhsNode.Tag == "!!null"
+			if flip {
+				value = !value
+			}
+			return createBooleanCandidate(lhs, value), nil
+		}
 
 		lhsNode := unwrapDoc(lhs.Node)
 		rhsNode := unwrapDoc(rhs.Node)
@@ -28,6 +49,6 @@ func isEquals(flip bool) func(d *dataTreeNavigator, context Context, lhs *Candid
 }
 
 func notEqualsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	log.Debugf("-- equalsOperation")
-	return crossFunction(d, context, expressionNode, isEquals(true))
+	log.Debugf("-- notEqualsOperator")
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, isEquals(true), true)
 }

pkg/yqlib/operator_equals_test.go

@@ -5,6 +5,29 @@ import (
 )
 
 var equalsOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		expression: ".a == .b",
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: cat`,
+		expression: ".a == .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: cat`,
+		expression: ".b == .a",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   "cat",
@@ -14,6 +37,60 @@ var equalsOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!bool)::false\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: "(.a == .b) as $x",
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: ".a == .b",
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: "(.a != .b) as $x",
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: ".a != .b",
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{a: {b: 10}}",
+		expression: "select(.c != null)",
+		expected:   []string{},
+	},
+	{
+		skipDoc:    true,
+		document:   "{a: {b: 10}}",
+		expression: "select(.d == .c)",
+		expected: []string{
+			"D0, P[], (doc)::{a: {b: 10}}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "{a: {b: 10}}",
+		expression: "select(null == .c)",
+		expected: []string{
+			"D0, P[], (doc)::{a: {b: 10}}\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   "{a: { b: {things: \"\"}, f: [1], g: [] }}",
@@ -87,6 +164,22 @@ var equalsOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!bool)::true\n",
 		},
 	},
+	{
+		description: "Non exisitant key doesn't equal a value",
+		document:    "a: frog",
+		expression:  `select(.b != "thing")`,
+		expected: []string{
+			"D0, P[], (doc)::a: frog\n",
+		},
+	},
+	{
+		description: "Two non existant keys are equal",
+		document:    "a: frog",
+		expression:  `select(.b == .c)`,
+		expected: []string{
+			"D0, P[], (doc)::a: frog\n",
+		},
+	},
 }
 
 func TestEqualOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_file_test.go

@@ -21,6 +21,16 @@ var fileOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!int)::0\n",
 		},
 	},
+	{
+		description: "Get file indices of multiple documents",
+		document:    `{a: cat}`,
+		document2:   `{a: cat}`,
+		expression:  `fileIndex`,
+		expected: []string{
+			"D0, P[], (!!int)::0\n",
+			"D0, P[], (!!int)::1\n",
+		},
+	},
 	{
 		description: "Get file index alias",
 		document:    `{a: cat}`,

pkg/yqlib/operator_has.go

@@ -12,14 +12,19 @@ func hasOperator(d *dataTreeNavigator, context Context, expressionNode *Expressi
 	log.Debugf("-- hasOperation")
 	var results = list.New()
 
-	rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
-	wanted := rhs.MatchingNodes.Front().Value.(*CandidateNode).Node
-	wantedKey := wanted.Value
+	rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 
 	if err != nil {
 		return Context{}, err
 	}
 
+	wantedKey := "null"
+	wanted := &yaml.Node{Tag: "!!null"}
+	if rhs.MatchingNodes.Len() != 0 {
+		wanted = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node
+		wantedKey = wanted.Value
+	}
+
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 

pkg/yqlib/operator_has_test.go

@@ -13,6 +13,22 @@ var hasOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!bool)::true\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `a: hello`,
+		expression: `has(.b) as $c`,
+		expected: []string{
+			"D0, P[], (doc)::a: hello\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: hello`,
+		expression: `has(.b)`,
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
 	{
 		description: "Has map key",
 		document: `- a: "yes"
@@ -28,6 +44,15 @@ var hasOperatorScenarios = []expressionScenario{
 			"D0, P[3], (!!bool)::false\n",
 		},
 	},
+	{
+		description:    "Select, checking for existence of deep paths",
+		subdescription: "Simply pipe in parent expressions into `has`",
+		document:       "- {a: {b: {c: cat}}}\n- {a: {b: {d: dog}}}",
+		expression:     `.[] | select(.a.b | has("c"))`,
+		expected: []string{
+			"D0, P[0], (!!map)::{a: {b: {c: cat}}}\n",
+		},
+	},
 	{
 		dontFormatInputForDoc: true,
 		description:           "Has array index",

pkg/yqlib/operator_multiply.go

@@ -10,17 +10,39 @@ import (
 )
 
 type multiplyPreferences struct {
-	AppendArrays  bool
-	TraversePrefs traversePreferences
+	AppendArrays    bool
+	DeepMergeArrays bool
+	TraversePrefs   traversePreferences
 }
 
 func multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- MultiplyOperator")
-	return crossFunction(d, context, expressionNode, multiply(expressionNode.Operation.Preferences.(multiplyPreferences)))
+	return crossFunction(d, context, expressionNode, multiply(expressionNode.Operation.Preferences.(multiplyPreferences)), false)
+}
+
+func getNewBlankNode(lhs *yaml.Node, rhs *yaml.Node) *yaml.Node {
+
+	blankNode := &yaml.Node{}
+
+	if lhs.HeadComment != "" {
+		blankNode.HeadComment = lhs.HeadComment
+	} else if rhs.HeadComment != "" {
+		blankNode.HeadComment = rhs.HeadComment
+	}
+
+	if lhs.FootComment != "" {
+		blankNode.FootComment = lhs.FootComment
+	} else if rhs.FootComment != "" {
+		blankNode.FootComment = rhs.FootComment
+	}
+
+	return blankNode
 }
 
 func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+		// need to do this before unWrapping the potential document node
+		newBlankNode := getNewBlankNode(lhs.Node, rhs.Node)
 		lhs.Node = unwrapDoc(lhs.Node)
 		rhs.Node = unwrapDoc(rhs.Node)
 		log.Debugf("Multipling LHS: %v", lhs.Node.Tag)
@@ -29,44 +51,66 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
 		if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode ||
 			(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
 
-			var newBlank = lhs.CreateChild(nil, &yaml.Node{})
-			var newThing, err = mergeObjects(d, context, newBlank, lhs, multiplyPreferences{})
+			var newBlank = lhs.CreateChild(nil, newBlankNode)
+			log.Debugf("merge - merge lhs into blank")
+			var newThing, err = mergeObjects(d, context.WritableClone(), newBlank, lhs, multiplyPreferences{})
 			if err != nil {
 				return nil, err
 			}
-			return mergeObjects(d, context, newThing, rhs, preferences)
+			return mergeObjects(d, context.WritableClone(), newThing, rhs, preferences)
 		} else if lhs.Node.Tag == "!!int" && rhs.Node.Tag == "!!int" {
 			return multiplyIntegers(lhs, rhs)
+		} else if (lhs.Node.Tag == "!!int" || lhs.Node.Tag == "!!float") && (rhs.Node.Tag == "!!int" || rhs.Node.Tag == "!!float") {
+			return multiplyFloats(lhs, rhs)
 		}
 		return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag)
 	}
 }
 
+func multiplyFloats(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	target := lhs.CreateChild(nil, &yaml.Node{})
+	target.Node.Kind = yaml.ScalarNode
+	target.Node.Style = lhs.Node.Style
+	target.Node.Tag = "!!float"
+
+	lhsNum, err := strconv.ParseFloat(lhs.Node.Value, 64)
+	if err != nil {
+		return nil, err
+	}
+	rhsNum, err := strconv.ParseFloat(rhs.Node.Value, 64)
+	if err != nil {
+		return nil, err
+	}
+	target.Node.Value = fmt.Sprintf("%v", lhsNum*rhsNum)
+	return target, nil
+}
+
 func multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	target := lhs.CreateChild(nil, &yaml.Node{})
 	target.Node.Kind = yaml.ScalarNode
 	target.Node.Style = lhs.Node.Style
 	target.Node.Tag = "!!int"
 
-	lhsNum, err := strconv.Atoi(lhs.Node.Value)
+	format, lhsNum, err := parseInt(lhs.Node.Value)
 	if err != nil {
 		return nil, err
 	}
-	rhsNum, err := strconv.Atoi(rhs.Node.Value)
+	_, rhsNum, err := parseInt(rhs.Node.Value)
 	if err != nil {
 		return nil, err
 	}
-	target.Node.Value = fmt.Sprintf("%v", lhsNum*rhsNum)
+	target.Node.Value = fmt.Sprintf(format, lhsNum*rhsNum)
 	return target, nil
 }
 
 func mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode, preferences multiplyPreferences) (*CandidateNode, error) {
-	shouldAppendArrays := preferences.AppendArrays
 	var results = list.New()
 
-	// shouldn't recurse arrays if appending
-	prefs := recursiveDescentPreferences{RecurseArray: !shouldAppendArrays,
+	// only need to recurse the array if we are doing a deep merge
+	prefs := recursiveDescentPreferences{RecurseArray: preferences.DeepMergeArrays,
 		TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: true}}
+	log.Debugf("merge - preferences.DeepMergeArrays %v", preferences.DeepMergeArrays)
+	log.Debugf("merge - preferences.AppendArrays %v", preferences.AppendArrays)
 	err := recursiveDecent(d, results, context.SingleChildContext(rhs), prefs)
 	if err != nil {
 		return nil, err
@@ -96,13 +140,22 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom
 	log.Debugf("merge - applyAssignment lhs %v, rhs: %v", lhs.GetKey(), rhs.GetKey())
 
 	lhsPath := rhs.Path[pathIndexToStartFrom:]
+	log.Debugf("merge - lhsPath %v", lhsPath)
 
 	assignmentOp := &Operation{OperationType: assignAttributesOpType}
-	if rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode {
+	if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode {
+		assignmentOp.OperationType = addAssignOpType
+		log.Debugf("merge - assignmentOp.OperationType = addAssignOpType")
+	} else if !preferences.DeepMergeArrays && rhs.Node.Kind == yaml.SequenceNode ||
+		(rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode) {
 		assignmentOp.OperationType = assignOpType
 		assignmentOp.UpdateAssign = false
-	} else if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode {
-		assignmentOp.OperationType = addAssignOpType
+		log.Debugf("merge - rhs.Node.Kind == yaml.SequenceNode: %v", rhs.Node.Kind == yaml.SequenceNode)
+		log.Debugf("merge - rhs.Node.Kind == yaml.ScalarNode: %v", rhs.Node.Kind == yaml.ScalarNode)
+		log.Debugf("merge - rhs.Node.Kind == yaml.AliasNode: %v", rhs.Node.Kind == yaml.AliasNode)
+		log.Debugf("merge - assignmentOp.OperationType = assignOpType, no updateassign")
+	} else {
+		log.Debugf("merge - assignmentOp := &Operation{OperationType: assignAttributesOpType}")
 	}
 	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs}
 

pkg/yqlib/operator_multiply_test.go

@@ -4,7 +4,178 @@ import (
 	"testing"
 )
 
+var doc1 = `list:
+# Hi this is a comment.
+# Hello this is another comment.
+- "abc"`
+
+var doc2 = `list2:
+# This is yet another comment.
+# Indeed this is yet another comment.
+- "123"`
+
+var docExpected = `D0, P[], (!!map)::list:
+    # Hi this is a comment.
+    # Hello this is another comment.
+    - "abc"
+list2:
+    # This is yet another comment.
+    # Indeed this is yet another comment.
+    - "123"
+`
+
+var mergeArrayWithAnchors = `sample:
+- &a
+- <<: *a
+`
+
+var mergeArraysObjectKeysText = `It's a complex command, the trickyness comes from needing to have the right context in the expressions.
+First we save the second array into a variable '$two' which lets us reference it later.
+We then need to update the first array. We will use the relative update (|=) because we need to update relative to the current element of the array in the LHS in the RHS expression. 
+We set the current element of the first array as $cur. Now we multiply (merge) $cur with the matching entry in $two, by passing $two through a select filter.
+`
+
+var docWithHeader = `
+# here
+
+a: apple
+`
+
+var nodeWithHeader = `
+# here
+a: apple
+`
+
+var docNoComments = `
+b: banana
+`
+
+var docWithFooter = `
+a: apple
+
+# footer
+`
+
+var nodeWithFooter = `
+a: apple
+# footer`
+
 var multiplyOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   mergeArrayWithAnchors,
+		expression: `. * .`,
+		expected: []string{
+			"D0, P[], (!!map)::sample:\n    - &a\n    - !!merge <<: *a\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   docWithHeader,
+		document2:  docNoComments,
+		expression: `select(fi == 0) * select(fi == 1)`,
+		expected: []string{
+			"D0, P[], (!!map)::# here\na: apple\nb: banana\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   nodeWithHeader,
+		document2:  docNoComments,
+		expression: `select(fi == 0) * select(fi == 1)`,
+		expected: []string{
+			"D0, P[], (!!map)::# here\na: apple\nb: banana\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   docNoComments,
+		document2:  docWithHeader,
+		expression: `select(fi == 0) * select(fi == 1)`,
+		expected: []string{
+			"D0, P[], (!!map)::# here\nb: banana\na: apple\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   docNoComments,
+		document2:  nodeWithHeader,
+		expression: `select(fi == 0) * select(fi == 1)`,
+		expected: []string{
+			"D0, P[], (!!map)::b: banana\n# here\na: apple\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   docWithFooter,
+		document2:  docNoComments,
+		expression: `select(fi == 0) * select(fi == 1)`,
+		expected: []string{
+			"D0, P[], (!!map)::a: apple\nb: banana\n\n# footer\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   nodeWithFooter,
+		document2:  docNoComments,
+		expression: `select(fi == 0) * select(fi == 1)`,
+		expected: []string{ // not sure why there's an extra newline *shrug*
+			"D0, P[], (!!map)::a: apple\n# footer\n\nb: banana\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   docNoComments,
+		document2:  docWithFooter,
+		expression: `select(fi == 0) * select(fi == 1)`,
+		expected: []string{
+			"D0, P[], (!!map)::b: banana\na: apple\n\n# footer\n",
+		},
+	},
+	{
+		description: "Multiply integers",
+		expression:  `3 * 4`,
+		expected: []string{
+			"D0, P[], (!!int)::12\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   doc1,
+		document2:  doc2,
+		expression: `select(fi == 0) * select(fi == 1)`,
+		expected: []string{
+			docExpected,
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `.x =  {"things": "whatever"} * {}`,
+		expected: []string{
+			"D0, P[], ()::x:\n    things: whatever\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `.x = {} * {"things": "whatever"}`,
+		expected: []string{
+			"D0, P[], ()::x:\n    things: whatever\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `3 * 4.5`,
+		expected: []string{
+			"D0, P[], (!!float)::13.5\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `4.5 * 3`,
+		expected: []string{
+			"D0, P[], (!!float)::13.5\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `{a: {also: [1]}, b: {also: me}}`,
@@ -157,7 +328,7 @@ var multiplyOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{a: [{thing: one}], b: [{missing: two, thing: two}]}`,
-		expression: `.a *? .b`,
+		expression: `.a *?d .b`,
 		expected: []string{
 			"D0, P[a], (!!seq)::[{thing: two}]\n",
 		},
@@ -186,6 +357,25 @@ var multiplyOperatorScenarios = []expressionScenario{
 			"D0, P[a], (!!map)::{thing: [1, 2, 3, 4]}\n",
 		},
 	},
+	{
+		description:    "Merge, deeply merging arrays",
+		subdescription: "Merging arrays deeply means arrays are merge like objects, with indexes as their key. In this case, we merge the first item in the array, and do nothing with the second.",
+		document:       `{a: [{name: fred, age: 12}, {name: bob, age: 32}], b: [{name: fred, age: 34}]}`,
+		expression:     `.a *d .b`,
+		expected: []string{
+			"D0, P[a], (!!seq)::[{name: fred, age: 34}, {name: bob, age: 32}]\n",
+		},
+	},
+	{
+		description:    "Merge arrays of objects together, matching on a key",
+		subdescription: mergeArraysObjectKeysText,
+		document:       `[{a: apple, b: appleB}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB}]`,
+		document2:      `[{a: banana, c: bananaC}, {a: apple, b: appleB2}, {a: dingo, c: dingoC}]`,
+		expression:     `(select(fi==1) | .[]) as $two | select(fi==0) | .[] |= (. as $cur |  $cur * ($two | select(.a == $cur.a)))`,
+		expected: []string{
+			"D0, P[], (doc)::[{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC}]\n",
+		},
+	},
 	{
 		description: "Merge to prefix an element",
 		document:    `{a: cat, b: dog}`,
@@ -224,5 +414,5 @@ func TestMultiplyOperatorScenarios(t *testing.T) {
 	for _, tt := range multiplyOperatorScenarios {
 		testScenario(t, &tt)
 	}
-	documentScenarios(t, "Multiply", multiplyOperatorScenarios)
+	documentScenarios(t, "Multiply (Merge)", multiplyOperatorScenarios)
 }