Bumping version

This reverts commit ffb6cbf769.

.github/workflows/test-yq.yml

@@ -0,0 +1,19 @@
+name: Test Yq Action
+on: [push]
+jobs:
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Get test
+        uses: mikefarah/yq@master
+        with:
+          cmd: yq eval '.a' examples/data1.yaml
+      - name: Write inplace test
+        id: lookupSdkVersion
+        uses: mikefarah/yq@master
+        with:
+          cmd: yq eval -i '.a.b = 5' examples/data1.yaml

.gitignore

@@ -38,3 +38,5 @@ parts/
 prime/
 .snapcraft/
 yq*.snap
+test.yml
+test2.yml

Dockerfile

@@ -12,7 +12,12 @@ RUN CGO_ENABLED=0 make local build
 
 # Choose alpine as a base image to make this useful for CI, as many
 # CI tools expect an interactive shell inside the container
-FROM alpine:3.12.3 as production
+FROM alpine:3.13.5 as production
+
+RUN mkdir /home/yq/
+RUN addgroup -g 1000 yq && \
+    adduser -u 1000 -G yq -s /bin/bash -h /home/yq -D yq
+RUN chown -R yq:yq /home/yq/
 
 COPY --from=builder /go/src/mikefarah/yq/yq /usr/bin/yq
 RUN chmod +x /usr/bin/yq
@@ -20,6 +25,8 @@ RUN chmod +x /usr/bin/yq
 ARG VERSION=none
 LABEL version=${VERSION}
 
+USER yq
+
 WORKDIR /workdir
 
 ENTRYPOINT ["/usr/bin/yq"]

README.md

@@ -7,12 +7,43 @@ a lightweight and portable command-line YAML processor. `yq` uses [jq](https://g
 
 yq is written in go - so you can download a dependency free binary for your platform and you are good to go! If you prefer there are a variety of package managers that can be used as well as docker, all listed below.
 
-## V4 released!
-V4 is now officially released, it's quite different from V3 (sorry for the migration), however it is much more similar to ```jq```, using a similar expression syntax and therefore support much more complex functionality! 
+## Quick Usage Guide
 
-If you've been using v3 and want/need to upgrade, checkout the [upgrade guide](https://mikefarah.gitbook.io/yq/v/v4.x/upgrading-from-v3).
+Read a value:
+```bash
+yq e '.a.b[0].c' file.yaml
+```
 
-Support for v3 will cease August 2021, until then, critical bug and security fixes will still get applied if required.
+Pipe from STDIN:
+```bash
+cat file.yaml | yq e '.a.b[0].c' -
+```
+
+Update a yaml file, inplace
+```bash
+yq e -i '.a.b[0].c = "cool"' file.yaml
+```
+
+Update using environment variables
+```bash
+NAME=mike yq e -i '.a.b[0].c = strenv(NAME)' file.yaml
+```
+
+Merge multiple files
+```
+yq ea '. as $item ireduce ({}; . * $item )' path/to/*.yml
+```
+
+Multiple updates to a yaml file
+```bash
+yq e -i '
+  .a.b[0].c = "cool" |
+  .x.y.z = "foobar" |
+  .person.name = strenv(NAME)
+' file.yaml
+```
+
+See the [documentation](https://mikefarah.gitbook.io/yq/) for more.
 
 ## Install
 
@@ -87,6 +118,14 @@ rm /etc/myfile.tmp
 docker run --rm -v "${PWD}":/workdir mikefarah/yq <command> [flags] [expression ]FILE...
 ```
 
+#### Pipe in via STDIN:
+
+You'll need to pass the `-i\--interactive` flag to docker:
+
+```bash
+cat myfile.yml | docker run -i --rm mikefarah/yq e . -
+```
+
 #### Run commands interactively:
 
 ```bash
@@ -101,6 +140,36 @@ yq() {
 }
 ```
 
+
+#### Running as root:
+
+`yq`'s docker image no longer runs under root (https://github.com/mikefarah/yq/pull/860). If you'd like to install more things in the docker image, or you're having permissions issues when attempting to read/write files you'll need to either:
+
+
+```
+docker run --user="root" -it --entrypoint sh mikefarah/yq
+```
+
+Or, in your docker file:
+
+```
+FROM mikefarah/yq
+
+USER root
+RUN apk add bash
+USER yq
+```
+
+### GitHub Action
+```
+  - name: Set foobar to cool
+    uses: mikefarah/yq@master
+    with:
+      cmd: yq eval -i '.foo.bar = "cool"' 'config.yml'
+```
+
+See https://mikefarah.gitbook.io/yq/usage/github-action for more.
+
 ### Go Get:
 ```
 GO111MODULE=on go get github.com/mikefarah/yq/v4
@@ -157,6 +226,7 @@ Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
 - Written in portable go, so you can download a lovely dependency free binary
 - Uses similar syntax as `jq` but works with YAML and JSON files
 - Fully supports multi document yaml files
+- Supports yaml [front matter](https://mikefarah.gitbook.io/yq/usage/front-matter) blocks (e.g. jekyll/assemble)
 - Colorized yaml output
 - [Deeply traverse yaml](https://mikefarah.gitbook.io/yq/operators/traverse-read)
 - [Sort yaml by keys](https://mikefarah.gitbook.io/yq/operators/sort-keys)
@@ -164,10 +234,12 @@ Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
 - [Update yaml inplace](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags)
 - [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/operators/select#select-and-update-matching-values-in-map)
 - Keeps yaml formatting and comments when updating (though there are issues with whitespace)
-- [Convert to/from json to yaml](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
+- [Convert to/from json](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
+- [Convert to properties](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties)
 - [Pipe data in by using '-'](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate)
 - [General shell completion scripts (bash/zsh/fish/powershell)](https://mikefarah.gitbook.io/yq/v/v4.x/commands/shell-completion)
 - [Reduce](https://mikefarah.gitbook.io/yq/operators/reduce) to merge multiple files or sum an array or other fancy things.
+- [Github Action](https://mikefarah.gitbook.io/yq/usage/github-action) to use in your automated pipeline (thanks @devorbitus)
 
 ## [Usage](https://mikefarah.gitbook.io/yq/)
 
@@ -179,33 +251,31 @@ Usage:
   yq [command]
 
 Available Commands:
-  eval             Apply expression to each document in each yaml file given in sequence
+  eval             Apply the expression to each document in each yaml file in sequence
   eval-all         Loads _all_ yaml documents of _all_ yaml files and runs expression once
   help             Help about any command
   shell-completion Generate completion script
 
 Flags:
-  -C, --colors        force print with colors
-  -e, --exit-status   set exit status if there are no matches or null or false is returned
-  -h, --help          help for yq
-  -I, --indent int    sets indent level for output (default 2)
-  -i, --inplace       update the yaml file inplace of first yaml file given.
-  -M, --no-colors     force print with no colors
-  -N, --no-doc        Don't print document separators (---)
-  -n, --null-input    Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.
-  -P, --prettyPrint    pretty print, shorthand for '... style = ""' 
-  -j, --tojson        output as json. Set indent to 0 to print json in one line.
-  -v, --verbose       verbose mode
-  -V, --version       Print version information and quit
+  -C, --colors                force print with colors
+  -e, --exit-status           set exit status if there are no matches or null or false is returned
+  -f, --front-matter string   (extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact
+  -h, --help                  help for yq
+  -I, --indent int            sets indent level for output (default 2)
+  -i, --inplace               update the yaml file inplace of first yaml file given.
+  -M, --no-colors             force print with no colors
+  -N, --no-doc                Don't print document separators (---)
+  -n, --null-input            Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.
+  -P, --prettyPrint           pretty print, shorthand for '... style = ""'
+  -j, --tojson                output as json. Set indent to 0 to print json in one line.
+      --unwrapScalar          unwrap scalar, print the value with no quotes, colors or comments (default true)
+  -v, --verbose               verbose mode
+  -V, --version               Print version information and quit
 
 Use "yq [command] --help" for more information about a command.
 ```
-
-Simple Example:
-
-```bash
-yq e '.a.b | length' f1.yml f2.yml 
-```
-
 ## Known Issues / Missing Features
 - `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details)
+- Powershell has its own...opinions: https://mikefarah.gitbook.io/yq/usage/tips-and-tricks#quotes-in-windows-powershell
+
+See [tips and tricks](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks) for more common problems and solutions.

acceptance_tests/basic.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+setUp() {
+  rm -f test.yml
+}
+
+testBasicEvalRoundTrip() {
+  ./yq e -n ".a = 123" > test.yml
+  X=$(./yq e '.a' test.yml)
+  assertEquals 123 "$X"
+}
+
+testBasicUpdateInPlaceSequence() {
+  cat >test.yml <<EOL
+a: 0
+EOL
+  ./yq e -i ".a = 10" test.yml
+  X=$(./yq e '.a' test.yml)
+  assertEquals "10" "$X"
+}
+
+testBasicUpdateInPlaceSequenceEvalAll() {
+  cat >test.yml <<EOL
+a: 0
+EOL
+  ./yq ea -i ".a = 10" test.yml
+  X=$(./yq e '.a' test.yml)
+  assertEquals "10" "$X"
+}
+
+testBasicNoExitStatus() {
+  echo "a: cat" > test.yml
+  X=$(./yq e '.z' test.yml)
+  assertEquals "null" "$X"
+}
+
+testBasicExitStatus() {
+  echo "a: cat" > test.yml
+  X=$(./yq e -e '.z' test.yml 2&>/dev/null)
+  assertEquals 1 "$?"
+}
+
+testBasicExtractFieldWithSeperator() {
+    cat >test.yml <<EOL
+---
+name: chart-name
+version: 1.2.3
+EOL
+  X=$(./yq e '.name' test.yml)
+  assertEquals "chart-name" "$X"
+}
+
+testBasicExtractMultipleFieldWithSeperator() {
+    cat >test.yml <<EOL
+---
+name: chart-name
+version: 1.2.3
+---
+name: thing
+version: 1.2.3
+EOL
+
+read -r -d '' expected << EOM
+chart-name
+---
+thing
+EOM
+  X=$(./yq e '.name' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+source ./scripts/shunit2

acceptance_tests/empty.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+setUp() {
+  cat >test.yml <<EOL
+# comment
+EOL
+}
+
+testEmptyEval() {
+  X=$(./yq e test.yml)
+  expected=$(cat test.yml)
+  assertEquals 0 $?
+  assertEquals "$expected" "$X"
+}
+
+testEmptyEvalNoNewLine() {
+  echo -n "#comment" >test.yml
+  X=$(./yq e test.yml)
+  expected=$(cat test.yml)
+  assertEquals 0 $?
+  assertEquals "$expected" "$X"
+}
+
+testEmptyEvalNoNewLineWithExpression() {
+  echo -n "# comment" >test.yml
+  X=$(./yq e '.apple = "tree"' test.yml)
+  read -r -d '' expected << EOM
+# comment
+apple: tree
+EOM
+  assertEquals "$expected" "$X"
+}
+
+testEmptyEvalPipe() {
+  X=$(./yq e - < test.yml)
+  assertEquals 0 $?
+}
+
+testEmptyCommentsWithExpressionEval() {
+  read -r -d '' expected << EOM
+# comment
+apple: tree
+EOM
+
+  X=$(./yq e '.apple="tree"' test.yml)
+
+  assertEquals "$expected" "$X"
+}
+
+testEmptyCommentsWithExpressionEvalAll() {
+  read -r -d '' expected << EOM
+# comment
+apple: tree
+EOM
+
+  X=$(./yq ea '.apple="tree"' test.yml)
+
+  assertEquals "$expected" "$X"
+}
+
+testEmptyWithExpressionEval() {
+  rm test.yml
+  touch test.yml
+  expected="apple: tree"
+
+  X=$(./yq e '.apple="tree"' test.yml)
+
+  assertEquals "$expected" "$X"
+}
+
+testEmptyWithExpressionEvalAll() {
+  rm test.yml
+  touch test.yml
+  expected="apple: tree"
+
+  X=$(./yq ea '.apple="tree"' test.yml)
+
+  assertEquals "$expected" "$X"
+}
+
+
+source ./scripts/shunit2

acceptance_tests/front-matter.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+setUp() {
+  cat >test.yml <<EOL
+---
+a: apple
+b: cat
+---
+not yaml
+c: at
+EOL
+}
+
+testFrontMatterProcessEval() {
+  read -r -d '' expected << EOM
+---
+a: apple
+b: dog
+---
+not yaml
+c: at
+EOM
+  ./yq e --front-matter="process" '.b = "dog"' test.yml -i
+  assertEquals "$expected" "$(cat test.yml)"
+}
+
+testFrontMatterProcessEvalAll() {
+  read -r -d '' expected << EOM
+---
+a: apple
+b: dog
+---
+not yaml
+c: at
+EOM
+  ./yq ea --front-matter="process" '.b = "dog"' test.yml -i
+  assertEquals "$expected" "$(cat test.yml)"
+}
+
+testFrontMatterExtractEval() {
+    cat >test.yml <<EOL
+a: apple
+b: cat
+---
+not yaml
+c: at
+EOL
+
+  read -r -d '' expected << EOM
+a: apple
+b: dog
+EOM
+  ./yq e --front-matter="extract" '.b = "dog"' test.yml -i
+  assertEquals "$expected" "$(cat test.yml)"
+}
+
+testFrontMatterExtractEvalAll() {
+  cat >test.yml <<EOL
+a: apple
+b: cat
+---
+not yaml
+c: at
+EOL
+
+  read -r -d '' expected << EOM
+a: apple
+b: dog
+EOM
+  ./yq ea --front-matter="extract" '.b = "dog"' test.yml -i
+  assertEquals "$expected" "$(cat test.yml)"
+}
+
+
+source ./scripts/shunit2

acceptance_tests/leading-seperator.sh

@@ -0,0 +1,449 @@
+#!/bin/bash
+
+setUp() {
+  cat >test.yml <<EOL
+---
+a: test
+EOL
+}
+
+testLeadingSeperatorWithDoc() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+---
+a: test
+---
+b: cool
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+---
+a: thing
+---
+b: cool
+EOM
+
+  X=$(./yq e '(select(di == 0) | .a) = "thing"' - < test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorPipeIntoEvalSeq() {
+  X=$(./yq e - < test.yml)
+  expected=$(cat test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorExtractField() {
+  X=$(./yq e '.a' - < test.yml)
+  assertEquals "test" "$X"
+}
+
+testLeadingSeperatorExtractFieldWithCommentsAfterSep() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+EOL
+  X=$(./yq e '.a' test.yml)
+  assertEquals "test" "$X"
+}
+
+testLeadingSeperatorExtractFieldWithCommentsBeforeSep() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+---
+a: test
+EOL
+  X=$(./yq e '.a' test.yml)
+  assertEquals "test" "$X"
+}
+
+
+testLeadingSeperatorExtractFieldMultiDoc() {
+  cat >test.yml <<EOL
+---
+a: test
+---
+a: test2
+EOL
+
+  read -r -d '' expected << EOM
+test
+---
+test2
+EOM
+  X=$(./yq e '.a' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorExtractFieldMultiDocWithComments() {
+  cat >test.yml <<EOL
+# here
+---
+# there
+a: test
+# whereever
+---
+# you are
+a: test2
+# woop
+EOL
+
+  read -r -d '' expected << EOM
+test
+---
+test2
+EOM
+  X=$(./yq e '.a' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+
+testLeadingSeperatorEvalSeq() {
+  X=$(./yq e test.yml)
+  expected=$(cat test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorPipeIntoEvalAll() {
+  X=$(./yq ea - < test.yml)
+  expected=$(cat test.yml)
+  assertEquals "$expected" "$X"
+}
+
+
+testLeadingSeperatorEvalAll() {
+  X=$(./yq ea test.yml)
+  expected=$(cat test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalSimple() {
+  read -r -d '' expected << EOM
+---
+a: test
+---
+version: 3
+application: MyApp
+EOM
+
+
+  X=$(./yq e '.' test.yml examples/order.yaml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocInOneFile() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+---
+b: things
+EOL
+  expected=$(cat test.yml)
+  X=$(./yq e '.' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocInOneFileEvalAll() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+---
+b: things
+EOL
+  expected=$(cat test.yml)
+  X=$(./yq ea '.' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalComments() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+a: test
+EOL
+
+cat >test2.yml <<EOL
+# this is another doc
+# great
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+a: test
+---
+# this is another doc
+# great
+b: sane
+EOM
+
+
+  X=$(./yq e '.' test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalCommentsTrailingSep() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+---
+a: test
+EOL
+
+cat >test2.yml <<EOL
+# this is another doc
+# great
+---
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+---
+a: test
+---
+# this is another doc
+# great
+---
+b: sane
+EOM
+
+
+  X=$(./yq e '.' test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiMultiDocEvalCommentsTrailingSep() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+---
+a: test
+---
+a1: test2
+EOL
+
+cat >test2.yml <<EOL
+# this is another doc
+# great
+---
+b: sane
+---
+b2: cool
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+---
+a: test
+---
+a1: test2
+---
+# this is another doc
+# great
+---
+b: sane
+---
+b2: cool
+EOM
+
+
+  X=$(./yq e '.' test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalCommentsLeadingSep() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+EOL
+
+cat >test2.yml <<EOL
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+---
+# hi peeps
+# cool
+a: test
+---
+# this is another doc
+# great
+b: sane
+EOM
+
+
+  X=$(./yq e '.' test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+# https://github.com/mikefarah/yq/issues/919
+testLeadingSeparatorDoesNotBreakCommentsOnOtherFiles() {
+  cat >test.yml <<EOL
+# a1
+a: 1
+# a2
+EOL
+
+cat >test2.yml <<EOL
+# b1
+b: 2
+# b2
+EOL
+
+  read -r -d '' expected << EOM
+# a1
+a: 1
+# a2
+
+# b1
+b: 2
+# b2
+EOM
+
+
+  X=$(./yq ea 'select(fi == 0) * select(fi == 1)' test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalCommentsStripComments() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  # it will be hard to remove that top level separator
+  read -r -d '' expected << EOM
+a: test
+---
+b: sane
+EOM
+
+  X=$(./yq e '... comments=""'  test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalCommentsLeadingSepNoDocFlag() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+a: test
+# this is another doc
+# great
+b: sane
+EOM
+
+
+  X=$(./yq e '.' --no-doc test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalJsonFlag() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+EOL
+
+cat >test2.yml <<EOL
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+{
+  "a": "test"
+}
+{
+  "b": "sane"
+}
+EOM
+
+
+  X=$(./yq e '.' -j test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalAllJsonFlag() {
+  cat >test.yml <<EOL
+---
+# hi peeps
+# cool
+a: test
+EOL
+
+cat >test2.yml <<EOL
+---
+# this is another doc
+# great
+b: sane
+EOL
+
+  read -r -d '' expected << EOM
+{
+  "a": "test"
+}
+{
+  "b": "sane"
+}
+EOM
+
+
+  X=$(./yq ea '.' -j test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeperatorMultiDocEvalAll() {
+  read -r -d '' expected << EOM
+---
+a: test
+---
+version: 3
+application: MyApp
+EOM
+
+
+  X=$(./yq ea '.' test.yml examples/order.yaml)
+  assertEquals "$expected" "$X"
+}
+
+source ./scripts/shunit2

acceptance_tests/output-format.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+
+testOutputJsonDeprecated() {
+  cat >test.yml <<EOL
+a: {b: ["cat"]}
+EOL
+
+  read -r -d '' expected << EOM
+{
+  "a": {
+    "b": [
+      "cat"
+    ]
+  }
+}
+EOM
+
+  X=$(./yq e -j test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -j test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testOutputJson() {
+  cat >test.yml <<EOL
+a: {b: ["cat"]}
+EOL
+
+  read -r -d '' expected << EOM
+{
+  "a": {
+    "b": [
+      "cat"
+    ]
+  }
+}
+EOM
+
+  X=$(./yq e --output-format=json test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --output-format=json test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testOutputJsonShort() {
+  cat >test.yml <<EOL
+a: {b: ["cat"]}
+EOL
+
+  read -r -d '' expected << EOM
+{
+  "a": {
+    "b": [
+      "cat"
+    ]
+  }
+}
+EOM
+
+  X=$(./yq e -o=j test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -o=j test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testOutputProperties() {
+  cat >test.yml <<EOL
+a: {b: {c: ["cat"]}}
+EOL
+
+  read -r -d '' expected << EOM
+a.b.c.0 = cat
+EOM
+
+  X=$(./yq e --output-format=props test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --output-format=props test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testOutputPropertiesShort() {
+  cat >test.yml <<EOL
+a: {b: {c: ["cat"]}}
+EOL
+
+  read -r -d '' expected << EOM
+a.b.c.0 = cat
+EOM
+
+  X=$(./yq e -o=p test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -o=p test.yml)
+  assertEquals "$expected" "$X"
+}
+
+source ./scripts/shunit2

acceptance_tests/pretty-print.sh

@@ -0,0 +1,173 @@
+#!/bin/bash
+
+
+testPrettyPrintWithBooleans() {
+  cat >test.yml <<EOL
+leaveUnquoted: [yes, no, on, off, y, n, true, false]
+leaveQuoted: ["yes", "no", "on", "off", "y", "n", "true", "false"]
+
+EOL
+
+  read -r -d '' expected << EOM
+leaveUnquoted:
+  - yes
+  - no
+  - on
+  - off
+  - y
+  - n
+  - true
+  - false
+leaveQuoted:
+  - "yes"
+  - "no"
+  - "on"
+  - "off"
+  - "y"
+  - "n"
+  - "true"
+  - "false"
+EOM
+
+  X=$(./yq e --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testPrettyPrintWithBooleansCapitals() {
+  cat >test.yml <<EOL
+leaveUnquoted: [YES, NO, ON, OFF, Y, N, TRUE, FALSE]
+leaveQuoted: ["YES", "NO", "ON", "OFF", "Y", "N", "TRUE", "FALSE"]
+EOL
+
+  read -r -d '' expected << EOM
+leaveUnquoted:
+  - YES
+  - NO
+  - ON
+  - OFF
+  - Y
+  - N
+  - TRUE
+  - FALSE
+leaveQuoted:
+  - "YES"
+  - "NO"
+  - "ON"
+  - "OFF"
+  - "Y"
+  - "N"
+  - "TRUE"
+  - "FALSE"
+EOM
+
+  X=$(./yq e --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testPrettyPrintOtherStringValues() {
+  cat >test.yml <<EOL
+leaveUnquoted: [yesSir, hellno, bonapite]
+makeUnquoted: ["yesSir", "hellno", "bonapite"]
+EOL
+
+  read -r -d '' expected << EOM
+leaveUnquoted:
+  - yesSir
+  - hellno
+  - bonapite
+makeUnquoted:
+  - yesSir
+  - hellno
+  - bonapite
+EOM
+
+  X=$(./yq e --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testPrettyPrintKeys() {
+  cat >test.yml <<EOL
+"removeQuotes": "please"
+EOL
+
+  read -r -d '' expected << EOM
+removeQuotes: please
+EOM
+
+  X=$(./yq e --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testPrettyPrintOtherStringValues() {
+  cat >test.yml <<EOL
+leaveUnquoted: [yesSir, hellno, bonapite]
+makeUnquoted: ["yesSir", "hellno", "bonapite"]
+EOL
+
+  read -r -d '' expected << EOM
+leaveUnquoted:
+  - yesSir
+  - hellno
+  - bonapite
+makeUnquoted:
+  - yesSir
+  - hellno
+  - bonapite
+EOM
+
+  X=$(./yq e --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testPrettyPrintStringBlocks() {
+  cat >test.yml <<EOL
+"removeQuotes": | 
+  "please"
+EOL
+
+  read -r -d '' expected << EOM
+removeQuotes: |
+  "please"
+EOM
+
+  X=$(./yq e --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testPrettyPrintWithExpression() {
+  cat >test.yml <<EOL
+a: {b: {c: ["cat"]}}
+EOL
+
+  read -r -d '' expected << EOM
+b:
+  c:
+    - cat
+EOM
+
+  X=$(./yq e '.a' --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea '.a' --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+}
+
+source ./scripts/shunit2

action.yml

@@ -1,11 +1,15 @@
 name: 'yq - portable yaml processor'
 description: 'create, read, update, delete, merge, validate and do more with yaml'
-icon: command
-color: gray-dark
+branding:
+  icon: command
+  color: gray-dark
 inputs:
   cmd:
     description: 'The Command which should be run'
     required: true
+outputs:
+  result:
+    description: "The complete result from the yq command being run"
 runs:
   using: 'docker'
   image: 'github-action/Dockerfile'

cmd/constant.go

@@ -1,9 +1,11 @@
 package cmd
 
+var leadingContentPreProcessing = true
 var unwrapScalar = true
 
 var writeInplace = false
 var outputToJSON = false
+var outputFormat = "yaml"
 
 var exitStatus = false
 var forceColor = false
@@ -16,4 +18,7 @@ var verbose = false
 var version = false
 var prettyPrint = false
 
+// can be either "" (off), "extract" or "process"
+var frontMatter = ""
+
 var completedSuccessfully = false

cmd/evaluate_all_command.go

@@ -15,13 +15,27 @@ func createEvaluateAllCommand() *cobra.Command {
 		Aliases: []string{"ea"},
 		Short:   "Loads _all_ yaml documents of _all_ yaml files and runs expression once",
 		Example: `
-# merges f2.yml into f1.yml (inplace)
-yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' f1.yml f2.yml		
+# Merge f2.yml into f1.yml (inplace)
+yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' f1.yml f2.yml
+## the same command and expression using shortened names:
+yq ea -i 'select(fi == 0) * select(fi == 1)' f1.yml f2.yml
 
-# use '-' as a filename to read from STDIN
+
+# Merge all given files
+yq ea '. as $item ireduce ({}; . * $item )' file1.yml file2.yml ...
+
+# Pipe from STDIN
+## use '-' as a filename to pipe from STDIN
 cat file2.yml | yq ea '.a.b' file1.yml - file3.yml
 `,
-		Long: "Evaluate All:\nUseful when you need to run an expression across several yaml documents or files. Consumes more memory than eval",
+		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
+See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.
+
+## Evaluate All ##
+This command loads _all_ yaml documents of _all_ yaml files and runs expression once
+Useful when you need to run an expression across several yaml documents or files (like merge).
+Note that it consumes more memory than eval.
+`,
 		RunE: evaluateAll,
 	}
 	return cmdEvalAll
@@ -72,26 +86,51 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 	if nullInput && len(args) > 1 {
 		return errors.New("Cannot pass files in when using null-input flag")
 	}
+	// backwards compatibilty
+	if outputToJSON {
+		outputFormat = "json"
+	}
 
-	printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+	format, err := yqlib.OutputFormatFromString(outputFormat)
+	if err != nil {
+		return err
+	}
+
+	printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+
+	if frontMatter != "" {
+		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])
+		err = frontMatterHandler.Split()
+		if err != nil {
+			return err
+		}
+		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
+
+		if frontMatter == "process" {
+			reader := frontMatterHandler.GetContentReader()
+			printer.SetAppendix(reader)
+			defer yqlib.SafelyCloseReader(reader)
+		}
+		defer frontMatterHandler.CleanUp()
+	}
 
 	allAtOnceEvaluator := yqlib.NewAllAtOnceEvaluator()
 	switch len(args) {
 	case 0:
 		if pipingStdIn {
-			err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer)
+			err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer, leadingContentPreProcessing)
 		} else {
 			cmd.Println(cmd.UsageString())
 			return nil
 		}
 	case 1:
 		if nullInput {
-			err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(args[0]), printer)
+			err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(args[0]), printer, "")
 		} else {
-			err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer)
+			err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer, leadingContentPreProcessing)
 		}
 	default:
-		err = allAtOnceEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer)
+		err = allAtOnceEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing)
 	}
 
 	completedSuccessfully = err == nil

cmd/evalute_sequence_command.go

@@ -13,35 +13,42 @@ func createEvaluateSequenceCommand() *cobra.Command {
 	var cmdEvalSequence = &cobra.Command{
 		Use:     "eval [expression] [yaml_file1]...",
 		Aliases: []string{"e"},
-		Short:   "Apply expression to each document in each yaml file given in sequence",
+		Short:   "Apply the expression to each document in each yaml file in sequence",
 		Example: `
-# runs the expression against each file, in series
-yq e '.a.b | length' f1.yml f2.yml 
+# Reads field under the given path for each file
+yq e '.a.b' f1.yml f2.yml 
 
-# prints out the file
+# Prints out the file
 yq e sample.yaml 
 
-# use '-' as a filename to read from STDIN
+# Pipe from STDIN
+## use '-' as a filename to pipe from STDIN
 cat file2.yml | yq e '.a.b' file1.yml - file3.yml
 
-# prints a new yaml document
+# Creates a new yaml document
+## Note that editing an empty file does not work.
 yq e -n '.a.b.c = "cat"' 
 
-
-# updates file.yaml directly
+# Update a file inplace
 yq e '.a.b = "cool"' -i file.yaml 
 `,
-		Long: "Evaluate Sequence:\nIterate over each yaml document, apply the expression and print the results, in sequence.",
+		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
+See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.
+
+## Evaluate Sequence ##
+This command iterates over each yaml document from each given file, applies the 
+expression and prints the result in sequence.`,
 		RunE: evaluateSequence,
 	}
 	return cmdEvalSequence
 }
 
 func processExpression(expression string) string {
+	var prettyPrintExp = `(... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not))  ) style=""`
 	if prettyPrint && expression == "" {
-		return `... style=""`
+		return prettyPrintExp
 	} else if prettyPrint {
-		return fmt.Sprintf("%v | ... style= \"\"", expression)
+		return fmt.Sprintf("%v | %v", expression, prettyPrintExp)
 	}
 	return expression
 }
@@ -89,7 +96,17 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 		defer func() { writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully) }()
 	}
 
-	printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+	// backwards compatibilty
+	if outputToJSON {
+		outputFormat = "json"
+	}
+
+	format, err := yqlib.OutputFormatFromString(outputFormat)
+	if err != nil {
+		return err
+	}
+
+	printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
 
 	streamEvaluator := yqlib.NewStreamEvaluator()
 
@@ -97,22 +114,38 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 		return errors.New("Cannot pass files in when using null-input flag")
 	}
 
+	if frontMatter != "" {
+		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])
+		err = frontMatterHandler.Split()
+		if err != nil {
+			return err
+		}
+		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
+
+		if frontMatter == "process" {
+			reader := frontMatterHandler.GetContentReader()
+			printer.SetAppendix(reader)
+			defer yqlib.SafelyCloseReader(reader)
+		}
+		defer frontMatterHandler.CleanUp()
+	}
+
 	switch len(args) {
 	case 0:
 		if pipingStdIn {
-			err = streamEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer)
+			err = streamEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer, leadingContentPreProcessing)
 		} else {
 			cmd.Println(cmd.UsageString())
 			return nil
 		}
 	case 1:
 		if nullInput {
-			err = streamEvaluator.EvaluateNew(processExpression(args[0]), printer)
+			err = streamEvaluator.EvaluateNew(processExpression(args[0]), printer, "")
 		} else {
-			err = streamEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer)
+			err = streamEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer, leadingContentPreProcessing)
 		}
 	default:
-		err = streamEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer)
+		err = streamEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing)
 	}
 	completedSuccessfully = err == nil
 

cmd/root.go

@@ -11,7 +11,8 @@ func New() *cobra.Command {
 	var rootCmd = &cobra.Command{
 		Use:   "yq",
 		Short: "yq is a lightweight and portable command-line YAML processor.",
-		Long:  `yq is a lightweight and portable command-line YAML processor. It aims to be the jq or sed of yaml files.`,
+		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
+See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if version {
 				cmd.Print(GetVersionDisplay())
@@ -40,7 +41,14 @@ func New() *cobra.Command {
 	}
 
 	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "verbose mode")
-	rootCmd.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "output as json. Set indent to 0 to print json in one line.")
+
+	rootCmd.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "(deprecated) output as json. Set indent to 0 to print json in one line.")
+	err := rootCmd.PersistentFlags().MarkDeprecated("tojson", "please use -o=json instead")
+	if err != nil {
+		panic(err)
+	}
+
+	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "yaml", "[yaml|y|json|j|props|p] output format type.")
 	rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.")
 	rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)")
 
@@ -53,6 +61,8 @@ func New() *cobra.Command {
 
 	rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")
 	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
+	rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
+	rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and seperators before processing expression. This is a workaround for go-yaml to persist header content. This flag will be removed once this feature has been out in the wild for a while.")
 	rootCmd.AddCommand(
 		createEvaluateSequenceCommand(),
 		createEvaluateAllCommand(),

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.9.3"
+	Version = "4.13.1"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release
@@ -25,7 +25,7 @@ const ProductName = "yq"
 // GetVersionDisplay composes the parts of the version in a way that's suitable
 // for displaying to humans.
 func GetVersionDisplay() string {
-	return fmt.Sprintf("%s version %s\n", ProductName, getHumanVersion())
+	return fmt.Sprintf("yq (https://github.com/mikefarah/yq/) version %s\n", getHumanVersion())
 }
 
 func getHumanVersion() string {

cmd/version_test.go

@@ -3,7 +3,7 @@ package cmd
 import "testing"
 
 func TestGetVersionDisplay(t *testing.T) {
-	var expectedVersion = ProductName + " version " + Version
+	var expectedVersion = ProductName + " (https://github.com/mikefarah/yq/) version " + Version
 	if VersionPrerelease != "" {
 		expectedVersion = expectedVersion + "-" + VersionPrerelease
 	}

debian/changelog

@@ -1,3 +1,28 @@
+yq (4.13.0) focal; urgency=medium
+
+  * New `with` operator for making multiple changes to a given path
+  * New `contains` operator, works like the `jq` equivalent
+  * Subtract operator now supports subtracting elements from arrays!
+  * Fixed Swapping values using variables #934
+  * Github Action now properly supports multiline output #936, thanks @pjxiao
+  * Fixed missing closing bracket validation #932
+  * Fix processing of hex numbers #929
+  * Fixed alternative and union operator issues #930
+  * Can now convert yaml to properties properties format (`-o=props`), See [docs](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties) for more info.
+  * Fixed document header/footer comment handling when merging (https://github.com/mikefarah/yq/issues/919)
+  * pretty print yaml 1.1 compatibility  (https://github.com/mikefarah/yq/issues/914)
+
+ -- Roberto Mier Escandon <rmescandon@gmail.com>  Thu, 16 Sep 2021 20:58:30 +0200
+
+yq (4.9.6) focal; urgency=medium
+
+  * Added darwin/arm64 build, thanks @alecthomas
+  * Incremented docker alpine base version, thanks @da6d6i7-bronga
+  * Bug fix: multine expression
+  * Bug fix: special character
+
+ -- Roberto Mier Escandon <rmescandon@gmail.com>  Tue, 29 Jun 2021 21:32:14 +0200
+
 yq (3.3.2) focal; urgency=medium
 
   * Bug fix: existStatus bug (#459)

debian/control

@@ -3,8 +3,7 @@ Section: devel
 Priority: optional
 Maintainer: Roberto Mier Escandón <rmescandon@gmail.com>
 Build-Depends: debhelper (>=10),
-               dh-golang (>=1.34),
-               golang-1.13,
+               golang-1.15,
                rsync
 Standards-Version: 4.1.4
 Homepage: https://github.com/mikefarah/yq.git

debian/rules

@@ -17,7 +17,7 @@
 PROJECT := yq
 OWNER := mikefarah
 REPO := github.com
-GOVERSION := 1.13
+GOVERSION := 1.15
 
 export DH_OPTIONS
 export DH_GOPKG := ${REPO}/${OWNER}/${PROJECT}
@@ -34,7 +34,7 @@ BINDIR := /usr/bin
 ASSETSDIR := /usr/share/${PROJECT}
 
 %:
-	dh $@ --builddirectory=${GOPATH} --buildsystem=golang
+	dh $@
 
 override_dh_auto_build:
 	mkdir -p ${SRCDIR}

examples/data1.yaml

@@ -1,6 +1,5 @@
-a:
-  key1: "value1"
-  key2: 2.6
-  ab:
-    key1: 6
-    key2: "h"
+Resources:
+  S3Bucket:
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      BucketName: !Ref MyBucketNameA

examples/data2.yaml

@@ -1,7 +1,3 @@
-a: other # better than the original
-b: [3, 4]
-c:
-  toast: leave
-  test: 1
-  tell: 1
-  tasty.taco: cool
+#b1 
+b: 2
+#b2

examples/empty.yaml

@@ -1,2 +1 @@
-# a: apple
-# b: cat
+# comment

examples/front-matter.yaml

@@ -0,0 +1,6 @@
+---
+a: apple
+b: bannana
+---
+hello there
+apples: great

examples/leading-seperator.yaml

@@ -0,0 +1,2 @@
+---
+a: test

github-action/Dockerfile

@@ -1,5 +1,8 @@
-FROM mikefarah/yq:4.9.3
+FROM mikefarah/yq:4.13.1
 
 COPY entrypoint.sh /entrypoint.sh
 
+# this seems to be the default user in github actions
+USER 1001
+
 ENTRYPOINT ["/entrypoint.sh"]

github-action/entrypoint.sh

@@ -1,4 +1,9 @@
 #!/bin/sh -l
-
-echo "$1"
-eval $1
+set -e
+echo "::debug::\$cmd: $1"
+RESULT=$(eval "$1")
+RESULT="${RESULT//'%'/'%25'}"
+RESULT="${RESULT//$'\n'/'%0A'}"
+RESULT="${RESULT//$'\r'/'%0D'}"
+echo "::debug::\$RESULT: $RESULT"
+echo ::set-output name=result::"$RESULT"

go.mod

@@ -5,10 +5,11 @@ require (
 	github.com/fatih/color v1.10.0
 	github.com/goccy/go-yaml v1.8.9
 	github.com/jinzhu/copier v0.2.8
+	github.com/magiconair/properties v1.8.5
 	github.com/spf13/cobra v1.1.3
 	github.com/timtadh/data-structures v0.5.3 // indirect
 	github.com/timtadh/lexmachine v0.2.2
-	golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e // indirect
+	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )

go.sum

@@ -119,6 +119,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls=
+github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
@@ -256,8 +258,8 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e h1:XNp2Flc/1eWQGk5BLzqTAN7fQIwIbfyVTuVxXxZh73M=
-golang.org/x/sys v0.0.0-20210317225723-c4fcb01b228e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=

pkg/yqlib/all_at_once_evaluator.go

@@ -8,7 +8,7 @@ import (
 
 // A yaml expression evaluator that runs the expression once against all files/nodes in memory.
 type Evaluator interface {
-	EvaluateFiles(expression string, filenames []string, printer Printer) error
+	EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error
 
 	// EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes
 	EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error)
@@ -46,15 +46,21 @@ func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCand
 	return context.MatchingNodes, nil
 }
 
-func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error {
+func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error {
 	fileIndex := 0
+	firstFileLeadingContent := ""
 
 	var allDocuments *list.List = list.New()
 	for _, filename := range filenames {
-		reader, err := readStream(filename)
+		reader, leadingContent, err := readStream(filename, fileIndex == 0 && leadingContentPreProcessing)
 		if err != nil {
 			return err
 		}
+
+		if fileIndex == 0 {
+			firstFileLeadingContent = leadingContent
+		}
+
 		fileDocuments, err := readDocuments(reader, filename, fileIndex)
 		if err != nil {
 			return err
@@ -62,6 +68,19 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
 		allDocuments.PushBackList(fileDocuments)
 		fileIndex = fileIndex + 1
 	}
+
+	if allDocuments.Len() == 0 {
+		candidateNode := &CandidateNode{
+			Document:  0,
+			Filename:  "",
+			Node:      &yaml.Node{Kind: yaml.DocumentNode, HeadComment: firstFileLeadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
+			FileIndex: 0,
+		}
+		allDocuments.PushBack(candidateNode)
+	} else {
+		allDocuments.Front().Value.(*CandidateNode).Node.HeadComment = firstFileLeadingContent
+	}
+
 	matches, err := e.EvaluateCandidateNodes(expression, allDocuments)
 	if err != nil {
 		return err

pkg/yqlib/context.go

@@ -2,8 +2,10 @@ package yqlib
 
 import (
 	"container/list"
+	"fmt"
 
 	"github.com/jinzhu/copier"
+	logging "gopkg.in/op/go-logging.v1"
 )
 
 type Context struct {
@@ -51,9 +53,39 @@ func (n *Context) ChildContext(results *list.List) Context {
 	return clone
 }
 
+func (n *Context) ToString() string {
+	if !log.IsEnabledFor(logging.DEBUG) {
+		return ""
+	}
+	result := fmt.Sprintf("Context\nDontAutoCreate: %v\n", n.DontAutoCreate)
+	return result + NodesToString(n.MatchingNodes)
+}
+
+func (n *Context) DeepClone() Context {
+	clone := Context{}
+	err := copier.Copy(&clone, n)
+	// copier doesn't do lists properly for some reason
+	clone.MatchingNodes = list.New()
+	for el := n.MatchingNodes.Front(); el != nil; el = el.Next() {
+		clonedNode, err := el.Value.(*CandidateNode).Copy()
+		if err != nil {
+			log.Error("Error cloning context :(")
+			panic(err)
+		}
+		clone.MatchingNodes.PushBack(clonedNode)
+	}
+
+	if err != nil {
+		log.Error("Error cloning context :(")
+		panic(err)
+	}
+	return clone
+}
+
 func (n *Context) Clone() Context {
 	clone := Context{}
 	err := copier.Copy(&clone, n)
+
 	if err != nil {
 		log.Error("Error cloning context :(")
 		panic(err)
@@ -66,3 +98,9 @@ func (n *Context) ReadOnlyClone() Context {
 	clone.DontAutoCreate = true
 	return clone
 }
+
+func (n *Context) WritableClone() Context {
+	clone := n.Clone()
+	clone.DontAutoCreate = false
+	return clone
+}

pkg/yqlib/doc/Add.md

@@ -127,6 +127,32 @@ b:
   - 4
 ```
 
+## Append another array using +=
+Given a sample.yml file of:
+```yaml
+a:
+  - 1
+  - 2
+b:
+  - 3
+  - 4
+```
+then
+```bash
+yq eval '.a += .b' sample.yml
+```
+will output
+```yaml
+a:
+  - 1
+  - 2
+  - 3
+  - 4
+b:
+  - 3
+  - 4
+```
+
 ## Relative append
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Anchor and Alias Operators.md

@@ -299,3 +299,33 @@ foobar:
   thing: foobar_thing
 ```
 
+## Dereference and update a field
+`Use explode with multiply to dereference an object
+
+Given a sample.yml file of:
+```yaml
+item_value: &item_value
+  value: true
+thingOne:
+  name: item_1
+  !!merge <<: *item_value
+thingTwo:
+  name: item_2
+  !!merge <<: *item_value
+```
+then
+```bash
+yq eval '.thingOne |= explode(.) * {"value": false}' sample.yml
+```
+will output
+```yaml
+item_value: &item_value
+  value: true
+thingOne:
+  name: item_1
+  value: false
+thingTwo:
+  name: item_2
+  !!merge <<: *item_value
+```
+

pkg/yqlib/doc/Assign (Update).md

@@ -28,6 +28,24 @@ a:
   g: foof
 ```
 
+## Double elements in an array
+Given a sample.yml file of:
+```yaml
+- 1
+- 2
+- 3
+```
+then
+```bash
+yq eval '.[] |= . * 2' sample.yml
+```
+will output
+```yaml
+- 2
+- 4
+- 6
+```
+
 ## Update node from another file
 Note this will also work when the second file is a scalar (string/number)
 
@@ -75,7 +93,7 @@ c: fieldC
 ```
 then
 ```bash
-yq eval '(.a, .c) |= "potatoe"' sample.yml
+yq eval '(.a, .c) = "potatoe"' sample.yml
 ```
 will output
 ```yaml

pkg/yqlib/doc/Contains.md

@@ -0,0 +1,85 @@
+
+## Array contains array
+Array is equal or subset of
+
+Given a sample.yml file of:
+```yaml
+- foobar
+- foobaz
+- blarp
+```
+then
+```bash
+yq eval 'contains(["baz", "bar"])' sample.yml
+```
+will output
+```yaml
+true
+```
+
+## Object included in array
+Given a sample.yml file of:
+```yaml
+"foo": 12
+"bar":
+  - 1
+  - 2
+  - "barp": 12
+    "blip": 13
+```
+then
+```bash
+yq eval 'contains({"bar": [{"barp": 12}]})' sample.yml
+```
+will output
+```yaml
+true
+```
+
+## Object not included in array
+Given a sample.yml file of:
+```yaml
+"foo": 12
+"bar":
+  - 1
+  - 2
+  - "barp": 12
+    "blip": 13
+```
+then
+```bash
+yq eval 'contains({"foo": 12, "bar": [{"barp": 15}]})' sample.yml
+```
+will output
+```yaml
+false
+```
+
+## String contains substring
+Given a sample.yml file of:
+```yaml
+foobar
+```
+then
+```bash
+yq eval 'contains("bar")' sample.yml
+```
+will output
+```yaml
+true
+```
+
+## String equals string
+Given a sample.yml file of:
+```yaml
+meow
+```
+then
+```bash
+yq eval 'contains("meow")' sample.yml
+```
+will output
+```yaml
+true
+```
+

pkg/yqlib/doc/Entries.md

@@ -98,3 +98,21 @@ KEY_a: 1
 KEY_b: 2
 ```
 
+## Use with_entries to filter the map
+Given a sample.yml file of:
+```yaml
+a:
+  b: bird
+c:
+  d: dog
+```
+then
+```bash
+yq eval 'with_entries(select(.value | has("b")))' sample.yml
+```
+will output
+```yaml
+a:
+  b: bird
+```
+

pkg/yqlib/doc/String Operators.md

@@ -1,5 +1,47 @@
 # String Operators
 
+## RegEx
+This uses golangs native regex functions under the hood - See https://github.com/google/re2/wiki/Syntax for the supported syntax.
+
+
+# String blocks, bash and newlines
+Bash is notorious for chomping on precious trailing newline characters, making it tricky to set strings with newlines properly. In particular, the `$( exp )` _will trim trailing newlines_.
+
+For instance to get this yaml:
+
+```
+a: |
+  cat
+```
+
+Using `$( exp )` wont work, as it will trim the trailing new line.
+
+```
+m=$(echo "cat\n") yq e -n '.a = strenv(m)'
+a: cat
+```
+
+However, using printf works:
+```
+printf -v m "cat\n" ; m="$m" yq e -n '.a = strenv(m)'
+a: |
+  cat
+```
+
+As well as having multiline expressions:
+```
+m="cat
+"  yq e -n '.a = strenv(m)'
+a: |
+  cat
+```
+
+Similarly, if you're trying to set the content from a file, and want a trailing new line:
+
+```
+IFS= read -rd '' output < <(cat my_file)
+output=$output ./yq e '.data.values = strenv(output)' first.yml
+```
 ## Join strings
 Given a sample.yml file of:
 ```yaml
@@ -18,6 +60,171 @@ will output
 cat; meow; 1; ; true
 ```
 
+## Match string
+Given a sample.yml file of:
+```yaml
+foo bar foo
+```
+then
+```bash
+yq eval 'match("foo")' sample.yml
+```
+will output
+```yaml
+string: foo
+offset: 0
+length: 3
+captures: []
+```
+
+## Match string, case insensitive
+Given a sample.yml file of:
+```yaml
+foo bar FOO
+```
+then
+```bash
+yq eval '[match("(?i)foo"; "g")]' sample.yml
+```
+will output
+```yaml
+- string: foo
+  offset: 0
+  length: 3
+  captures: []
+- string: FOO
+  offset: 8
+  length: 3
+  captures: []
+```
+
+## Match with capture groups
+Given a sample.yml file of:
+```yaml
+abc abc
+```
+then
+```bash
+yq eval '[match("(abc)+"; "g")]' sample.yml
+```
+will output
+```yaml
+- string: abc
+  offset: 0
+  length: 3
+  captures:
+    - string: abc
+      offset: 0
+      length: 3
+- string: abc
+  offset: 4
+  length: 3
+  captures:
+    - string: abc
+      offset: 4
+      length: 3
+```
+
+## Match with named capture groups
+Given a sample.yml file of:
+```yaml
+foo bar foo foo  foo
+```
+then
+```bash
+yq eval '[match("foo (?P<bar123>bar)? foo"; "g")]' sample.yml
+```
+will output
+```yaml
+- string: foo bar foo
+  offset: 0
+  length: 11
+  captures:
+    - string: bar
+      offset: 4
+      length: 3
+      name: bar123
+- string: foo  foo
+  offset: 12
+  length: 8
+  captures:
+    - string: null
+      offset: -1
+      length: 0
+      name: bar123
+```
+
+## Capture named groups into a map
+Given a sample.yml file of:
+```yaml
+xyzzy-14
+```
+then
+```bash
+yq eval 'capture("(?P<a>[a-z]+)-(?P<n>[0-9]+)")' sample.yml
+```
+will output
+```yaml
+a: xyzzy
+n: "14"
+```
+
+## Match without global flag
+Given a sample.yml file of:
+```yaml
+cat cat
+```
+then
+```bash
+yq eval 'match("cat")' sample.yml
+```
+will output
+```yaml
+string: cat
+offset: 0
+length: 3
+captures: []
+```
+
+## Match with global flag
+Given a sample.yml file of:
+```yaml
+cat cat
+```
+then
+```bash
+yq eval '[match("cat"; "g")]' sample.yml
+```
+will output
+```yaml
+- string: cat
+  offset: 0
+  length: 3
+  captures: []
+- string: cat
+  offset: 4
+  length: 3
+  captures: []
+```
+
+## Test using regex
+Like jq'q equivalant, this works like match but only returns true/false instead of full match details
+
+Given a sample.yml file of:
+```yaml
+- cat
+- dog
+```
+then
+```bash
+yq eval '.[] | test("at")' sample.yml
+```
+will output
+```yaml
+true
+false
+```
+
 ## Substitute / Replace string
 This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)
 Note the use of `|=` to run in context of the current string value.

pkg/yqlib/doc/Style.md

@@ -18,8 +18,6 @@ a:
 ```
 
 ## Update and set style of a particular node using path variables
-You can use a variable to re-use a path
-
 Given a sample.yml file of:
 ```yaml
 a:
@@ -28,7 +26,7 @@ a:
 ```
 then
 ```bash
-yq eval '.a.b as $x | $x = "new" | $x style="double"' sample.yml
+yq eval 'with(.a.b ; . = "new" | . style="double")' sample.yml
 ```
 will output
 ```yaml

pkg/yqlib/doc/Subtract.md

@@ -1,4 +1,43 @@
 
+## Array subtraction
+Running
+```bash
+yq eval --null-input '[1,2] - [2,3]'
+```
+will output
+```yaml
+- 1
+```
+
+## Array subtraction with nested array
+Running
+```bash
+yq eval --null-input '[[1], 1, 2] - [[1], 3]'
+```
+will output
+```yaml
+- 1
+- 2
+```
+
+## Array subtraction with nested object
+Note that order of the keys does not matter
+
+Given a sample.yml file of:
+```yaml
+- a: b
+  c: d
+- a: b
+```
+then
+```bash
+yq eval '. - [{"c": "d", "a": "b"}]' sample.yml
+```
+will output
+```yaml
+- a: b
+```
+
 ## Number subtraction - float
 If the lhs or rhs are floats then the expression will be calculated with floats.
 

pkg/yqlib/doc/Traverse (Read).md

@@ -1,4 +1,4 @@
-This is the simplest (and perhaps most used) operator, it is used to navigate deeply into yaml structurse.
+This is the simplest (and perhaps most used) operator, it is used to navigate deeply into yaml structures.
 ## Simple map navigation
 Given a sample.yml file of:
 ```yaml
@@ -63,6 +63,22 @@ will output
 frog
 ```
 
+## Nested special characters
+Given a sample.yml file of:
+```yaml
+a:
+  "key.withdots":
+    "another.key": apple
+```
+then
+```bash
+yq eval '.a["key.withdots"]["another.key"]' sample.yml
+```
+will output
+```yaml
+apple
+```
+
 ## Keys with spaces
 Use quotes with brackets around path elements with special characters
 
@@ -211,6 +227,20 @@ will output
 1
 ```
 
+## Traversing nested arrays by index
+Given a sample.yml file of:
+```yaml
+[[], [cat]]
+```
+then
+```bash
+yq eval '.[1][0]' sample.yml
+```
+will output
+```yaml
+cat
+```
+
 ## Maps with numeric keys
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Variable Operators.md

@@ -1,4 +1,6 @@
-For more complex scenarios, variables can be used to hold values of expression to be used in other expressions.
+Like the `jq` equivalents, variables are sometimes required for the more complex expressions (or swapping values between fields).
+
+Note that there is also an additional `ref` operator that holds a reference (instead of a copy) of the path, allowing you to make multiple changes to the same path.
 
 ## Single value variable
 Given a sample.yml file of:
@@ -56,3 +58,39 @@ title: A well-written article
 author: Person McPherson
 ```
 
+## Using variables to swap values
+Given a sample.yml file of:
+```yaml
+a: a_value
+b: b_value
+```
+then
+```bash
+yq eval '.a as $x  | .b as $y | .b = $x | .a = $y' sample.yml
+```
+will output
+```yaml
+a: b_value
+b: a_value
+```
+
+## Use ref to reference a path repeatedly
+Note: You may find the `with` operator more useful.
+
+Given a sample.yml file of:
+```yaml
+a:
+  b: thing
+  c: something
+```
+then
+```bash
+yq eval '.a.b ref $x | $x = "new" | $x style="double"' sample.yml
+```
+will output
+```yaml
+a:
+  b: "new"
+  c: something
+```
+

pkg/yqlib/doc/With.md

@@ -0,0 +1,62 @@
+Use the `with` operator to conveniently make multiple updates to a deeply nested path, or to update array elements relatively to each other. The first argument expression sets the root context, and the second expression runs against that root context.
+
+## Update and style
+Given a sample.yml file of:
+```yaml
+a:
+  deeply:
+    nested: value
+```
+then
+```bash
+yq eval 'with(.a.deeply.nested; . = "newValue" | . style="single")' sample.yml
+```
+will output
+```yaml
+a:
+  deeply:
+    nested: 'newValue'
+```
+
+## Update multiple deeply nested properties
+Given a sample.yml file of:
+```yaml
+a:
+  deeply:
+    nested: value
+    other: thing
+```
+then
+```bash
+yq eval 'with(.a.deeply; .nested = "newValue" | .other= "newThing")' sample.yml
+```
+will output
+```yaml
+a:
+  deeply:
+    nested: newValue
+    other: newThing
+```
+
+## Update array elements relatively
+The second expression runs with each element of the array as it's contextual root. This allows you to make updates relative to the element.
+
+Given a sample.yml file of:
+```yaml
+myArray:
+  - a: apple
+  - a: banana
+```
+then
+```bash
+yq eval 'with(.myArray[]; .b = .a + " yum")' sample.yml
+```
+will output
+```yaml
+myArray:
+  - a: apple
+    b: apple yum
+  - a: banana
+    b: banana yum
+```
+

pkg/yqlib/doc/aa.md

@@ -71,7 +71,7 @@ dog
 Both sides have now been evaluated, so now the operator copies across the value from the RHS to the value on the LHS, and it returns the now updated context:
 
 ```yaml
-a: cat
+a: dog
 b: dog
 ```
 

pkg/yqlib/doc/headers/String Operators.md

@@ -1 +1,44 @@
 # String Operators
+
+## RegEx
+This uses golangs native regex functions under the hood - See https://github.com/google/re2/wiki/Syntax for the supported syntax.
+
+
+# String blocks, bash and newlines
+Bash is notorious for chomping on precious trailing newline characters, making it tricky to set strings with newlines properly. In particular, the `$( exp )` _will trim trailing newlines_.
+
+For instance to get this yaml:
+
+```
+a: |
+  cat
+```
+
+Using `$( exp )` wont work, as it will trim the trailing new line.
+
+```
+m=$(echo "cat\n") yq e -n '.a = strenv(m)'
+a: cat
+```
+
+However, using printf works:
+```
+printf -v m "cat\n" ; m="$m" yq e -n '.a = strenv(m)'
+a: |
+  cat
+```
+
+As well as having multiline expressions:
+```
+m="cat
+"  yq e -n '.a = strenv(m)'
+a: |
+  cat
+```
+
+Similarly, if you're trying to set the content from a file, and want a trailing new line:
+
+```
+IFS= read -rd '' output < <(cat my_file)
+output=$output ./yq e '.data.values = strenv(output)' first.yml
+```

pkg/yqlib/doc/headers/Traverse (Read).md

@@ -1 +1 @@
-This is the simplest (and perhaps most used) operator, it is used to navigate deeply into yaml structurse.
+This is the simplest (and perhaps most used) operator, it is used to navigate deeply into yaml structures.

pkg/yqlib/doc/headers/Variable Operators.md

@@ -1 +1,3 @@
-For more complex scenarios, variables can be used to hold values of expression to be used in other expressions.
+Like the `jq` equivalents, variables are sometimes required for the more complex expressions (or swapping values between fields).
+
+Note that there is also an additional `ref` operator that holds a reference (instead of a copy) of the path, allowing you to make multiple changes to the same path.

pkg/yqlib/doc/headers/With.md

@@ -0,0 +1 @@
+Use the `with` operator to conveniently make multiple updates to a deeply nested path, or to update array elements relatively to each other. The first argument expression sets the root context, and the second expression runs against that root context.

pkg/yqlib/encoder_properties.go

@@ -0,0 +1,80 @@
+package yqlib
+
+import (
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/magiconair/properties"
+	yaml "gopkg.in/yaml.v3"
+)
+
+type propertiesEncoder struct {
+	destination io.Writer
+}
+
+func NewPropertiesEncoder(destination io.Writer) Encoder {
+	return &propertiesEncoder{destination}
+}
+
+func (pe *propertiesEncoder) Encode(node *yaml.Node) error {
+	mapKeysToStrings(node)
+	p := properties.NewProperties()
+	err := pe.doEncode(p, node, "")
+	if err != nil {
+		return err
+	}
+
+	_, err = p.WriteComment(pe.destination, "#", properties.UTF8)
+	return err
+}
+
+func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yaml.Node, path string) error {
+	p.SetComment(path,
+		strings.Replace(node.HeadComment, "#", "", 1)+
+			strings.Replace(node.LineComment, "#", "", 1))
+	switch node.Kind {
+	case yaml.ScalarNode:
+		_, _, err := p.Set(path, node.Value)
+		return err
+	case yaml.DocumentNode:
+		return pe.doEncode(p, node.Content[0], path)
+	case yaml.SequenceNode:
+		return pe.encodeArray(p, node.Content, path)
+	case yaml.MappingNode:
+		return pe.encodeMap(p, node.Content, path)
+	case yaml.AliasNode:
+		return pe.doEncode(p, node.Alias, path)
+	default:
+		return fmt.Errorf("Unsupported node %v", node.Tag)
+	}
+}
+
+func (pe *propertiesEncoder) appendPath(path string, key interface{}) string {
+	if path == "" {
+		return fmt.Sprintf("%v", key)
+	}
+	return fmt.Sprintf("%v.%v", path, key)
+}
+
+func (pe *propertiesEncoder) encodeArray(p *properties.Properties, kids []*yaml.Node, path string) error {
+	for index, child := range kids {
+		err := pe.doEncode(p, child, pe.appendPath(path, index))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (pe *propertiesEncoder) encodeMap(p *properties.Properties, kids []*yaml.Node, path string) error {
+	for index := 0; index < len(kids); index = index + 2 {
+		key := kids[index]
+		value := kids[index+1]
+		err := pe.doEncode(p, value, pe.appendPath(path, key.Value))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/yqlib/encoder_properties_test.go

@@ -0,0 +1,78 @@
+package yqlib
+
+import (
+	"bufio"
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+func yamlToProps(sampleYaml string) string {
+	var output bytes.Buffer
+	writer := bufio.NewWriter(&output)
+
+	var propsEncoder = NewPropertiesEncoder(writer)
+	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0)
+	if err != nil {
+		panic(err)
+	}
+	node := inputs.Front().Value.(*CandidateNode).Node
+	err = propsEncoder.Encode(node)
+	if err != nil {
+		panic(err)
+	}
+	writer.Flush()
+
+	return strings.TrimSuffix(output.String(), "\n")
+}
+
+func TestPropertiesEncoderSimple(t *testing.T) {
+	var sampleYaml = `a: 'bob cool'`
+
+	var expectedJson = `a = bob cool`
+	var actualProps = yamlToProps(sampleYaml)
+	test.AssertResult(t, expectedJson, actualProps)
+}
+
+func TestPropertiesEncoderSimpleWithComments(t *testing.T) {
+	var sampleYaml = `a: 'bob cool' # line`
+
+	var expectedJson = `# line
+a = bob cool`
+	var actualProps = yamlToProps(sampleYaml)
+	test.AssertResult(t, expectedJson, actualProps)
+}
+
+func TestPropertiesEncoderDeep(t *testing.T) {
+	var sampleYaml = `a: 
+  b: "bob cool"
+`
+
+	var expectedJson = `a.b = bob cool`
+	var actualProps = yamlToProps(sampleYaml)
+	test.AssertResult(t, expectedJson, actualProps)
+}
+
+func TestPropertiesEncoderDeepWithComments(t *testing.T) {
+	var sampleYaml = `a:  # a thing
+  b: "bob cool" # b thing
+`
+
+	var expectedJson = `# b thing
+a.b = bob cool`
+	var actualProps = yamlToProps(sampleYaml)
+	test.AssertResult(t, expectedJson, actualProps)
+}
+
+func TestPropertiesEncoderArray(t *testing.T) {
+	var sampleYaml = `a: 
+  b: [{c: dog}, {c: cat}]
+`
+
+	var expectedJson = `a.b.0.c = dog
+a.b.1.c = cat`
+	var actualProps = yamlToProps(sampleYaml)
+	test.AssertResult(t, expectedJson, actualProps)
+}

pkg/yqlib/expression_parser_test.go

@@ -6,37 +6,66 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
-func TestPathTreeNoArgsForTwoArgOp(t *testing.T) {
+func TestParserNoMatchingCloseCollect(t *testing.T) {
+	_, err := NewExpressionParser().ParseExpression("[1,2")
+	test.AssertResultComplex(t, "Bad expression, could not find matching `]`", err.Error())
+}
+func TestParserNoMatchingCloseObjectInCollect(t *testing.T) {
+	_, err := NewExpressionParser().ParseExpression(`[{"b": "c"]`)
+	test.AssertResultComplex(t, "Bad expression, could not find matching `}`", err.Error())
+}
+
+func TestParserNoMatchingCloseInCollect(t *testing.T) {
+	_, err := NewExpressionParser().ParseExpression(`[(.a]`)
+	test.AssertResultComplex(t, "Bad expression, could not find matching `)`", err.Error())
+}
+
+func TestParserNoMatchingCloseCollectObject(t *testing.T) {
+	_, err := NewExpressionParser().ParseExpression(`{"a": "b"`)
+	test.AssertResultComplex(t, "Bad expression, could not find matching `}`", err.Error())
+}
+
+func TestParserNoMatchingCloseCollectInCollectObject(t *testing.T) {
+	_, err := NewExpressionParser().ParseExpression(`{"b": [1}`)
+	test.AssertResultComplex(t, "Bad expression, could not find matching `]`", err.Error())
+}
+
+func TestParserNoMatchingCloseBracketInCollectObject(t *testing.T) {
+	_, err := NewExpressionParser().ParseExpression(`{"b": (1}`)
+	test.AssertResultComplex(t, "Bad expression, could not find matching `)`", err.Error())
+}
+
+func TestParserNoArgsForTwoArgOp(t *testing.T) {
 	_, err := NewExpressionParser().ParseExpression("=")
 	test.AssertResultComplex(t, "'=' expects 2 args but there is 0", err.Error())
 }
 
-func TestPathTreeOneLhsArgsForTwoArgOp(t *testing.T) {
+func TestParserOneLhsArgsForTwoArgOp(t *testing.T) {
 	_, err := NewExpressionParser().ParseExpression(".a =")
 	test.AssertResultComplex(t, "'=' expects 2 args but there is 1", err.Error())
 }
 
-func TestPathTreeOneRhsArgsForTwoArgOp(t *testing.T) {
+func TestParserOneRhsArgsForTwoArgOp(t *testing.T) {
 	_, err := NewExpressionParser().ParseExpression("= .a")
 	test.AssertResultComplex(t, "'=' expects 2 args but there is 1", err.Error())
 }
 
-func TestPathTreeTwoArgsForTwoArgOp(t *testing.T) {
+func TestParserTwoArgsForTwoArgOp(t *testing.T) {
 	_, err := NewExpressionParser().ParseExpression(".a = .b")
 	test.AssertResultComplex(t, nil, err)
 }
 
-func TestPathTreeNoArgsForOneArgOp(t *testing.T) {
+func TestParserNoArgsForOneArgOp(t *testing.T) {
 	_, err := NewExpressionParser().ParseExpression("explode")
 	test.AssertResultComplex(t, "'explode' expects 1 arg but received none", err.Error())
 }
 
-func TestPathTreeOneArgForOneArgOp(t *testing.T) {
+func TestParserOneArgForOneArgOp(t *testing.T) {
 	_, err := NewExpressionParser().ParseExpression("explode(.)")
 	test.AssertResultComplex(t, nil, err)
 }
 
-func TestPathTreeExtraArgs(t *testing.T) {
+func TestParserExtraArgs(t *testing.T) {
 	_, err := NewExpressionParser().ParseExpression("sortKeys(.) explode(.)")
 	test.AssertResultComplex(t, "Bad expression, please check expression syntax", err.Error())
 }

pkg/yqlib/expression_postfix.go

@@ -2,6 +2,7 @@ package yqlib
 
 import (
 	"errors"
+	"fmt"
 
 	logging "gopkg.in/op/go-logging.v1"
 )
@@ -24,6 +25,17 @@ func popOpToResult(opStack []*token, result []*Operation) ([]*token, []*Operatio
 	return opStack, append(result, newOp.Operation)
 }
 
+func validateNoOpenTokens(token *token) error {
+	if token.TokenType == openCollect {
+		return fmt.Errorf(("Bad expression, could not find matching `]`"))
+	} else if token.TokenType == openCollectObject {
+		return fmt.Errorf(("Bad expression, could not find matching `}`"))
+	} else if token.TokenType == openBracket {
+		return fmt.Errorf(("Bad expression, could not find matching `)`"))
+	}
+	return nil
+}
+
 func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Operation, error) {
 	var result []*Operation
 	// surround the whole thing with brackets
@@ -45,6 +57,10 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
 			}
 
 			for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != opener {
+				missingClosingTokenErr := validateNoOpenTokens(opStack[len(opStack)-1])
+				if missingClosingTokenErr != nil {
+					return nil, missingClosingTokenErr
+				}
 				opStack, result = popOpToResult(opStack, result)
 			}
 			if len(opStack) == 0 {
@@ -76,6 +92,11 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
 
 		case closeBracket:
 			for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != openBracket {
+				missingClosingTokenErr := validateNoOpenTokens(opStack[len(opStack)-1])
+				if missingClosingTokenErr != nil {
+					return nil, missingClosingTokenErr
+				}
+
 				opStack, result = popOpToResult(opStack, result)
 			}
 			if len(opStack) == 0 {
@@ -98,6 +119,8 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
 		}
 	}
 
+	log.Debugf("opstackLen: %v", len(opStack))
+
 	if log.IsEnabledFor(logging.DEBUG) {
 		log.Debugf("PostFix Result:")
 		for _, currentToken := range result {

pkg/yqlib/expression_processing_test.go

@@ -7,11 +7,39 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
+var variableWithNewLine = `"cat
+"`
+
 var pathTests = []struct {
 	path            string
 	expectedTokens  []interface{}
 	expectedPostFix []interface{}
 }{
+	{
+		"with(.a;.=3)",
+		append(make([]interface{}, 0), "WITH", "(", "a", "BLOCK", "SELF", "ASSIGN", "3 (int64)", ")"),
+		append(make([]interface{}, 0), "a", "SELF", "3 (int64)", "ASSIGN", "BLOCK", "WITH"),
+	},
+	{
+		"0x12",
+		append(make([]interface{}, 0), "18 (int64)"),
+		append(make([]interface{}, 0), "18 (int64)"),
+	},
+	{
+		"0X12",
+		append(make([]interface{}, 0), "18 (int64)"),
+		append(make([]interface{}, 0), "18 (int64)"),
+	},
+	{
+		".a\n",
+		append(make([]interface{}, 0), "a"),
+		append(make([]interface{}, 0), "a"),
+	},
+	{
+		variableWithNewLine,
+		append(make([]interface{}, 0), "cat\n (string)"),
+		append(make([]interface{}, 0), "cat\n (string)"),
+	},
 	{
 		`.[0]`,
 		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),

pkg/yqlib/expression_tokeniser.go

@@ -166,6 +166,20 @@ func numberValue() lex.Action {
 	}
 }
 
+func hexValue() lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		var originalString = string(m.Bytes)
+		var numberString = originalString[2:]
+		log.Debugf("numberString: %v", numberString)
+		var number, errParsingInt = strconv.ParseInt(numberString, 16, 64) // nolint
+		if errParsingInt != nil {
+			return nil, errParsingInt
+		}
+
+		return &token{TokenType: operationToken, Operation: createValueOperation(number, originalString)}, nil
+	}
+}
+
 func floatValue() lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		var numberString = string(m.Bytes)
@@ -276,11 +290,15 @@ func initLexer() (*lex.Lexer, error) {
 
 	lexer.Add([]byte(`join`), opToken(joinStringOpType))
 	lexer.Add([]byte(`sub`), opToken(subStringOpType))
+	lexer.Add([]byte(`match`), opToken(matchOpType))
+	lexer.Add([]byte(`capture`), opToken(captureOpType))
+	lexer.Add([]byte(`test`), opToken(testOpType))
 
 	lexer.Add([]byte(`any`), opToken(anyOpType))
 	lexer.Add([]byte(`any_c`), opToken(anyConditionOpType))
 	lexer.Add([]byte(`all`), opToken(allOpType))
 	lexer.Add([]byte(`all_c`), opToken(allConditionOpType))
+	lexer.Add([]byte(`contains`), opToken(containsOpType))
 
 	lexer.Add([]byte(`split`), opToken(splitStringOpType))
 	lexer.Add([]byte(`keys`), opToken(keysOpType))
@@ -298,6 +316,8 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`from_entries`), opToken(fromEntriesOpType))
 	lexer.Add([]byte(`with_entries`), opToken(withEntriesOpType))
 
+	lexer.Add([]byte(`with`), opToken(withOpType))
+
 	lexer.Add([]byte(`lineComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}))
 
 	lexer.Add([]byte(`headComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{HeadComment: true}))
@@ -320,11 +340,12 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte("( |\t|\n|\r)+"), skip)
 
 	lexer.Add([]byte(`\."[^ "]+"\??`), pathToken(true))
-	lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=]+\??`), pathToken(false))
+	lexer.Add([]byte(`\.[^ ;\}\{\:\[\],\|\.\[\(\)=\n]+\??`), pathToken(false))
 	lexer.Add([]byte(`\.`), selfToken())
 
 	lexer.Add([]byte(`\|`), opToken(pipeOpType))
 
+	lexer.Add([]byte(`0[xX][0-9A-Fa-f]+`), hexValue())
 	lexer.Add([]byte(`-?\d+(\.\d+)`), floatValue())
 	lexer.Add([]byte(`-?[1-9](\.\d+)?[Ee][-+]?\d+`), floatValue())
 	lexer.Add([]byte(`-?\d+`), numberValue())
@@ -349,7 +370,8 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`\-`), opToken(subtractOpType))
 	lexer.Add([]byte(`\-=`), opToken(subtractAssignOpType))
 	lexer.Add([]byte(`\$[a-zA-Z_-0-9]+`), getVariableOpToken())
-	lexer.Add([]byte(`as`), opToken(assignVariableOpType))
+	lexer.Add([]byte(`as`), opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{}))
+	lexer.Add([]byte(`ref`), opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{IsReference: true}))
 
 	err := lexer.CompileNFA()
 	if err != nil {

pkg/yqlib/file_utils.go

@@ -2,6 +2,7 @@ package yqlib
 
 import (
 	"io"
+	"io/ioutil"
 	"os"
 )
 
@@ -23,9 +24,20 @@ func safelyRenameFile(from string, to string) {
 	}
 }
 
+func tryRemoveFile(filename string) {
+	log.Debug("Removing temp file: %v", filename)
+	removeErr := os.Remove(filename)
+	if removeErr != nil {
+		log.Errorf("Failed to remove temp file: %v", filename)
+	}
+}
+
 // thanks https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang
 func copyFileContents(src, dst string) (err error) {
-	in, err := os.Open(src) // nolint gosec
+	// ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory,
+	// and ensuring that it's not possible to give a path to a file outside thar directory.
+
+	in, err := os.Open(src) // #nosec
 	if err != nil {
 		return err
 	}
@@ -41,6 +53,13 @@ func copyFileContents(src, dst string) (err error) {
 	return out.Sync()
 }
 
+func SafelyCloseReader(reader io.Reader) {
+	switch reader := reader.(type) {
+	case *os.File:
+		safelyCloseFile(reader)
+	}
+}
+
 func safelyCloseFile(file *os.File) {
 	err := file.Close()
 	if err != nil {
@@ -48,3 +67,22 @@ func safelyCloseFile(file *os.File) {
 		log.Error(err.Error())
 	}
 }
+
+func createTempFile() (*os.File, error) {
+	_, err := os.Stat(os.TempDir())
+	if os.IsNotExist(err) {
+		err = os.Mkdir(os.TempDir(), 0700)
+		if err != nil {
+			return nil, err
+		}
+	} else if err != nil {
+		return nil, err
+	}
+
+	file, err := ioutil.TempFile("", "temp")
+	if err != nil {
+		return nil, err
+	}
+
+	return file, err
+}

pkg/yqlib/front_matter.go

@@ -0,0 +1,93 @@
+package yqlib
+
+import (
+	"bufio"
+	"io"
+	"os"
+)
+
+type frontMatterHandler interface {
+	Split() error
+	GetYamlFrontMatterFilename() string
+	GetContentReader() io.Reader
+	CleanUp()
+}
+
+type frontMatterHandlerImpl struct {
+	originalFilename        string
+	yamlFrontMatterFilename string
+	contentReader           io.Reader
+}
+
+func NewFrontMatterHandler(originalFilename string) frontMatterHandler {
+	return &frontMatterHandlerImpl{originalFilename, "", nil}
+}
+
+func (f *frontMatterHandlerImpl) GetYamlFrontMatterFilename() string {
+	return f.yamlFrontMatterFilename
+}
+
+func (f *frontMatterHandlerImpl) GetContentReader() io.Reader {
+	return f.contentReader
+}
+
+func (f *frontMatterHandlerImpl) CleanUp() {
+	tryRemoveFile(f.yamlFrontMatterFilename)
+}
+
+// Splits the given file by yaml front matter
+// yaml content will be saved to first temporary file
+// remaining content will be saved to second temporary file
+func (f *frontMatterHandlerImpl) Split() error {
+	var reader *bufio.Reader
+	var err error
+	if f.originalFilename == "-" {
+		reader = bufio.NewReader(os.Stdin)
+	} else {
+		file, err := os.Open(f.originalFilename) // #nosec
+		if err != nil {
+			return err
+		}
+		reader = bufio.NewReader(file)
+	}
+	f.contentReader = reader
+
+	yamlTempFile, err := createTempFile()
+	if err != nil {
+		return err
+	}
+	f.yamlFrontMatterFilename = yamlTempFile.Name()
+	log.Debug("yamlTempFile: %v", yamlTempFile.Name())
+
+	lineCount := 0
+
+	for {
+		peekBytes, err := reader.Peek(3)
+		if err == io.EOF {
+			// we've finished reading the yaml content..I guess
+			break
+		} else if err != nil {
+			return err
+		}
+		if lineCount > 0 && string(peekBytes) == "---" {
+			// we've finished reading the yaml content..
+			break
+		}
+		line, errReading := reader.ReadString('\n')
+		lineCount = lineCount + 1
+		if errReading != nil && errReading != io.EOF {
+			return errReading
+		}
+
+		_, errWriting := yamlTempFile.Write([]byte(line))
+
+		if errWriting != nil {
+			return errWriting
+		}
+	}
+
+	safelyCloseFile(yamlTempFile)
+
+	return nil
+
+}

pkg/yqlib/front_matter_test.go

@@ -0,0 +1,142 @@
+package yqlib
+
+import (
+	"io/ioutil"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+func createTestFile(content string) string {
+	tempFile, err := createTempFile()
+	if err != nil {
+		panic(err)
+	}
+
+	_, err = tempFile.Write([]byte(content))
+	if err != nil {
+		panic(err)
+	}
+
+	safelyCloseFile(tempFile)
+
+	return tempFile.Name()
+}
+
+func readFile(filename string) string {
+	bytes, err := ioutil.ReadFile(filename)
+	if err != nil {
+		panic(err)
+	}
+	return string(bytes)
+}
+
+func TestFrontMatterSplitWithLeadingSep(t *testing.T) {
+	file := createTestFile(`---
+a: apple
+b: banana
+---
+not a 
+yaml: doc
+`)
+
+	expectedYamlFm := `---
+a: apple
+b: banana
+`
+
+	expectedContent := `---
+not a 
+yaml: doc
+`
+
+	fmHandler := NewFrontMatterHandler(file)
+	err := fmHandler.Split()
+	if err != nil {
+		panic(err)
+	}
+
+	yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename())
+
+	test.AssertResult(t, expectedYamlFm, yamlFm)
+
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
+	if err != nil {
+		panic(err)
+	}
+	test.AssertResult(t, expectedContent, string(contentBytes))
+
+	tryRemoveFile(file)
+	fmHandler.CleanUp()
+}
+
+func TestFrontMatterSplitWithNoLeadingSep(t *testing.T) {
+	file := createTestFile(`a: apple
+b: banana
+---
+not a 
+yaml: doc
+`)
+
+	expectedYamlFm := `a: apple
+b: banana
+`
+
+	expectedContent := `---
+not a 
+yaml: doc
+`
+
+	fmHandler := NewFrontMatterHandler(file)
+	err := fmHandler.Split()
+	if err != nil {
+		panic(err)
+	}
+
+	yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename())
+
+	test.AssertResult(t, expectedYamlFm, yamlFm)
+
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
+	if err != nil {
+		panic(err)
+	}
+	test.AssertResult(t, expectedContent, string(contentBytes))
+
+	tryRemoveFile(file)
+	fmHandler.CleanUp()
+}
+
+func TestFrontMatterSplitWithArray(t *testing.T) {
+	file := createTestFile(`[1,2,3]
+---
+not a 
+yaml: doc
+`)
+
+	expectedYamlFm := "[1,2,3]\n"
+
+	expectedContent := `---
+not a 
+yaml: doc
+`
+
+	fmHandler := NewFrontMatterHandler(file)
+	err := fmHandler.Split()
+	if err != nil {
+		panic(err)
+	}
+
+	yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename())
+
+	test.AssertResult(t, expectedYamlFm, yamlFm)
+
+	contentBytes, err := ioutil.ReadAll(fmHandler.GetContentReader())
+	if err != nil {
+		panic(err)
+	}
+	test.AssertResult(t, expectedContent, string(contentBytes))
+
+	tryRemoveFile(file)
+	fmHandler.CleanUp()
+}

pkg/yqlib/lib.go

@@ -6,6 +6,8 @@ import (
 	"bytes"
 	"container/list"
 	"fmt"
+	"strconv"
+	"strings"
 
 	logging "gopkg.in/op/go-logging.v1"
 	yaml "gopkg.in/yaml.v3"
@@ -60,6 +62,7 @@ var collectOpType = &operationType{Type: "COLLECT", NumArgs: 0, Precedence: 50,
 
 var anyOpType = &operationType{Type: "ANY", NumArgs: 0, Precedence: 50, Handler: anyOperator}
 var allOpType = &operationType{Type: "ALL", NumArgs: 0, Precedence: 50, Handler: allOperator}
+var containsOpType = &operationType{Type: "CONTAINS", NumArgs: 1, Precedence: 50, Handler: containsOperator}
 var anyConditionOpType = &operationType{Type: "ANY_CONDITION", NumArgs: 1, Precedence: 50, Handler: anyOperator}
 var allConditionOpType = &operationType{Type: "ALL_CONDITION", NumArgs: 1, Precedence: 50, Handler: allOperator}
 
@@ -67,6 +70,8 @@ var toEntriesOpType = &operationType{Type: "TO_ENTRIES", NumArgs: 0, Precedence:
 var fromEntriesOpType = &operationType{Type: "FROM_ENTRIES", NumArgs: 0, Precedence: 50, Handler: fromEntriesOperator}
 var withEntriesOpType = &operationType{Type: "WITH_ENTRIES", NumArgs: 1, Precedence: 50, Handler: withEntriesOperator}
 
+var withOpType = &operationType{Type: "WITH", NumArgs: 1, Precedence: 50, Handler: withOperator}
+
 var splitDocumentOpType = &operationType{Type: "SPLIT_DOC", NumArgs: 0, Precedence: 50, Handler: splitDocumentOperator}
 var getVariableOpType = &operationType{Type: "GET_VARIABLE", NumArgs: 0, Precedence: 55, Handler: getVariableOperator}
 var getStyleOpType = &operationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: getStyleOperator}
@@ -83,6 +88,9 @@ var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50,
 var sortKeysOpType = &operationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: sortKeysOperator}
 var joinStringOpType = &operationType{Type: "JOIN", NumArgs: 1, Precedence: 50, Handler: joinStringOperator}
 var subStringOpType = &operationType{Type: "SUBSTR", NumArgs: 1, Precedence: 50, Handler: substituteStringOperator}
+var matchOpType = &operationType{Type: "MATCH", NumArgs: 1, Precedence: 50, Handler: matchOperator}
+var captureOpType = &operationType{Type: "CAPTURE", NumArgs: 1, Precedence: 50, Handler: captureOperator}
+var testOpType = &operationType{Type: "TEST", NumArgs: 1, Precedence: 50, Handler: testOperator}
 var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 50, Handler: splitStringOperator}
 
 var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 50, Handler: keysOperator}
@@ -114,8 +122,77 @@ type Operation struct {
 	UpdateAssign  bool // used for assign ops, when true it means we evaluate the rhs given the lhs
 }
 
-func createValueOperation(value interface{}, stringValue string) *Operation {
-	var node yaml.Node = yaml.Node{Kind: yaml.ScalarNode}
+func recurseNodeArrayEqual(lhs *yaml.Node, rhs *yaml.Node) bool {
+	if len(lhs.Content) != len(rhs.Content) {
+		return false
+	}
+
+	for index := 0; index < len(lhs.Content); index = index + 1 {
+		if !recursiveNodeEqual(lhs.Content[index], rhs.Content[index]) {
+			return false
+		}
+	}
+	return true
+}
+
+func findInArray(array *yaml.Node, item *yaml.Node) int {
+
+	for index := 0; index < len(array.Content); index = index + 1 {
+		if recursiveNodeEqual(array.Content[index], item) {
+			return index
+		}
+	}
+	return -1
+}
+
+func recurseNodeObjectEqual(lhs *yaml.Node, rhs *yaml.Node) bool {
+	if len(lhs.Content) != len(rhs.Content) {
+		return false
+	}
+
+	for index := 0; index < len(lhs.Content); index = index + 2 {
+		key := lhs.Content[index]
+		value := lhs.Content[index+1]
+
+		indexInRhs := findInArray(rhs, key)
+
+		if indexInRhs == -1 || !recursiveNodeEqual(value, rhs.Content[indexInRhs+1]) {
+			return false
+		}
+	}
+	return true
+}
+
+func recursiveNodeEqual(lhs *yaml.Node, rhs *yaml.Node) bool {
+	if lhs.Kind != rhs.Kind || lhs.Tag != rhs.Tag {
+		return false
+	} else if lhs.Tag == "!!null" {
+		return true
+
+	} else if lhs.Kind == yaml.ScalarNode {
+		return lhs.Value == rhs.Value
+	} else if lhs.Kind == yaml.SequenceNode {
+		return recurseNodeArrayEqual(lhs, rhs)
+	} else if lhs.Kind == yaml.MappingNode {
+		return recurseNodeObjectEqual(lhs, rhs)
+	}
+	return false
+
+}
+
+// yaml numbers can be hex encoded...
+func parseInt(numberString string) (string, int64, error) {
+	if strings.HasPrefix(numberString, "0x") ||
+		strings.HasPrefix(numberString, "0X") {
+		num, err := strconv.ParseInt(numberString[2:], 16, 64) // nolint
+		return "0x%X", num, err
+	}
+	num, err := strconv.ParseInt(numberString, 10, 64) // nolint
+	return "%v", num, err
+}
+
+func createScalarNode(value interface{}, stringValue string) *yaml.Node {
+	var node = &yaml.Node{Kind: yaml.ScalarNode}
 	node.Value = stringValue
 
 	switch value.(type) {
@@ -130,17 +207,25 @@ func createValueOperation(value interface{}, stringValue string) *Operation {
 	case nil:
 		node.Tag = "!!null"
 	}
+	return node
+}
+
+func createValueOperation(value interface{}, stringValue string) *Operation {
+	var node *yaml.Node = createScalarNode(value, stringValue)
 
 	return &Operation{
 		OperationType: valueOpType,
 		Value:         value,
 		StringValue:   stringValue,
-		CandidateNode: &CandidateNode{Node: &node},
+		CandidateNode: &CandidateNode{Node: node},
 	}
 }
 
 // debugging purposes only
 func (p *Operation) toString() string {
+	if p == nil {
+		return "OP IS NIL"
+	}
 	if p.OperationType == traversePathOpType {
 		return fmt.Sprintf("%v", p.Value)
 	} else if p.OperationType == selfReferenceOpType {
@@ -158,7 +243,7 @@ func NodesToString(collection *list.List) string {
 		return ""
 	}
 
-	result := ""
+	result := fmt.Sprintf("%v results\n", collection.Len())
 	for el := collection.Front(); el != nil; el = el.Next() {
 		result = result + "\n" + NodeToString(el.Value.(*CandidateNode))
 	}
@@ -179,7 +264,10 @@ func NodeToString(node *CandidateNode) string {
 	if errorEncoding != nil {
 		log.Error("Error debugging node, %v", errorEncoding.Error())
 	}
-	encoder.Close()
+	errorClosingEncoder := encoder.Close()
+	if errorClosingEncoder != nil {
+		log.Error("Error closing encoder: ", errorClosingEncoder.Error())
+	}
 	tag := value.Tag
 	if value.Kind == yaml.DocumentNode {
 		tag = "doc"

pkg/yqlib/operator_add.go

@@ -15,11 +15,7 @@ func createAddOp(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode {
 }
 
 func addAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	assignmentOp := &Operation{OperationType: assignOpType}
-	assignmentOp.UpdateAssign = true
-	selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
-	assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: expressionNode.Lhs, Rhs: createAddOp(selfExpression, expressionNode.Rhs)}
-	return d.GetMatchingNodes(context, assignmentOpNode)
+	return compoundAssignFunction(d, context, expressionNode, createAddOp)
 }
 
 func toNodes(candidate *CandidateNode) []*yaml.Node {
@@ -80,17 +76,17 @@ func addScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*Candida
 		target.Node.Tag = "!!str"
 		target.Node.Value = lhs.Value + rhs.Value
 	} else if lhs.Tag == "!!int" && rhs.Tag == "!!int" {
-		lhsNum, err := strconv.Atoi(lhs.Value)
+		format, lhsNum, err := parseInt(lhs.Value)
 		if err != nil {
 			return nil, err
 		}
-		rhsNum, err := strconv.Atoi(rhs.Value)
+		_, rhsNum, err := parseInt(rhs.Value)
 		if err != nil {
 			return nil, err
 		}
 		sum := lhsNum + rhsNum
 		target.Node.Tag = "!!int"
-		target.Node.Value = fmt.Sprintf("%v", sum)
+		target.Node.Value = fmt.Sprintf(format, sum)
 	} else if (lhs.Tag == "!!int" || lhs.Tag == "!!float") && (rhs.Tag == "!!int" || rhs.Tag == "!!float") {
 		lhsNum, err := strconv.ParseFloat(lhs.Value, 64)
 		if err != nil {

pkg/yqlib/operator_add_test.go

@@ -86,6 +86,14 @@ var addOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: [1, 2, 3, 4], b: [3, 4]}\n",
 		},
 	},
+	{
+		description: "Append another array using +=",
+		document:    `{a: [1,2], b: [3,4]}`,
+		expression:  `.a += .b`,
+		expected: []string{
+			"D0, P[], (doc)::{a: [1, 2, 3, 4], b: [3, 4]}\n",
+		},
+	},
 	{
 		description: "Relative append",
 		document:    `a: { a1: {b: [cat]}, a2: {b: [dog]}, a3: {} }`,

pkg/yqlib/operator_alternative.go

@@ -2,13 +2,16 @@ package yqlib
 
 func alternativeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- alternative")
-	return crossFunction(d, context.ReadOnlyClone(), expressionNode, alternativeFunc, true)
+	return crossFunction(d, context, expressionNode, alternativeFunc, true)
 }
 
 func alternativeFunc(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	if lhs == nil {
 		return rhs, nil
 	}
+	if rhs == nil {
+		return lhs, nil
+	}
 	lhs.Node = unwrapDoc(lhs.Node)
 	rhs.Node = unwrapDoc(rhs.Node)
 	log.Debugf("Alternative LHS: %v", lhs.Node.Tag)

pkg/yqlib/operator_alternative_test.go

@@ -5,6 +5,15 @@ import (
 )
 
 var alternativeOperatorScenarios = []expressionScenario{
+	{
+		// to match jq - we do not use a readonly clone context on the LHS.
+		skipDoc:    true,
+		expression: `.b // .c`,
+		document:   `a: bridge`,
+		expected: []string{
+			"D0, P[c], (!!null)::null\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		expression: `(.b // "hello") as $x`,
@@ -13,6 +22,14 @@ var alternativeOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: bridge\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		expression: `.a // .b`,
+		document:   `a: 2`,
+		expected: []string{
+			"D0, P[a], (!!int)::2\n",
+		},
+	},
 	{
 		description: "LHS is defined",
 		expression:  `.a // "hello"`,

pkg/yqlib/operator_anchors_aliases_test.go

@@ -12,6 +12,29 @@ var specDocument = `- &CENTER { x: 1, y: 2 }
 
 var expectedSpecResult = "D0, P[4], (!!map)::x: 1\ny: 2\nr: 10\n"
 
+var simpleArrayRef = `
+item_value: &item_value
+  value: true
+
+thingOne:
+  name: item_1
+  <<: *item_value
+
+thingTwo:
+  name: item_2
+  <<: *item_value
+`
+
+var expectedUpdatedArrayRef = `D0, P[], (doc)::item_value: &item_value
+    value: true
+thingOne:
+    name: item_1
+    value: false
+thingTwo:
+    name: item_2
+    !!merge <<: *item_value
+`
+
 var anchorOperatorScenarios = []expressionScenario{
 	{
 		description:    "Merge one map",
@@ -197,6 +220,13 @@ foobar:
 			"D0, P[], (doc)::{f: {a: cat, b: {f: cat}, cat: {f: cat}}}\n",
 		},
 	},
+	{
+		description:    "Dereference and update a field",
+		subdescription: "`Use explode with multiply to dereference an object",
+		document:       simpleArrayRef,
+		expression:     `.thingOne |= explode(.) * {"value": false}`,
+		expected:       []string{expectedUpdatedArrayRef},
+	},
 }
 
 func TestAnchorAliasOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_assign.go

@@ -1,21 +1,27 @@
 package yqlib
 
+func assignUpdateFunc(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	rhs.Node = unwrapDoc(rhs.Node)
+	lhs.UpdateFrom(rhs)
+	return lhs, nil
+}
+
 func assignUpdateOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	lhs, err := d.GetMatchingNodes(context, expressionNode.Lhs)
 	if err != nil {
 		return Context{}, err
 	}
-	var rhs Context
+
 	if !expressionNode.Operation.UpdateAssign {
-		rhs, err = d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
+		// this works because we already ran against LHS with an editable context.
+		_, err := crossFunction(d, context.ReadOnlyClone(), expressionNode, assignUpdateFunc, false)
+		return context, err
 	}
 
 	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 
-		if expressionNode.Operation.UpdateAssign {
-			rhs, err = d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
-		}
+		rhs, err := d.GetMatchingNodes(context.SingleChildContext(candidate), expressionNode.Rhs)
 
 		if err != nil {
 			return Context{}, err

pkg/yqlib/operator_assign_test.go

@@ -36,6 +36,14 @@ var assignOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: {g: foof}}\n",
 		},
 	},
+	{
+		description: "Double elements in an array",
+		document:    `[1,2,3]`,
+		expression:  `.[] |= . * 2`,
+		expected: []string{
+			"D0, P[], (doc)::[2, 4, 6]\n",
+		},
+	},
 	{
 		description:    "Update node from another file",
 		subdescription: "Note this will also work when the second file is a scalar (string/number)",
@@ -57,7 +65,7 @@ var assignOperatorScenarios = []expressionScenario{
 	{
 		description: "Updated multiple paths",
 		document:    `{a: fieldA, b: fieldB, c: fieldC}`,
-		expression:  `(.a, .c) |= "potatoe"`,
+		expression:  `(.a, .c) = "potatoe"`,
 		expected: []string{
 			"D0, P[], (doc)::{a: potatoe, b: fieldB, c: potatoe}\n",
 		},

pkg/yqlib/operator_booleans_test.go

@@ -12,6 +12,22 @@ var booleanOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!bool)::true\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   "b: hi",
+		expression: `.a or .c`,
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "b: false",
+		expression: `.b or .c`,
+		expected: []string{
+			"D0, P[b], (!!bool)::false\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   "b: hi",

pkg/yqlib/operator_collect_object.go

@@ -20,8 +20,7 @@ import (
 func collectObjectOperator(d *dataTreeNavigator, originalContext Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- collectObjectOperation")
 
-	context := originalContext.Clone()
-	context.DontAutoCreate = false
+	context := originalContext.WritableClone()
 
 	if context.MatchingNodes.Len() == 0 {
 		node := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map", Value: "{}"}

pkg/yqlib/operator_contains.go

@@ -0,0 +1,111 @@
+package yqlib
+
+import (
+	"fmt"
+	"strings"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+func containsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, containsWithNodes, false)
+}
+
+func containsArrayElement(array *yaml.Node, item *yaml.Node) (bool, error) {
+	for index := 0; index < len(array.Content); index = index + 1 {
+		containedInArray, err := contains(array.Content[index], item)
+		if err != nil {
+			return false, err
+		}
+		if containedInArray {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func containsArray(lhs *yaml.Node, rhs *yaml.Node) (bool, error) {
+	if rhs.Kind != yaml.SequenceNode {
+		return containsArrayElement(lhs, rhs)
+	}
+	for index := 0; index < len(rhs.Content); index = index + 1 {
+		itemInArray, err := containsArrayElement(lhs, rhs.Content[index])
+		if err != nil {
+			return false, err
+		}
+		if !itemInArray {
+			return false, nil
+		}
+	}
+	return true, nil
+}
+
+func containsObject(lhs *yaml.Node, rhs *yaml.Node) (bool, error) {
+	if rhs.Kind != yaml.MappingNode {
+		return false, nil
+	}
+	for index := 0; index < len(rhs.Content); index = index + 2 {
+		rhsKey := rhs.Content[index]
+		rhsValue := rhs.Content[index+1]
+		log.Debugf("Looking for %v in the lhs", rhsKey.Value)
+		lhsKeyIndex := findInArray(lhs, rhsKey)
+		log.Debugf("index is %v", lhsKeyIndex)
+		if lhsKeyIndex < 0 || lhsKeyIndex%2 != 0 {
+			return false, nil
+		}
+		lhsValue := lhs.Content[lhsKeyIndex+1]
+		log.Debugf("lhsValue is %v", lhsValue.Value)
+
+		itemInArray, err := contains(lhsValue, rhsValue)
+		log.Debugf("rhsValue is %v", rhsValue.Value)
+		if err != nil {
+			return false, err
+		}
+		if !itemInArray {
+			return false, nil
+		}
+	}
+	return true, nil
+}
+
+func containsScalars(lhs *yaml.Node, rhs *yaml.Node) (bool, error) {
+	if lhs.Tag == "!!str" {
+		return strings.Contains(lhs.Value, rhs.Value), nil
+	}
+	return lhs.Value == rhs.Value, nil
+}
+
+func contains(lhs *yaml.Node, rhs *yaml.Node) (bool, error) {
+	switch lhs.Kind {
+	case yaml.MappingNode:
+		return containsObject(lhs, rhs)
+	case yaml.SequenceNode:
+		return containsArray(lhs, rhs)
+	case yaml.ScalarNode:
+		if rhs.Kind != yaml.ScalarNode || lhs.Tag != rhs.Tag {
+			return false, nil
+		}
+		if lhs.Tag == "!!null" {
+			return rhs.Tag == "!!null", nil
+		}
+		return containsScalars(lhs, rhs)
+	}
+
+	return false, fmt.Errorf("%v not yet supported for contains", lhs.Tag)
+}
+
+func containsWithNodes(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	lhs.Node = unwrapDoc(lhs.Node)
+	rhs.Node = unwrapDoc(rhs.Node)
+
+	if lhs.Node.Kind != rhs.Node.Kind {
+		return nil, fmt.Errorf("%v cannot check contained in %v", rhs.Node.Tag, lhs.Node.Tag)
+	}
+
+	result, err := contains(lhs.Node, rhs.Node)
+	if err != nil {
+		return nil, err
+	}
+
+	return createBooleanCandidate(lhs, result), nil
+}

pkg/yqlib/operator_contains_test.go

@@ -0,0 +1,82 @@
+package yqlib
+
+import "testing"
+
+var containsOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		expression: `null | contains(~)`,
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `3 | contains(3)`,
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `3 | contains(32)`,
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		description:    "Array contains array",
+		subdescription: "Array is equal or subset of",
+		document:       `["foobar", "foobaz", "blarp"]`,
+		expression:     `contains(["baz", "bar"])`,
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `["dog", "cat", "giraffe"] | contains(["camel"])`,
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		description: "Object included in array",
+		document:    `{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}`,
+		expression:  `contains({"bar": [{"barp": 12}]})`,
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		description: "Object not included in array",
+		document:    `{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}`,
+		expression:  `contains({"foo": 12, "bar": [{"barp": 15}]})`,
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		description: "String contains substring",
+		document:    `"foobar"`,
+		expression:  `contains("bar")`,
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		description: "String equals string",
+		document:    `"meow"`,
+		expression:  `contains("meow")`,
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+}
+
+func TestContainsOperatorScenarios(t *testing.T) {
+	for _, tt := range containsOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "Contains", containsOperatorScenarios)
+}

pkg/yqlib/operator_entries_test.go

@@ -52,6 +52,14 @@ var entriesOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!map)::KEY_a: 1\nKEY_b: 2\n",
 		},
 	},
+	{
+		description: "Use with_entries to filter the map",
+		document:    `{a: { b: bird }, c: { d: dog }}`,
+		expression:  `with_entries(select(.value | has("b")))`,
+		expected: []string{
+			"D0, P[], (!!map)::a: {b: bird}\n",
+		},
+	},
 }
 
 func TestEntriesOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_equals_test.go

@@ -5,6 +5,29 @@ import (
 )
 
 var equalsOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		expression: ".a == .b",
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: cat`,
+		expression: ".a == .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: cat`,
+		expression: ".b == .a",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   "cat",

pkg/yqlib/operator_multiply.go

@@ -20,8 +20,29 @@ func multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *Exp
 	return crossFunction(d, context, expressionNode, multiply(expressionNode.Operation.Preferences.(multiplyPreferences)), false)
 }
 
+func getNewBlankNode(lhs *yaml.Node, rhs *yaml.Node) *yaml.Node {
+
+	blankNode := &yaml.Node{}
+
+	if lhs.HeadComment != "" {
+		blankNode.HeadComment = lhs.HeadComment
+	} else if rhs.HeadComment != "" {
+		blankNode.HeadComment = rhs.HeadComment
+	}
+
+	if lhs.FootComment != "" {
+		blankNode.FootComment = lhs.FootComment
+	} else if rhs.FootComment != "" {
+		blankNode.FootComment = rhs.FootComment
+	}
+
+	return blankNode
+}
+
 func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+		// need to do this before unWrapping the potential document node
+		newBlankNode := getNewBlankNode(lhs.Node, rhs.Node)
 		lhs.Node = unwrapDoc(lhs.Node)
 		rhs.Node = unwrapDoc(rhs.Node)
 		log.Debugf("Multipling LHS: %v", lhs.Node.Tag)
@@ -30,13 +51,13 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
 		if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode ||
 			(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
 
-			var newBlank = lhs.CreateChild(nil, &yaml.Node{})
-
-			var newThing, err = mergeObjects(d, context, newBlank, lhs, multiplyPreferences{})
+			var newBlank = lhs.CreateChild(nil, newBlankNode)
+			log.Debugf("merge - merge lhs into blank")
+			var newThing, err = mergeObjects(d, context.WritableClone(), newBlank, lhs, multiplyPreferences{})
 			if err != nil {
 				return nil, err
 			}
-			return mergeObjects(d, context, newThing, rhs, preferences)
+			return mergeObjects(d, context.WritableClone(), newThing, rhs, preferences)
 		} else if lhs.Node.Tag == "!!int" && rhs.Node.Tag == "!!int" {
 			return multiplyIntegers(lhs, rhs)
 		} else if (lhs.Node.Tag == "!!int" || lhs.Node.Tag == "!!float") && (rhs.Node.Tag == "!!int" || rhs.Node.Tag == "!!float") {
@@ -70,25 +91,26 @@ func multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, e
 	target.Node.Style = lhs.Node.Style
 	target.Node.Tag = "!!int"
 
-	lhsNum, err := strconv.Atoi(lhs.Node.Value)
+	format, lhsNum, err := parseInt(lhs.Node.Value)
 	if err != nil {
 		return nil, err
 	}
-	rhsNum, err := strconv.Atoi(rhs.Node.Value)
+	_, rhsNum, err := parseInt(rhs.Node.Value)
 	if err != nil {
 		return nil, err
 	}
-	target.Node.Value = fmt.Sprintf("%v", lhsNum*rhsNum)
+	target.Node.Value = fmt.Sprintf(format, lhsNum*rhsNum)
 	return target, nil
 }
 
 func mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode, preferences multiplyPreferences) (*CandidateNode, error) {
-	shouldAppendArrays := preferences.AppendArrays
 	var results = list.New()
 
-	// shouldn't recurse arrays if appending
-	prefs := recursiveDescentPreferences{RecurseArray: !shouldAppendArrays,
+	// only need to recurse the array if we are doing a deep merge
+	prefs := recursiveDescentPreferences{RecurseArray: preferences.DeepMergeArrays,
 		TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: true}}
+	log.Debugf("merge - preferences.DeepMergeArrays %v", preferences.DeepMergeArrays)
+	log.Debugf("merge - preferences.AppendArrays %v", preferences.AppendArrays)
 	err := recursiveDecent(d, results, context.SingleChildContext(rhs), prefs)
 	if err != nil {
 		return nil, err
@@ -118,14 +140,22 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom
 	log.Debugf("merge - applyAssignment lhs %v, rhs: %v", lhs.GetKey(), rhs.GetKey())
 
 	lhsPath := rhs.Path[pathIndexToStartFrom:]
+	log.Debugf("merge - lhsPath %v", lhsPath)
 
 	assignmentOp := &Operation{OperationType: assignAttributesOpType}
 	if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode {
 		assignmentOp.OperationType = addAssignOpType
+		log.Debugf("merge - assignmentOp.OperationType = addAssignOpType")
 	} else if !preferences.DeepMergeArrays && rhs.Node.Kind == yaml.SequenceNode ||
 		(rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode) {
 		assignmentOp.OperationType = assignOpType
 		assignmentOp.UpdateAssign = false
+		log.Debugf("merge - rhs.Node.Kind == yaml.SequenceNode: %v", rhs.Node.Kind == yaml.SequenceNode)
+		log.Debugf("merge - rhs.Node.Kind == yaml.ScalarNode: %v", rhs.Node.Kind == yaml.ScalarNode)
+		log.Debugf("merge - rhs.Node.Kind == yaml.AliasNode: %v", rhs.Node.Kind == yaml.AliasNode)
+		log.Debugf("merge - assignmentOp.OperationType = assignOpType, no updateassign")
+	} else {
+		log.Debugf("merge - assignmentOp := &Operation{OperationType: assignAttributesOpType}")
 	}
 	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs}
 

pkg/yqlib/operator_multiply_test.go

@@ -24,13 +24,114 @@ list2:
     - "123"
 `
 
+var mergeArrayWithAnchors = `sample:
+- &a
+- <<: *a
+`
+
 var mergeArraysObjectKeysText = `It's a complex command, the trickyness comes from needing to have the right context in the expressions.
 First we save the second array into a variable '$two' which lets us reference it later.
 We then need to update the first array. We will use the relative update (|=) because we need to update relative to the current element of the array in the LHS in the RHS expression. 
 We set the current element of the first array as $cur. Now we multiply (merge) $cur with the matching entry in $two, by passing $two through a select filter.
 `
 
+var docWithHeader = `
+# here
+
+a: apple
+`
+
+var nodeWithHeader = `
+# here
+a: apple
+`
+
+var docNoComments = `
+b: banana
+`
+
+var docWithFooter = `
+a: apple
+
+# footer
+`
+
+var nodeWithFooter = `
+a: apple
+# footer`
+
 var multiplyOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   mergeArrayWithAnchors,
+		expression: `. * .`,
+		expected: []string{
+			"D0, P[], (!!map)::sample:\n    - &a\n    - !!merge <<: *a\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   docWithHeader,
+		document2:  docNoComments,
+		expression: `select(fi == 0) * select(fi == 1)`,
+		expected: []string{
+			"D0, P[], (!!map)::# here\na: apple\nb: banana\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   nodeWithHeader,
+		document2:  docNoComments,
+		expression: `select(fi == 0) * select(fi == 1)`,
+		expected: []string{
+			"D0, P[], (!!map)::# here\na: apple\nb: banana\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   docNoComments,
+		document2:  docWithHeader,
+		expression: `select(fi == 0) * select(fi == 1)`,
+		expected: []string{
+			"D0, P[], (!!map)::# here\nb: banana\na: apple\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   docNoComments,
+		document2:  nodeWithHeader,
+		expression: `select(fi == 0) * select(fi == 1)`,
+		expected: []string{
+			"D0, P[], (!!map)::b: banana\n# here\na: apple\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   docWithFooter,
+		document2:  docNoComments,
+		expression: `select(fi == 0) * select(fi == 1)`,
+		expected: []string{
+			"D0, P[], (!!map)::a: apple\nb: banana\n\n# footer\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   nodeWithFooter,
+		document2:  docNoComments,
+		expression: `select(fi == 0) * select(fi == 1)`,
+		expected: []string{ // not sure why there's an extra newline *shrug*
+			"D0, P[], (!!map)::a: apple\n# footer\n\nb: banana\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   docNoComments,
+		document2:  docWithFooter,
+		expression: `select(fi == 0) * select(fi == 1)`,
+		expected: []string{
+			"D0, P[], (!!map)::b: banana\na: apple\n\n# footer\n",
+		},
+	},
 	{
 		description: "Multiply integers",
 		expression:  `3 * 4`,
@@ -47,6 +148,20 @@ var multiplyOperatorScenarios = []expressionScenario{
 			docExpected,
 		},
 	},
+	{
+		skipDoc:    true,
+		expression: `.x =  {"things": "whatever"} * {}`,
+		expected: []string{
+			"D0, P[], ()::x:\n    things: whatever\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `.x = {} * {"things": "whatever"}`,
+		expected: []string{
+			"D0, P[], ()::x:\n    things: whatever\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		expression: `3 * 4.5`,

pkg/yqlib/operator_strings.go

@@ -74,6 +74,232 @@ func substituteStringOperator(d *dataTreeNavigator, context Context, expressionN
 
 }
 
+func addMatch(original []*yaml.Node, match string, offset int, name string) []*yaml.Node {
+
+	newContent := append(original,
+		createScalarNode("string", "string"))
+
+	if offset < 0 {
+		// offset of -1 means there was no match, force a null value like jq
+		newContent = append(newContent,
+			createScalarNode(nil, "null"),
+		)
+	} else {
+		newContent = append(newContent,
+			createScalarNode(match, match),
+		)
+	}
+
+	newContent = append(newContent,
+		createScalarNode("offset", "offset"),
+		createScalarNode(offset, fmt.Sprintf("%v", offset)),
+		createScalarNode("length", "length"),
+		createScalarNode(len(match), fmt.Sprintf("%v", len(match))))
+
+	if name != "" {
+		newContent = append(newContent,
+			createScalarNode("name", "name"),
+			createScalarNode(name, name),
+		)
+	}
+	return newContent
+}
+
+type matchPreferences struct {
+	Global bool
+}
+
+func getMatches(matchPrefs matchPreferences, regEx *regexp.Regexp, value string) ([][]string, [][]int) {
+	var allMatches [][]string
+	var allIndices [][]int
+
+	if matchPrefs.Global {
+		allMatches = regEx.FindAllStringSubmatch(value, -1)
+		allIndices = regEx.FindAllStringSubmatchIndex(value, -1)
+	} else {
+		allMatches = [][]string{regEx.FindStringSubmatch(value)}
+		allIndices = [][]int{regEx.FindStringSubmatchIndex(value)}
+	}
+
+	log.Debug("allMatches, %v", allMatches)
+	return allMatches, allIndices
+}
+
+func match(matchPrefs matchPreferences, regEx *regexp.Regexp, candidate *CandidateNode, value string, results *list.List) {
+	subNames := regEx.SubexpNames()
+	allMatches, allIndices := getMatches(matchPrefs, regEx, value)
+
+	// if all matches just has an empty array in it,
+	// then nothing matched
+	if len(allMatches) > 0 && len(allMatches[0]) == 0 {
+		return
+	}
+
+	for i, matches := range allMatches {
+		capturesNode := &yaml.Node{Kind: yaml.SequenceNode}
+		match, submatches := matches[0], matches[1:]
+		for j, submatch := range submatches {
+			captureNode := &yaml.Node{Kind: yaml.MappingNode}
+			captureNode.Content = addMatch(capturesNode.Content, submatch, allIndices[i][2+j*2], subNames[j+1])
+			capturesNode.Content = append(capturesNode.Content, captureNode)
+		}
+
+		node := &yaml.Node{Kind: yaml.MappingNode}
+		node.Content = addMatch(node.Content, match, allIndices[i][0], "")
+		node.Content = append(node.Content,
+			createScalarNode("captures", "captures"),
+			capturesNode,
+		)
+		results.PushBack(candidate.CreateChild(nil, node))
+
+	}
+
+}
+
+func capture(matchPrefs matchPreferences, regEx *regexp.Regexp, candidate *CandidateNode, value string, results *list.List) {
+	subNames := regEx.SubexpNames()
+	allMatches, allIndices := getMatches(matchPrefs, regEx, value)
+
+	// if all matches just has an empty array in it,
+	// then nothing matched
+	if len(allMatches) > 0 && len(allMatches[0]) == 0 {
+		return
+	}
+
+	for i, matches := range allMatches {
+		capturesNode := &yaml.Node{Kind: yaml.MappingNode}
+
+		_, submatches := matches[0], matches[1:]
+		for j, submatch := range submatches {
+			capturesNode.Content = append(capturesNode.Content,
+				createScalarNode(subNames[j+1], subNames[j+1]))
+
+			offset := allIndices[i][2+j*2]
+			// offset of -1 means there was no match, force a null value like jq
+			if offset < 0 {
+				capturesNode.Content = append(capturesNode.Content,
+					createScalarNode(nil, "null"),
+				)
+			} else {
+				capturesNode.Content = append(capturesNode.Content,
+					createScalarNode(submatch, submatch),
+				)
+			}
+		}
+
+		results.PushBack(candidate.CreateChild(nil, capturesNode))
+
+	}
+
+}
+
+func extractMatchArguments(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (*regexp.Regexp, matchPreferences, error) {
+	regExExpNode := expressionNode.Rhs
+
+	matchPrefs := matchPreferences{}
+
+	// we got given parameters e.g. match(exp; params)
+	if expressionNode.Rhs.Operation.OperationType == blockOpType {
+		block := expressionNode.Rhs
+		regExExpNode = block.Lhs
+		replacementNodes, err := d.GetMatchingNodes(context, block.Rhs)
+		if err != nil {
+			return nil, matchPrefs, err
+		}
+		paramText := ""
+		if replacementNodes.MatchingNodes.Front() != nil {
+			paramText = replacementNodes.MatchingNodes.Front().Value.(*CandidateNode).Node.Value
+		}
+		if strings.Contains(paramText, "g") {
+			paramText = strings.ReplaceAll(paramText, "g", "")
+			matchPrefs.Global = true
+		}
+		if strings.Contains(paramText, "i") {
+			return nil, matchPrefs, fmt.Errorf(`'i' is not a valid option for match. To ignore case, use an expression like match("(?i)cat")`)
+		}
+		if len(paramText) > 0 {
+			return nil, matchPrefs, fmt.Errorf(`Unrecognised match params '%v', please see docs at https://mikefarah.gitbook.io/yq/operators/string-operators`, paramText)
+		}
+	}
+
+	regExNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), regExExpNode)
+	if err != nil {
+		return nil, matchPrefs, err
+	}
+	log.Debug(NodesToString(regExNodes.MatchingNodes))
+	regExStr := ""
+	if regExNodes.MatchingNodes.Front() != nil {
+		regExStr = regExNodes.MatchingNodes.Front().Value.(*CandidateNode).Node.Value
+	}
+	log.Debug("regEx %v", regExStr)
+	regEx, err := regexp.Compile(regExStr)
+	return regEx, matchPrefs, err
+}
+
+func matchOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	regEx, matchPrefs, err := extractMatchArguments(d, context, expressionNode)
+	if err != nil {
+		return Context{}, err
+	}
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		node := unwrapDoc(candidate.Node)
+		if node.Tag != "!!str" {
+			return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
+		}
+
+		match(matchPrefs, regEx, candidate, node.Value, results)
+	}
+
+	return context.ChildContext(results), nil
+}
+
+func captureOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	regEx, matchPrefs, err := extractMatchArguments(d, context, expressionNode)
+	if err != nil {
+		return Context{}, err
+	}
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		node := unwrapDoc(candidate.Node)
+		if node.Tag != "!!str" {
+			return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
+		}
+		capture(matchPrefs, regEx, candidate, node.Value, results)
+
+	}
+
+	return context.ChildContext(results), nil
+}
+
+func testOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	regEx, _, err := extractMatchArguments(d, context, expressionNode)
+	if err != nil {
+		return Context{}, err
+	}
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		node := unwrapDoc(candidate.Node)
+		if node.Tag != "!!str" {
+			return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
+		}
+		matches := regEx.FindStringSubmatch(node.Value)
+		results.PushBack(createBooleanCandidate(candidate, len(matches) > 0))
+
+	}
+
+	return context.ChildContext(results), nil
+}
+
 func joinStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- joinStringOperator")
 	joinStr := ""

pkg/yqlib/operator_strings_test.go

@@ -13,6 +13,110 @@ var stringsOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!str)::cat; meow; 1; ; true\n",
 		},
 	},
+	{
+		description: "Match string",
+		document:    `foo bar foo`,
+		expression:  `match("foo")`,
+		expected: []string{
+			"D0, P[], ()::string: foo\noffset: 0\nlength: 3\ncaptures: []\n",
+		},
+	},
+	{
+		description: "Match string, case insensitive",
+		document:    `foo bar FOO`,
+		expression:  `[match("(?i)foo"; "g")]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- string: foo\n  offset: 0\n  length: 3\n  captures: []\n- string: FOO\n  offset: 8\n  length: 3\n  captures: []\n",
+		},
+	},
+	{
+		description: "Match with capture groups",
+		document:    `abc abc`,
+		expression:  `[match("(abc)+"; "g")]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- string: abc\n  offset: 0\n  length: 3\n  captures:\n    - string: abc\n      offset: 0\n      length: 3\n- string: abc\n  offset: 4\n  length: 3\n  captures:\n    - string: abc\n      offset: 4\n      length: 3\n",
+		},
+	},
+	{
+		description: "Match with named capture groups",
+		document:    `foo bar foo foo  foo`,
+		expression:  `[match("foo (?P<bar123>bar)? foo"; "g")]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- string: foo bar foo\n  offset: 0\n  length: 11\n  captures:\n    - string: bar\n      offset: 4\n      length: 3\n      name: bar123\n- string: foo  foo\n  offset: 12\n  length: 8\n  captures:\n    - string: null\n      offset: -1\n      length: 0\n      name: bar123\n",
+		},
+	},
+	{
+		description: "Capture named groups into a map",
+		document:    `xyzzy-14`,
+		expression:  `capture("(?P<a>[a-z]+)-(?P<n>[0-9]+)")`,
+		expected: []string{
+			"D0, P[], ()::a: xyzzy\nn: \"14\"\n",
+		},
+	},
+	{
+		skipDoc:     true,
+		description: "Capture named groups into a map, with null",
+		document:    `xyzzy-14`,
+		expression:  `capture("(?P<a>[a-z]+)-(?P<n>[0-9]+)(?P<bar123>bar)?")`,
+		expected: []string{
+			"D0, P[], ()::a: xyzzy\nn: \"14\"\nbar123: null\n",
+		},
+	},
+	{
+		description: "Match without global flag",
+		document:    `cat cat`,
+		expression:  `match("cat")`,
+		expected: []string{
+			"D0, P[], ()::string: cat\noffset: 0\nlength: 3\ncaptures: []\n",
+		},
+	},
+	{
+		description: "Match with global flag",
+		document:    `cat cat`,
+		expression:  `[match("cat"; "g")]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- string: cat\n  offset: 0\n  length: 3\n  captures: []\n- string: cat\n  offset: 4\n  length: 3\n  captures: []\n",
+		},
+	},
+	{
+		skipDoc:     true,
+		description: "No match",
+		document:    `dog`,
+		expression:  `match("cat"; "g")`,
+		expected:    []string{},
+	},
+	{
+		skipDoc:     true,
+		description: "No match",
+		expression:  `"dog" | match("cat", "g")`,
+		expected:    []string{},
+	},
+	{
+		skipDoc:     true,
+		description: "No match",
+		expression:  `"dog" | match("cat")`,
+		expected:    []string{},
+	},
+	{
+		description:    "Test using regex",
+		subdescription: "Like jq'q equivalant, this works like match but only returns true/false instead of full match details",
+		document:       `["cat", "dog"]`,
+		expression:     `.[] | test("at")`,
+		expected: []string{
+			"D0, P[0], (!!bool)::true\n",
+			"D0, P[1], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `["cat*", "cat*", "cat"]`,
+		expression: `.[] | test("cat\*")`,
+		expected: []string{
+			"D0, P[0], (!!bool)::true\n",
+			"D0, P[1], (!!bool)::true\n",
+			"D0, P[2], (!!bool)::false\n",
+		},
+	},
 	{
 		description:    "Substitute / Replace string",
 		subdescription: "This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)\nNote the use of `|=` to run in context of the current string value.",

pkg/yqlib/operator_style_test.go

@@ -14,10 +14,9 @@ var styleOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description:    "Update and set style of a particular node using path variables",
-		subdescription: "You can use a variable to re-use a path",
-		document:       `a: {b: thing, c: something}`,
-		expression:     `.a.b as $x | $x = "new" | $x style="double"`,
+		description: "Update and set style of a particular node using path variables",
+		document:    `a: {b: thing, c: something}`,
+		expression:  `with(.a.b ; . = "new" | . style="double")`,
 		expected: []string{
 			"D0, P[], (doc)::a: {b: \"new\", c: something}\n",
 		},

pkg/yqlib/operator_subtract.go

@@ -5,7 +5,7 @@ import (
 
 	"strconv"
 
-	yaml "gopkg.in/yaml.v3"
+	"gopkg.in/yaml.v3"
 )
 
 func createSubtractOp(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode {
@@ -15,11 +15,7 @@ func createSubtractOp(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode
 }
 
 func subtractAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	assignmentOp := &Operation{OperationType: assignOpType}
-	assignmentOp.UpdateAssign = true
-	selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
-	assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: expressionNode.Lhs, Rhs: createSubtractOp(selfExpression, expressionNode.Rhs)}
-	return d.GetMatchingNodes(context, assignmentOpNode)
+	return compoundAssignFunction(d, context, expressionNode, createSubtractOp)
 }
 
 func subtractOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
@@ -28,6 +24,24 @@ func subtractOperator(d *dataTreeNavigator, context Context, expressionNode *Exp
 	return crossFunction(d, context.ReadOnlyClone(), expressionNode, subtract, false)
 }
 
+func subtractArray(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	newLhsArray := make([]*yaml.Node, 0)
+
+	for lindex := 0; lindex < len(lhs.Node.Content); lindex = lindex + 1 {
+		shouldInclude := true
+		for rindex := 0; rindex < len(rhs.Node.Content) && shouldInclude; rindex = rindex + 1 {
+			if recursiveNodeEqual(lhs.Node.Content[lindex], rhs.Node.Content[rindex]) {
+				shouldInclude = false
+			}
+		}
+		if shouldInclude {
+			newLhsArray = append(newLhsArray, lhs.Node.Content[lindex])
+		}
+	}
+	lhs.Node.Content = newLhsArray
+	return lhs, nil
+}
+
 func subtract(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	lhs.Node = unwrapDoc(lhs.Node)
 	rhs.Node = unwrapDoc(rhs.Node)
@@ -44,14 +58,13 @@ func subtract(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Ca
 	case yaml.MappingNode:
 		return nil, fmt.Errorf("Maps not yet supported for subtraction")
 	case yaml.SequenceNode:
-		return nil, fmt.Errorf("Sequences not yet supported for subtraction")
-		// target.Node.Kind = yaml.SequenceNode
-		// target.Node.Style = lhsNode.Style
-		// target.Node.Tag = "!!seq"
-		// target.Node.Content = append(lhsNode.Content, toNodes(rhs)...)
+		if rhs.Node.Kind != yaml.SequenceNode {
+			return nil, fmt.Errorf("%v (%v) cannot be subtracted from %v", rhs.Node.Tag, rhs.Path, lhsNode.Tag)
+		}
+		return subtractArray(d, context, lhs, rhs)
 	case yaml.ScalarNode:
 		if rhs.Node.Kind != yaml.ScalarNode {
-			return nil, fmt.Errorf("%v (%v) cannot be added to a %v", rhs.Node.Tag, rhs.Path, lhsNode.Tag)
+			return nil, fmt.Errorf("%v (%v) cannot be subtracted from %v", rhs.Node.Tag, rhs.Path, lhsNode.Tag)
 		}
 		target.Node.Kind = yaml.ScalarNode
 		target.Node.Style = lhsNode.Style
@@ -66,17 +79,17 @@ func subtractScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*Ca
 	if lhs.Tag == "!!str" {
 		return nil, fmt.Errorf("strings cannot be subtracted")
 	} else if lhs.Tag == "!!int" && rhs.Tag == "!!int" {
-		lhsNum, err := strconv.Atoi(lhs.Value)
+		format, lhsNum, err := parseInt(lhs.Value)
 		if err != nil {
 			return nil, err
 		}
-		rhsNum, err := strconv.Atoi(rhs.Value)
+		_, rhsNum, err := parseInt(rhs.Value)
 		if err != nil {
 			return nil, err
 		}
 		result := lhsNum - rhsNum
 		target.Node.Tag = "!!int"
-		target.Node.Value = fmt.Sprintf("%v", result)
+		target.Node.Value = fmt.Sprintf(format, result)
 	} else if (lhs.Tag == "!!int" || lhs.Tag == "!!float") && (rhs.Tag == "!!int" || rhs.Tag == "!!float") {
 		lhsNum, err := strconv.ParseFloat(lhs.Value, 64)
 		if err != nil {

pkg/yqlib/operator_subtract_test.go

@@ -13,6 +13,51 @@ var subtractOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{}\n",
 		},
 	},
+	{
+		description: "Array subtraction",
+		expression:  `[1,2] - [2,3]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- 1\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `[2,1,2,2] - [2,3]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- 1\n",
+		},
+	},
+	{
+		description: "Array subtraction with nested array",
+		expression:  `[[1], 1, 2] - [[1], 3]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- 1\n- 2\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `[[1], 1, [[[2]]]] - [[1], [[[3]]]]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- 1\n- - - - 2\n",
+		},
+	},
+	{
+		description:    "Array subtraction with nested object",
+		subdescription: `Note that order of the keys does not matter`,
+		document:       `[{a: b, c: d}, {a: b}]`,
+		expression:     `. - [{"c": "d", "a": "b"}]`,
+		expected: []string{
+			"D0, P[], (!!seq)::[{a: b}]\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{a: [1], c: d}, {a: [2], c: d}, {a: b}]`,
+		expression: `. - [{"c": "d", "a": [1]}]`,
+		expected: []string{
+			"D0, P[], (!!seq)::[{a: [2], c: d}, {a: b}]\n",
+		},
+	},
 	{
 		description:    "Number subtraction - float",
 		subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.",

pkg/yqlib/operator_traverse_path.go

@@ -127,9 +127,16 @@ func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesT
 	node := matchingNode.Node
 	if node.Tag == "!!null" {
 		log.Debugf("OperatorArrayTraverse got a null - turning it into an empty array")
-		// auto vivification, make it into an empty array
+		// auto vivification
 		node.Tag = ""
 		node.Kind = yaml.SequenceNode
+		//check that the indices are numeric, if not, then we should create an object
+		if len(indicesToTraverse) != 0 {
+			_, err := strconv.ParseInt(indicesToTraverse[0].Value, 10, 64)
+			if err != nil {
+				node.Kind = yaml.MappingNode
+			}
+		}
 	}
 
 	if node.Kind == yaml.AliasNode {
@@ -308,6 +315,6 @@ func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *C
 
 func traverseArray(candidate *CandidateNode, operation *Operation, prefs traversePreferences) (*list.List, error) {
 	log.Debug("operation Value %v", operation.Value)
-	indices := []*yaml.Node{&yaml.Node{Value: operation.StringValue}}
+	indices := []*yaml.Node{{Value: operation.StringValue}}
 	return traverseArrayWithIndices(candidate, indices, prefs)
 }

pkg/yqlib/operator_traverse_path_test.go

@@ -35,6 +35,14 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[0 0], (!!int)::1\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `b: cat`,
+		expression: ".b\n",
+		expected: []string{
+			"D0, P[b], (!!str)::cat\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `[[[1]]]`,
@@ -43,6 +51,13 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[0 0 0], (!!int)::1\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		expression: `.["cat"] = "thing"`,
+		expected: []string{
+			"D0, P[], ()::cat: thing\n",
+		},
+	},
 	{
 		description: "Simple map navigation",
 		document:    `{a: {b: apple}}`,
@@ -77,6 +92,14 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[{}], (!!str)::frog\n",
 		},
 	},
+	{
+		description: "Nested special characters",
+		document:    `a: {"key.withdots": {"another.key": apple}}`,
+		expression:  `.a["key.withdots"]["another.key"]`,
+		expected: []string{
+			"D0, P[a key.withdots another.key], (!!str)::apple\n",
+		},
+	},
 	{
 		description:    "Keys with spaces",
 		subdescription: "Use quotes with brackets around path elements with special characters",
@@ -244,6 +267,15 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[0], (!!int)::1\n",
 		},
 	},
+	{
+		description:           "Traversing nested arrays by index",
+		dontFormatInputForDoc: true,
+		document:              `[[], [cat]]`,
+		expression:            `.[1][0]`,
+		expected: []string{
+			"D0, P[1 0], (!!str)::cat\n",
+		},
+	},
 	{
 		description: "Maps with numeric keys",
 		document:    `{2: cat}`,

pkg/yqlib/operator_union.go

@@ -1,17 +1,36 @@
 package yqlib
 
 func unionOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	log.Debug("unionOperator")
+	log.Debug("context: %v", NodesToString(context.MatchingNodes))
 	lhs, err := d.GetMatchingNodes(context, expressionNode.Lhs)
 	if err != nil {
 		return Context{}, err
 	}
+	log.Debug("lhs: %v", NodesToString(lhs.MatchingNodes))
+	log.Debug("rhs input: %v", NodesToString(context.MatchingNodes))
+	log.Debug("rhs: %v", expressionNode.Rhs.Operation.toString())
 	rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+
 	if err != nil {
 		return Context{}, err
 	}
-	for el := rhs.MatchingNodes.Front(); el != nil; el = el.Next() {
-		node := el.Value.(*CandidateNode)
-		lhs.MatchingNodes.PushBack(node)
+	log.Debug("lhs: %v", lhs.ToString())
+	log.Debug("rhs: %v", rhs.ToString())
+
+	// this can happen when both expressions modify the context
+	// instead of creating their own.
+	/// (.foo = "bar"), (.thing = "cat")
+
+	if rhs.MatchingNodes != lhs.MatchingNodes {
+
+		for el := rhs.MatchingNodes.Front(); el != nil; el = el.Next() {
+			node := el.Value.(*CandidateNode)
+			log.Debug("processing %v", NodeToString(node))
+
+			lhs.MatchingNodes.PushBack(node)
+		}
 	}
+	log.Debug("all together: %v", lhs.ToString())
 	return lhs, nil
 }

pkg/yqlib/operator_union_test.go

@@ -5,20 +5,21 @@ import (
 )
 
 var unionOperatorScenarios = []expressionScenario{
-	// {
-	// 	skipDoc:    true,
-	// 	document:   "{}",
-	// 	expression: `(.a, .b.c) as $x`,
-	// 	expected: []string{
-	// 		"D0, P[], (doc)::{}\n",
-	// 	},
-	// },
-	// {
-	// 	skipDoc:    true,
-	// 	document:   "{}",
-	// 	expression: `(.a, .b.c)`,
-	// 	expected:   []string{},
-	// },
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: `(.a, .b.c) as $x`,
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `(.foo = "bar"), (.toe = "jam")`,
+		expected: []string{
+			"D0, P[], ()::foo: bar\ntoe: jam\n",
+		},
+	},
 	{
 		description: "Combine scalars",
 		expression:  `1, true, "cat"`,

pkg/yqlib/operator_value_test.go

@@ -12,6 +12,76 @@ var valueOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!int)::1\n",
 		},
 	},
+	{
+		document:   ``,
+		expression: `0x9f`,
+		expected: []string{
+			"D0, P[], (!!int)::0x9f\n",
+		},
+	},
+	{
+		document:   ``,
+		expression: `0x1A`,
+		expected: []string{
+			"D0, P[], (!!int)::0x1A\n",
+		},
+	},
+	{
+		document:   ``,
+		expression: `0x1A + 2`,
+		expected: []string{
+			"D0, P[], (!!int)::0x1C\n",
+		},
+	},
+	{
+		document:   ``,
+		expression: `0x12 * 2`,
+		expected: []string{
+			"D0, P[], (!!int)::0x24\n",
+		},
+	},
+	{
+		document:   ``,
+		expression: `0xF - 1`,
+		expected: []string{
+			"D0, P[], (!!int)::0xE\n",
+		},
+	},
+	{
+		document:   ``,
+		expression: `12`,
+		expected: []string{
+			"D0, P[], (!!int)::12\n",
+		},
+	},
+	{
+		document:   ``,
+		expression: `12 + 2`,
+		expected: []string{
+			"D0, P[], (!!int)::14\n",
+		},
+	},
+	{
+		document:   ``,
+		expression: `12 * 2`,
+		expected: []string{
+			"D0, P[], (!!int)::24\n",
+		},
+	},
+	{
+		document:   ``,
+		expression: `12 - 2`,
+		expected: []string{
+			"D0, P[], (!!int)::10\n",
+		},
+	},
+	{
+		document:   ``,
+		expression: `0X12`,
+		expected: []string{
+			"D0, P[], (!!int)::0X12\n",
+		},
+	},
 	{
 		document:   ``,
 		expression: `-1`,

pkg/yqlib/operator_variables.go

@@ -15,6 +15,10 @@ func getVariableOperator(d *dataTreeNavigator, context Context, expressionNode *
 	return context.ChildContext(result), nil
 }
 
+type assignVarPreferences struct {
+	IsReference bool
+}
+
 func assignVariableOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	lhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Lhs)
 	if err != nil {
@@ -24,6 +28,15 @@ func assignVariableOperator(d *dataTreeNavigator, context Context, expressionNod
 		return Context{}, fmt.Errorf("RHS of 'as' operator must be a variable name e.g. $foo")
 	}
 	variableName := expressionNode.Rhs.Operation.StringValue
-	context.SetVariable(variableName, lhs.MatchingNodes)
+
+	prefs := expressionNode.Operation.Preferences.(assignVarPreferences)
+
+	var variableValue *list.List
+	if prefs.IsReference {
+		variableValue = lhs.MatchingNodes
+	} else {
+		variableValue = lhs.DeepClone().MatchingNodes
+	}
+	context.SetVariable(variableName, variableValue)
 	return context, nil
 }

pkg/yqlib/operator_variables_test.go

@@ -34,15 +34,32 @@ var variableOperatorScenarios = []expressionScenario{
 		description:    "Using variables as a lookup",
 		subdescription: "Example taken from [jq](https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|...)",
 		document: `{"posts": [{"title": "Frist psot", "author": "anon"},
-		{"title": "A well-written article", "author": "person1"}],
-"realnames": {"anon": "Anonymous Coward",
-				"person1": "Person McPherson"}}`,
+			{"title": "A well-written article", "author": "person1"}],
+	"realnames": {"anon": "Anonymous Coward",
+					"person1": "Person McPherson"}}`,
 		expression: `.realnames as $names | .posts[] | {"title":.title, "author": $names[.author]}`,
 		expected: []string{
 			"D0, P[], (!!map)::title: \"Frist psot\"\nauthor: \"Anonymous Coward\"\n",
 			"D0, P[], (!!map)::title: \"A well-written article\"\nauthor: \"Person McPherson\"\n",
 		},
 	},
+	{
+		description: "Using variables to swap values",
+		document:    "a: a_value\nb: b_value",
+		expression:  `.a as $x  | .b as $y | .b = $x | .a = $y`,
+		expected: []string{
+			"D0, P[], (doc)::a: b_value\nb: a_value\n",
+		},
+	},
+	{
+		description:    "Use ref to reference a path repeatedly",
+		subdescription: "Note: You may find the `with` operator more useful.",
+		document:       `a: {b: thing, c: something}`,
+		expression:     `.a.b ref $x | $x = "new" | $x style="double"`,
+		expected: []string{
+			"D0, P[], (doc)::a: {b: \"new\", c: something}\n",
+		},
+	},
 }
 
 func TestVariableOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_with.go

@@ -0,0 +1,30 @@
+package yqlib
+
+import "fmt"
+
+func withOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	log.Debugf("-- withOperator")
+	// with(path, exp)
+
+	if expressionNode.Rhs.Operation.OperationType != blockOpType {
+		return Context{}, fmt.Errorf("with must be given a block, got %v instead", expressionNode.Rhs.Operation.OperationType.Type)
+	}
+
+	pathExp := expressionNode.Rhs.Lhs
+
+	updateContext, err := d.GetMatchingNodes(context, pathExp)
+
+	if err != nil {
+		return Context{}, err
+	}
+
+	updateExp := expressionNode.Rhs.Rhs
+
+	_, err = d.GetMatchingNodes(updateContext, updateExp)
+	if err != nil {
+		return Context{}, err
+	}
+
+	return context, nil
+
+}

pkg/yqlib/operator_with_test.go

@@ -0,0 +1,38 @@
+package yqlib
+
+import "testing"
+
+var withOperatorScenarios = []expressionScenario{
+	{
+		description: "Update and style",
+		document:    `a: {deeply: {nested: value}}`,
+		expression:  `with(.a.deeply.nested; . = "newValue" | . style="single")`,
+		expected: []string{
+			"D0, P[], (doc)::a: {deeply: {nested: 'newValue'}}\n",
+		},
+	},
+	{
+		description: "Update multiple deeply nested properties",
+		document:    `a: {deeply: {nested: value, other: thing}}`,
+		expression:  `with(.a.deeply; .nested = "newValue" | .other= "newThing")`,
+		expected: []string{
+			"D0, P[], (doc)::a: {deeply: {nested: newValue, other: newThing}}\n",
+		},
+	},
+	{
+		description:    "Update array elements relatively",
+		subdescription: "The second expression runs with each element of the array as it's contextual root. This allows you to make updates relative to the element.",
+		document:       `myArray: [{a: apple},{a: banana}]`,
+		expression:     `with(.myArray[]; .b = .a + " yum")`,
+		expected: []string{
+			"D0, P[], (doc)::myArray: [{a: apple, b: apple yum}, {a: banana, b: banana yum}]\n",
+		},
+	},
+}
+
+func TestWithOperatorScenarios(t *testing.T) {
+	for _, tt := range withOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "With", withOperatorScenarios)
+}

pkg/yqlib/operators.go

@@ -10,6 +10,32 @@ import (
 
 type operatorHandler func(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error)
 
+type compoundCalculation func(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode
+
+func compoundAssignFunction(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode, calculation compoundCalculation) (Context, error) {
+	lhs, err := d.GetMatchingNodes(context, expressionNode.Lhs)
+	if err != nil {
+		return Context{}, err
+	}
+
+	assignmentOp := &Operation{OperationType: assignOpType}
+	valueOp := &Operation{OperationType: valueOpType}
+
+	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		valueOp.CandidateNode = candidate
+		valueExpression := &ExpressionNode{Operation: valueOp}
+
+		assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: valueExpression, Rhs: calculation(valueExpression, expressionNode.Rhs)}
+
+		_, err = d.GetMatchingNodes(context, assignmentOpNode)
+		if err != nil {
+			return Context{}, err
+		}
+	}
+	return context, nil
+}
+
 func unwrapDoc(node *yaml.Node) *yaml.Node {
 	if node.Kind == yaml.DocumentNode {
 		return node.Content[0]
@@ -31,7 +57,9 @@ func resultsForRhs(d *dataTreeNavigator, context Context, lhsCandidate *Candidat
 		if err != nil {
 			return err
 		}
-		results.PushBack(resultCandidate)
+		if resultCandidate != nil {
+			results.PushBack(resultCandidate)
+		}
 		return nil
 	}
 
@@ -42,7 +70,9 @@ func resultsForRhs(d *dataTreeNavigator, context Context, lhsCandidate *Candidat
 		if err != nil {
 			return err
 		}
-		results.PushBack(resultCandidate)
+		if resultCandidate != nil {
+			results.PushBack(resultCandidate)
+		}
 	}
 	return nil
 }
@@ -91,9 +121,12 @@ func crossFunction(d *dataTreeNavigator, context Context, expressionNode *Expres
 		}
 	}
 	if evaluateAllTogether {
+		log.Debug("crossFunction evaluateAllTogether!")
 		return doCrossFunc(d, context, expressionNode, calculation, calcWhenEmpty)
 	}
 
+	log.Debug("crossFunction evaluate apart!")
+
 	for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {
 		innerResults, err := doCrossFunc(d, context.SingleChildContext(matchEl.Value.(*CandidateNode)), expressionNode, calculation, calcWhenEmpty)
 		if err != nil {

pkg/yqlib/operators_test.go

@@ -107,14 +107,14 @@ func copyFromHeader(title string, out *os.File) error {
 
 func formatYaml(yaml string, filename string) string {
 	var output bytes.Buffer
-	printer := NewPrinter(bufio.NewWriter(&output), false, true, false, 2, true)
+	printer := NewPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(".. style= \"\"")
 	if err != nil {
 		panic(err)
 	}
 	streamEvaluator := NewStreamEvaluator()
-	err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer)
+	_, err = streamEvaluator.Evaluate(filename, strings.NewReader(yaml), node, printer, "")
 	if err != nil {
 		panic(err)
 	}
@@ -216,7 +216,7 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
 func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formattedDoc string, formattedDoc2 string) {
 	var output bytes.Buffer
 	var err error
-	printer := NewPrinter(bufio.NewWriter(&output), false, true, false, 2, true)
+	printer := NewPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(s.expression)
 	if err != nil {

pkg/yqlib/printer.go

@@ -3,7 +3,9 @@ package yqlib
 import (
 	"bufio"
 	"container/list"
+	"fmt"
 	"io"
+	"strings"
 
 	yaml "gopkg.in/yaml.v3"
 )
@@ -11,10 +13,33 @@ import (
 type Printer interface {
 	PrintResults(matchingNodes *list.List) error
 	PrintedAnything() bool
+	//e.g. when given a front-matter doc, like jekyll
+	SetAppendix(reader io.Reader)
+}
+
+type PrinterOutputFormat uint32
+
+const (
+	YamlOutputFormat = 1 << iota
+	JsonOutputFormat
+	PropsOutputFormat
+)
+
+func OutputFormatFromString(format string) (PrinterOutputFormat, error) {
+	switch format {
+	case "yaml", "y":
+		return YamlOutputFormat, nil
+	case "json", "j":
+		return JsonOutputFormat, nil
+	case "props", "p":
+		return PropsOutputFormat, nil
+	default:
+		return 0, fmt.Errorf("Unknown fromat '%v' please use [yaml|json|props]", format)
+	}
 }
 
 type resultsPrinter struct {
-	outputToJSON       bool
+	outputFormat       PrinterOutputFormat
 	unwrapScalar       bool
 	colorsEnabled      bool
 	indent             int
@@ -25,21 +50,26 @@ type resultsPrinter struct {
 	previousFileIndex  int
 	printedMatches     bool
 	treeNavigator      DataTreeNavigator
+	appendixReader     io.Reader
 }
 
-func NewPrinter(writer io.Writer, outputToJSON bool, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
+func NewPrinter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
 	return &resultsPrinter{
 		writer:             writer,
-		outputToJSON:       outputToJSON,
+		outputFormat:       outputFormat,
 		unwrapScalar:       unwrapScalar,
 		colorsEnabled:      colorsEnabled,
 		indent:             indent,
-		printDocSeparators: !outputToJSON && printDocSeparators,
+		printDocSeparators: outputFormat == YamlOutputFormat && printDocSeparators,
 		firstTimePrinting:  true,
 		treeNavigator:      NewDataTreeNavigator(),
 	}
 }
 
+func (p *resultsPrinter) SetAppendix(reader io.Reader) {
+	p.appendixReader = reader
+}
+
 func (p *resultsPrinter) PrintedAnything() bool {
 	return p.printedMatches
 }
@@ -49,11 +79,14 @@ func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error {
 		(node.Tag != "!!bool" || node.Value != "false"))
 
 	var encoder Encoder
-	if node.Kind == yaml.ScalarNode && p.unwrapScalar && !p.outputToJSON {
+	if node.Kind == yaml.ScalarNode && p.unwrapScalar && p.outputFormat == YamlOutputFormat {
 		return p.writeString(writer, node.Value+"\n")
 	}
-	if p.outputToJSON {
+
+	if p.outputFormat == JsonOutputFormat {
 		encoder = NewJsonEncoder(writer, p.indent)
+	} else if p.outputFormat == PropsOutputFormat {
+		encoder = NewPropertiesEncoder(writer)
 	} else {
 		encoder = NewYamlEncoder(writer, p.indent, p.colorsEnabled)
 	}
@@ -74,7 +107,7 @@ func (p *resultsPrinter) safelyFlush(writer *bufio.Writer) {
 
 func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 	log.Debug("PrintResults for %v matches", matchingNodes.Len())
-	if p.outputToJSON {
+	if p.outputFormat != YamlOutputFormat {
 		explodeOp := Operation{OperationType: explodeOpType}
 		explodeNode := ExpressionNode{Operation: &explodeOp}
 		context, err := p.treeNavigator.GetMatchingNodes(Context{MatchingNodes: matchingNodes}, &explodeNode)
@@ -101,13 +134,56 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 	for el := matchingNodes.Front(); el != nil; el = el.Next() {
 		mappedDoc := el.Value.(*CandidateNode)
 		log.Debug("-- print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v, mappedDoc.Document: %v, printDocSeparators: %v", p.firstTimePrinting, p.previousDocIndex, mappedDoc.Document, p.printDocSeparators)
-		if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators {
+
+		commentStartsWithSeparator := strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$\n$yqDocSeperator$")
+
+		if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && p.printDocSeparators && !commentStartsWithSeparator {
 			log.Debug("-- writing doc sep")
 			if err := p.writeString(bufferedWriter, "---\n"); err != nil {
 				return err
 			}
 		}
 
+		if strings.Contains(mappedDoc.Node.HeadComment, "$yqLeadingContent$") {
+			log.Debug("headcommentwas %v", mappedDoc.Node.HeadComment)
+			log.Debug("finished headcomment")
+			reader := bufio.NewReader(strings.NewReader(mappedDoc.Node.HeadComment))
+			mappedDoc.Node.HeadComment = ""
+
+			for {
+
+				readline, errReading := reader.ReadString('\n')
+				if errReading != nil && errReading != io.EOF {
+					return errReading
+				}
+				if strings.Contains(readline, "$yqLeadingContent$") {
+					// skip this
+
+				} else if strings.Contains(readline, "$yqDocSeperator$") {
+					if p.printDocSeparators {
+						if err := p.writeString(bufferedWriter, "---\n"); err != nil {
+							return err
+						}
+					}
+				} else if p.outputFormat == YamlOutputFormat {
+					if err := p.writeString(bufferedWriter, readline); err != nil {
+						return err
+					}
+				}
+
+				if errReading == io.EOF {
+					if readline != "" {
+						// the last comment we read didn't have a new line, put one in
+						if err := p.writeString(bufferedWriter, "\n"); err != nil {
+							return err
+						}
+					}
+					break
+				}
+			}
+
+		}
+
 		if err := p.printNode(mappedDoc.Node, bufferedWriter); err != nil {
 			return err
 		}
@@ -115,5 +191,14 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 		p.previousDocIndex = mappedDoc.Document
 	}
 
+	if p.appendixReader != nil && p.outputFormat == YamlOutputFormat {
+		log.Debug("Piping appendix reader...")
+		betterReader := bufio.NewReader(p.appendixReader)
+		_, err := io.Copy(bufferedWriter, betterReader)
+		if err != nil {
+			return err
+		}
+	}
+
 	return nil
 }

pkg/yqlib/printer_test.go

@@ -17,6 +17,16 @@ a: apple
 a: coconut
 `
 
+var multiDocSampleLeadingExpected = `# go cats
+---
+a: banana
+---
+a: apple
+---
+# cool
+a: coconut
+`
+
 func nodeToList(candidate *CandidateNode) *list.List {
 	elMap := list.New()
 	elMap.PushBack(candidate)
@@ -26,7 +36,7 @@ func nodeToList(candidate *CandidateNode) *list.List {
 func TestPrinterMultipleDocsInSequence(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, false, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -61,10 +71,52 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) {
 	test.AssertResult(t, multiDocSample, output.String())
 }
 
+func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
+	var output bytes.Buffer
+	var writer = bufio.NewWriter(&output)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
+	if err != nil {
+		panic(err)
+	}
+
+	el := inputs.Front()
+	el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
+	sample1 := nodeToList(el.Value.(*CandidateNode))
+
+	el = el.Next()
+	el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
+	sample2 := nodeToList(el.Value.(*CandidateNode))
+
+	el = el.Next()
+	el.Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n# cool\n"
+	sample3 := nodeToList(el.Value.(*CandidateNode))
+
+	err = printer.PrintResults(sample1)
+	if err != nil {
+		panic(err)
+	}
+
+	err = printer.PrintResults(sample2)
+	if err != nil {
+		panic(err)
+	}
+
+	err = printer.PrintResults(sample3)
+	if err != nil {
+		panic(err)
+	}
+
+	writer.Flush()
+
+	test.AssertResult(t, multiDocSampleLeadingExpected, output.String())
+}
+
 func TestPrinterMultipleFilesInSequence(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, false, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -108,10 +160,60 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) {
 	test.AssertResult(t, multiDocSample, output.String())
 }
 
+func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
+	var output bytes.Buffer
+	var writer = bufio.NewWriter(&output)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
+	if err != nil {
+		panic(err)
+	}
+
+	el := inputs.Front()
+	elNode := el.Value.(*CandidateNode)
+	elNode.Document = 0
+	elNode.FileIndex = 0
+	elNode.Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
+	sample1 := nodeToList(elNode)
+
+	el = el.Next()
+	elNode = el.Value.(*CandidateNode)
+	elNode.Document = 0
+	elNode.FileIndex = 1
+	elNode.Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
+	sample2 := nodeToList(elNode)
+
+	el = el.Next()
+	elNode = el.Value.(*CandidateNode)
+	elNode.Document = 0
+	elNode.FileIndex = 2
+	elNode.Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n# cool\n"
+	sample3 := nodeToList(elNode)
+
+	err = printer.PrintResults(sample1)
+	if err != nil {
+		panic(err)
+	}
+
+	err = printer.PrintResults(sample2)
+	if err != nil {
+		panic(err)
+	}
+
+	err = printer.PrintResults(sample3)
+	if err != nil {
+		panic(err)
+	}
+
+	writer.Flush()
+	test.AssertResult(t, multiDocSampleLeadingExpected, output.String())
+}
+
 func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, false, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -127,18 +229,100 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 	test.AssertResult(t, multiDocSample, output.String())
 }
 
-func TestPrinterMultipleDocsJson(t *testing.T) {
+func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	// note printDocSeparators is true, it should still not print document separators
-	// when outputing JSON.
-	printer := NewPrinter(writer, true, true, false, 0, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
 		panic(err)
 	}
 
+	inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# go cats\n$yqDocSeperator$\n"
+
+	err = printer.PrintResults(inputs)
+	if err != nil {
+		panic(err)
+	}
+
+	writer.Flush()
+	expected := `# go cats
+---
+a: banana
+---
+a: apple
+---
+a: coconut
+`
+	test.AssertResult(t, expected, output.String())
+}
+
+func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
+	var output bytes.Buffer
+	var writer = bufio.NewWriter(&output)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
+	if err != nil {
+		panic(err)
+	}
+	inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n$yqDocSeperator$\n"
+	err = printer.PrintResults(inputs)
+	if err != nil {
+		panic(err)
+	}
+
+	writer.Flush()
+	expected := `---
+a: banana
+---
+a: apple
+---
+a: coconut
+`
+	test.AssertResult(t, expected, output.String())
+}
+
+func TestPrinterScalarWithLeadingCont(t *testing.T) {
+	var output bytes.Buffer
+	var writer = bufio.NewWriter(&output)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
+
+	node, err := NewExpressionParser().ParseExpression(".a")
+	if err != nil {
+		panic(err)
+	}
+	streamEvaluator := NewStreamEvaluator()
+	_, err = streamEvaluator.Evaluate("sample", strings.NewReader(multiDocSample), node, printer, "# blah\n")
+	if err != nil {
+		panic(err)
+	}
+
+	writer.Flush()
+	expected := `banana
+---
+apple
+---
+coconut
+`
+	test.AssertResult(t, expected, output.String())
+}
+
+func TestPrinterMultipleDocsJson(t *testing.T) {
+	var output bytes.Buffer
+	var writer = bufio.NewWriter(&output)
+	// note printDocSeparators is true, it should still not print document separators
+	// when outputing JSON.
+	printer := NewPrinter(writer, JsonOutputFormat, true, false, 0, true)
+
+	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
+	if err != nil {
+		panic(err)
+	}
+
+	inputs.Front().Value.(*CandidateNode).Node.HeadComment = "$yqLeadingContent$\n# ignore this\n"
+
 	err = printer.PrintResults(inputs)
 	if err != nil {
 		panic(err)

pkg/yqlib/stream_evaluator.go

@@ -12,9 +12,9 @@ import (
 // Uses less memory than loading all documents and running the expression once, but this cannot process
 // cross document expressions.
 type StreamEvaluator interface {
-	Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer) error
-	EvaluateFiles(expression string, filenames []string, printer Printer) error
-	EvaluateNew(expression string, printer Printer) error
+	Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, leadingContent string) (uint, error)
+	EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error
+	EvaluateNew(expression string, printer Printer, leadingContent string) error
 }
 
 type streamEvaluator struct {
@@ -27,7 +27,7 @@ func NewStreamEvaluator() StreamEvaluator {
 	return &streamEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: NewExpressionParser()}
 }
 
-func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error {
+func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadingContent string) error {
 	node, err := s.treeCreator.ParseExpression(expression)
 	if err != nil {
 		return err
@@ -35,7 +35,7 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error
 	candidateNode := &CandidateNode{
 		Document:  0,
 		Filename:  "",
-		Node:      &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode},
+		Node:      &yaml.Node{Kind: yaml.DocumentNode, HeadComment: leadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
 		FileIndex: 0,
 	}
 	inputList := list.New()
@@ -48,35 +48,47 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer) error
 	return printer.PrintResults(result.MatchingNodes)
 }
 
-func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer) error {
-
+func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error {
+	var totalProcessDocs uint = 0
 	node, err := s.treeCreator.ParseExpression(expression)
 	if err != nil {
 		return err
 	}
 
-	for _, filename := range filenames {
-		reader, err := readStream(filename)
+	var firstFileLeadingContent string
+
+	for index, filename := range filenames {
+		reader, leadingContent, err := readStream(filename, leadingContentPreProcessing)
+
+		if index == 0 {
+			firstFileLeadingContent = leadingContent
+		}
+
 		if err != nil {
 			return err
 		}
-		err = s.Evaluate(filename, reader, node, printer)
+		processedDocs, err := s.Evaluate(filename, reader, node, printer, leadingContent)
 		if err != nil {
 			return err
 		}
+		totalProcessDocs = totalProcessDocs + processedDocs
 
 		switch reader := reader.(type) {
 		case *os.File:
 			safelyCloseFile(reader)
 		}
 	}
+
+	if totalProcessDocs == 0 {
+		return s.EvaluateNew(expression, printer, firstFileLeadingContent)
+	}
+
 	return nil
 }
 
-func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer) error {
+func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, leadingContent string) (uint, error) {
 
 	var currentIndex uint
-
 	decoder := yaml.NewDecoder(reader)
 	for {
 		var dataBucket yaml.Node
@@ -84,9 +96,12 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr
 
 		if errorReading == io.EOF {
 			s.fileIndex = s.fileIndex + 1
-			return nil
+			return currentIndex, nil
 		} else if errorReading != nil {
-			return errorReading
+			return currentIndex, errorReading
+		}
+		if currentIndex == 0 {
+			dataBucket.HeadComment = leadingContent
 		}
 		candidateNode := &CandidateNode{
 			Document:  currentIndex,
@@ -99,11 +114,12 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr
 
 		result, errorParsing := s.treeNavigator.GetMatchingNodes(Context{MatchingNodes: inputList}, node)
 		if errorParsing != nil {
-			return errorParsing
+			return currentIndex, errorParsing
 		}
 		err := printer.PrintResults(result.MatchingNodes)
+
 		if err != nil {
-			return err
+			return currentIndex, err
 		}
 		currentIndex = currentIndex + 1
 	}

pkg/yqlib/utils.go

@@ -5,15 +5,62 @@ import (
 	"container/list"
 	"io"
 	"os"
+	"regexp"
+	"strings"
 
 	yaml "gopkg.in/yaml.v3"
 )
 
-func readStream(filename string) (io.Reader, error) {
+func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, string, error) {
+	var reader *bufio.Reader
 	if filename == "-" {
-		return bufio.NewReader(os.Stdin), nil
+		reader = bufio.NewReader(os.Stdin)
 	} else {
-		return os.Open(filename) // nolint gosec
+		// ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory,
+		// and ensuring that it's not possible to give a path to a file outside thar directory.
+		file, err := os.Open(filename) // #nosec
+		if err != nil {
+			return nil, "", err
+		}
+		reader = bufio.NewReader(file)
+	}
+
+	if !leadingContentPreProcessing {
+		return reader, "", nil
+	}
+	return processReadStream(reader)
+}
+
+func processReadStream(reader *bufio.Reader) (io.Reader, string, error) {
+	var commentLineRegEx = regexp.MustCompile(`^\s*#`)
+	var sb strings.Builder
+	sb.WriteString("$yqLeadingContent$\n")
+	for {
+		peekBytes, err := reader.Peek(3)
+		if err == io.EOF {
+			// EOF are handled else where..
+			return reader, sb.String(), nil
+		} else if err != nil {
+			return reader, sb.String(), err
+		} else if string(peekBytes) == "---" {
+			_, err := reader.ReadString('\n')
+			sb.WriteString("$yqDocSeperator$\n")
+			if err == io.EOF {
+				return reader, sb.String(), nil
+			} else if err != nil {
+				return reader, sb.String(), err
+			}
+		} else if commentLineRegEx.MatchString(string(peekBytes)) {
+			line, err := reader.ReadString('\n')
+			sb.WriteString(line)
+			if err == io.EOF {
+				return reader, sb.String(), nil
+			} else if err != nil {
+				return reader, sb.String(), err
+			}
+		} else {
+			return reader, sb.String(), nil
+		}
 	}
 }
 

pkg/yqlib/write_in_place_handler.go

@@ -1,7 +1,6 @@
 package yqlib
 
 import (
-	"io/ioutil"
 	"os"
 )
 
@@ -21,27 +20,21 @@ func NewWriteInPlaceHandler(inputFile string) writeInPlaceHandler {
 }
 
 func (w *writeInPlaceHandlerImpl) CreateTempFile() (*os.File, error) {
+	file, err := createTempFile()
+
+	if err != nil {
+		return nil, err
+	}
 	info, err := os.Stat(w.inputFilename)
 	if err != nil {
 		return nil, err
 	}
-	_, err = os.Stat(os.TempDir())
-	if os.IsNotExist(err) {
-		err = os.Mkdir(os.TempDir(), 0700)
-		if err != nil {
-			return nil, err
-		}
-	} else if err != nil {
-		return nil, err
-	}
+	err = os.Chmod(file.Name(), info.Mode())
 
-	file, err := ioutil.TempFile("", "temp")
 	if err != nil {
 		return nil, err
 	}
-
-	err = os.Chmod(file.Name(), info.Mode())
-	log.Debug("writing to tempfile: %v", file.Name())
+	log.Debug("WriteInPlaceHandler: writing to tempfile: %v", file.Name())
 	w.tempFile = file
 	return file, err
 }
@@ -50,10 +43,9 @@ func (w *writeInPlaceHandlerImpl) FinishWriteInPlace(evaluatedSuccessfully bool)
 	log.Debug("Going to write-inplace, evaluatedSuccessfully=%v, target=%v", evaluatedSuccessfully, w.inputFilename)
 	safelyCloseFile(w.tempFile)
 	if evaluatedSuccessfully {
-		log.Debug("moved temp file to target")
+		log.Debug("Moving temp file to target")
 		safelyRenameFile(w.tempFile.Name(), w.inputFilename)
 	} else {
-		log.Debug("removed temp file")
-		os.Remove(w.tempFile.Name())
+		tryRemoveFile(w.tempFile.Name())
 	}
 }

release_instructions.txt

@@ -1,3 +1,4 @@
+- update release notes
 - increment version in version.go
 - increment version in snapcraft.yaml
 - increment version in github-action/Dockerfile
@@ -12,6 +13,7 @@
 
 - snapcraft
   - will auto create a candidate, test it works then promote
+  - !! need to update v4/stable as well as latest
   
   sudo snap remove yq
   sudo snap install --edge yq

release_notes.txt

@@ -0,0 +1,33 @@
+4.13.1:
+
+- Update to `with` operator, allow for no leading space on the `;`.
+
+4.13.0:
+
+BREAKING CHANGE - the `as` variable operator (e.g. `.a as $x`) now makes a _copy_ of the node(s) at the 
+path rather than a reference. This is in order to make it work more like the `jq` equivalent. 
+
+This means any updates made to that variable do not update the original.
+
+There's a new operator `ref` that will make a reference (and allow multiple updates to the original path by referencing the variable).
+Sorry for any inconvenience caused!.
+
+
+- New `with` operator for making multiple changes to a given path
+- New `contains` operator, works like the `jq` equivalent
+- Subtract operator now supports subtracting elements from arrays!
+- Fixed Swapping values using variables #934
+- Github Action now properly supports multiline output #936, thanks @pjxiao
+- Fixed missing closing bracket validation #932
+
+4.12.2:
+- Fix processing of hex numbers #929
+- Fixed alternative and union operator issues #930
+
+4.12.1:
+ - Merge comment fix #919
+
+4.12.0:
+- Can now convert yaml to properties properties format (`-o=props`), See [docs](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties) for more info.
+- Fixed document header/footer comment handling when merging (https://github.com/mikefarah/yq/issues/919)
+- pretty print yaml 1.1 compatibility  (https://github.com/mikefarah/yq/issues/914)

scripts/acceptance.sh

@@ -1,64 +1,10 @@
-#!/bin/bash
-
+#! /bin/bash
 set -e
 
-# acceptance test
+for test in acceptance_tests/*.sh; do
+  echo "--------------------------------------------------------------"
+  echo "$test"
+  echo "--------------------------------------------------------------"
+  (exec "$test");
+done
 
-
-
-echo "test eval-sequence"
-random=$((1 + $RANDOM % 10))
-./yq e -n ".a = $random" > test.yml
-X=$(./yq e '.a' test.yml)
-
-if [[ $X != $random ]]; then
-  echo "Failed create: expected $random but was $X"
-  exit 1
-fi
-
-echo "--success"
-
-echo "test update-in-place"
-
-update=$(($random + 1))
-./yq e -i ".a = $update" test.yml
-
-X=$(./yq e '.a' test.yml)
-if [[ $X != $update ]]; then
-  echo "Failed to update inplace test: expected $update but was $X"
-  exit 1
-fi
-
-echo "--success"
-
-echo "test eval-all"
-./yq ea -n ".a = $random" > test-eval-all.yml
-Y=$(./yq ea '.a' test-eval-all.yml)
-
-if [[ $Y != $random ]]; then
-  echo "Failed create with eval all: expected $random but was $X"
-  exit 1
-fi
-echo "--success"
-
-echo "test no exit status"
-./yq e '.z' test.yml
-echo "--success"
-
-echo "test exit status"
-set +e
-
-./yq e -e '.z' test.yml
-
-if [[ $? != 1 ]]; then
-  echo "Expected error code 1 but was $?"
-  exit 1
-fi
-
-echo "--success"
-
-set -e
-
-rm test.yml
-rm test-eval-all.yml
-echo "acceptance tests passed"

scripts/format.sh

@@ -1,5 +1,6 @@
 #!/bin/bash
 
 find . \( -path ./vendor \) -prune -o -name "*.go" -exec goimports -w {} \;
+gofmt -w -s .
 go mod tidy
-go mod vendor
+go mod vendor

scripts/secure.sh

@@ -5,7 +5,7 @@ set -o pipefail
 
 if command -v gosec &> /dev/null
 then
-    gosec ${PWD}
+    gosec "${PWD}" ./...
 else
-  ./bin/gosec ${PWD}
+  ./bin/gosec "${PWD}" ./...
 fi

scripts/xcompile.sh

@@ -4,15 +4,16 @@ set -e
 # at https://github.com/inconshreveable/gonative
 
 
-CGO_ENABLED=0 gox -ldflags "${LDFLAGS}" -output="build/yq_{{.OS}}_{{.Arch}}" --osarch="darwin/amd64 freebsd/386 freebsd/amd64 freebsd/arm linux/386 linux/amd64 linux/arm linux/arm64 linux/mips linux/mips64 linux/mips64le linux/mipsle linux/ppc64 linux/ppc64le linux/s390x netbsd/386 netbsd/amd64 netbsd/arm openbsd/386 openbsd/amd64 windows/386 windows/amd64"
+CGO_ENABLED=0 gox -ldflags "${LDFLAGS}" -output="build/yq_{{.OS}}_{{.Arch}}" --osarch="darwin/amd64 darwin/arm64 freebsd/386 freebsd/amd64 freebsd/arm linux/386 linux/amd64 linux/arm linux/arm64 linux/mips linux/mips64 linux/mips64le linux/mipsle linux/ppc64 linux/ppc64le linux/s390x netbsd/386 netbsd/amd64 netbsd/arm openbsd/386 openbsd/amd64 windows/386 windows/amd64"
 
 cd build
+
+find . -executable -type f | xargs -I {} tar czvf {}.tar.gz {}
+
 rhash -r -a . -o checksums
 
 rhash --list-hashes > checksums_hashes_order
 
-find . -executable -type f | xargs -I {} tar czvf {}.tar.gz {}
-
 # just in case find thinks this is executable...
 rm -f checksums_hashes_order.tar.gz 
 rm -f checksums.tar.gz
@@ -21,4 +22,4 @@ rm yq_windows_386.exe.tar.gz
 rm yq_windows_amd64.exe.tar.gz
 
 zip yq_windows_386.zip yq_windows_386.exe
-zip yq_windows_amd64.zip yq_windows_amd64.exe
+zip yq_windows_amd64.zip yq_windows_amd64.exe