Man page release workflow wip

This reverts commit 4ce9433468.

.github/workflows/release-test.yml

@@ -0,0 +1,95 @@
+name: Test Release YQ
+on:
+  push:
+    tags:
+      - 'releaseTest-*'
+
+jobs:
+  publishGitRelease:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '^1.17.2'
+      - name: Compile man page markup
+        id: gen-man-page-md
+        run: |
+          ./scripts/generate-man-page-md.sh
+          echo "::set-output name=man-page-md::man.md"
+
+      - name: Get the version
+        id: get_version
+        run: echo ::set-output name=VERSION::${GITHUB_REF##*/}
+
+      - name: Generate man page
+        uses: docker://pandoc/core:2.14.2
+        id: gen-man-page
+        with:
+          args: >-
+            --standalone
+            --to man
+            --variable=title:"YQ"
+            --variable=section:"1"
+            --variable=header:"yq (https://github.com/mikefarah/yq/) version ${{ steps.get_version.outputs.VERSION }}"
+            --variable=author:"Mike Farah"
+            --output=yq.1
+            ${{ steps.gen-man-page-md.outputs.man-page-md }}
+        
+      - name: Cross compile
+        run: |
+          sudo apt-get install rhash -y
+          go get github.com/mitchellh/gox
+          mkdir -p build
+          cat yq.1
+          cp yq.1 build/yq.1
+          ./scripts/xcompile.sh
+
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1.0.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.ref }}
+          release_name: TEST ${{ github.ref }}
+          draft: true
+          prerelease: false
+
+      - uses: shogo82148/actions-upload-release-asset@v1
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: build/*
+
+  # publishDocker:
+  #   environment: dockerhub
+  #   env:
+  #     IMAGE_NAME: mikefarah/yq
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v2
+      
+  #     - name: Set up QEMU
+  #       uses: docker/setup-qemu-action@v1
+  #       with:
+  #         platforms: all
+  
+  #     - name: Set up Docker Buildx
+  #       id: buildx
+  #       uses: docker/setup-buildx-action@v1
+  #       with:
+  #         version: latest
+
+  #     - name: Available platforms
+  #       run: echo ${{ steps.buildx.outputs.platforms }} && docker version
+ 
+  #     - name: Build image
+  #       run: |
+  #         IMAGE_V_VERSION="$(git describe --tags --abbrev=0)"
+  #         IMAGE_VERSION=${IMAGE_V_VERSION:1}
+
+  #         SHORT_SHA1=$(git rev-parse --short HEAD)
+  #         PLATFORMS="linux/amd64,linux/ppc64le,linux/arm64"
+  #         echo "Building version ${IMAGE_VERSION} of image ${IMAGE_NAME}"
+  #         docker buildx build --platform "${PLATFORMS}" .
+    

.github/workflows/release.yml

@@ -11,7 +11,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
-          go-version: '^1.15'
+          go-version: '^1.17.2'
       - name: Cross compile
         run: |
           sudo apt-get install rhash -y

.gitignore

@@ -40,3 +40,8 @@ prime/
 yq*.snap
 test.yml
 test2.yml
+
+
+# man page
+man.md
+yq.1

Dockerfile

@@ -1,18 +1,16 @@
-FROM golang:1.15 as builder
+FROM golang:1.17 as builder
 
 WORKDIR /go/src/mikefarah/yq
 
-# cache devtools
-COPY ./scripts/devtools.sh /go/src/mikefarah/yq/scripts/devtools.sh
-RUN ./scripts/devtools.sh
-
 COPY . /go/src/mikefarah/yq
 
-RUN CGO_ENABLED=0 make local build
+RUN CGO_ENABLED=0 go build .
+RUN ./scripts/test.sh
+RUN ./scripts/acceptance.sh
 
 # Choose alpine as a base image to make this useful for CI, as many
 # CI tools expect an interactive shell inside the container
-FROM alpine:3.13.5 as production
+FROM alpine:3 as production
 
 RUN mkdir /home/yq/
 RUN addgroup -g 1000 yq && \

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM golang:1.15
+FROM golang:1.17
 
 COPY scripts/devtools.sh /opt/devtools.sh
 

README.md

@@ -7,12 +7,43 @@ a lightweight and portable command-line YAML processor. `yq` uses [jq](https://g
 
 yq is written in go - so you can download a dependency free binary for your platform and you are good to go! If you prefer there are a variety of package managers that can be used as well as docker, all listed below.
 
-## V4 released!
-V4 is now officially released, it's quite different from V3 (sorry for the migration), however it is much more similar to ```jq```, using a similar expression syntax and therefore support much more complex functionality! 
+## Quick Usage Guide
 
-If you've been using v3 and want/need to upgrade, checkout the [upgrade guide](https://mikefarah.gitbook.io/yq/v/v4.x/upgrading-from-v3).
+Read a value:
+```bash
+yq e '.a.b[0].c' file.yaml
+```
 
-Support for v3 will cease August 2021, until then, critical bug and security fixes will still get applied if required.
+Pipe from STDIN:
+```bash
+cat file.yaml | yq e '.a.b[0].c' -
+```
+
+Update a yaml file, inplace
+```bash
+yq e -i '.a.b[0].c = "cool"' file.yaml
+```
+
+Update using environment variables
+```bash
+NAME=mike yq e -i '.a.b[0].c = strenv(NAME)' file.yaml
+```
+
+Merge multiple files
+```
+yq ea '. as $item ireduce ({}; . * $item )' path/to/*.yml
+```
+
+Multiple updates to a yaml file
+```bash
+yq e -i '
+  .a.b[0].c = "cool" |
+  .x.y.z = "foobar" |
+  .person.name = strenv(NAME)
+' file.yaml
+```
+
+See the [documentation](https://mikefarah.gitbook.io/yq/) for more.
 
 ## Install
 
@@ -87,6 +118,14 @@ rm /etc/myfile.tmp
 docker run --rm -v "${PWD}":/workdir mikefarah/yq <command> [flags] [expression ]FILE...
 ```
 
+#### Pipe in via STDIN:
+
+You'll need to pass the `-i\--interactive` flag to docker:
+
+```bash
+cat myfile.yml | docker run -i --rm mikefarah/yq e . -
+```
+
 #### Run commands interactively:
 
 ```bash
@@ -101,6 +140,7 @@ yq() {
 }
 ```
 
+
 #### Running as root:
 
 `yq`'s docker image no longer runs under root (https://github.com/mikefarah/yq/pull/860). If you'd like to install more things in the docker image, or you're having permissions issues when attempting to read/write files you'll need to either:
@@ -120,6 +160,16 @@ RUN apk add bash
 USER yq
 ```
 
+### GitHub Action
+```
+  - name: Set foobar to cool
+    uses: mikefarah/yq@master
+    with:
+      cmd: yq eval -i '.foo.bar = "cool"' 'config.yml'
+```
+
+See https://mikefarah.gitbook.io/yq/usage/github-action for more.
+
 ### Go Get:
 ```
 GO111MODULE=on go get github.com/mikefarah/yq/v4
@@ -184,7 +234,8 @@ Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
 - [Update yaml inplace](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags)
 - [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/operators/select#select-and-update-matching-values-in-map)
 - Keeps yaml formatting and comments when updating (though there are issues with whitespace)
-- [Convert to/from json to yaml](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
+- [Convert to/from json](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
+- [Convert to properties](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties)
 - [Pipe data in by using '-'](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate)
 - [General shell completion scripts (bash/zsh/fish/powershell)](https://mikefarah.gitbook.io/yq/v/v4.x/commands/shell-completion)
 - [Reduce](https://mikefarah.gitbook.io/yq/operators/reduce) to merge multiple files or sum an array or other fancy things.
@@ -209,27 +260,21 @@ Flags:
   -C, --colors                force print with colors
   -e, --exit-status           set exit status if there are no matches or null or false is returned
   -f, --front-matter string   (extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact
+      --header-preprocess     Slurp any header comments and seperators before processing expression. This is a workaround for go-yaml to persist header content (default true)
   -h, --help                  help for yq
   -I, --indent int            sets indent level for output (default 2)
   -i, --inplace               update the yaml file inplace of first yaml file given.
   -M, --no-colors             force print with no colors
   -N, --no-doc                Don't print document separators (---)
   -n, --null-input            Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.
+  -o, --output-format string  [yaml|y|json|j|props|p] output format type. (default "yaml")
   -P, --prettyPrint           pretty print, shorthand for '... style = ""'
-  -j, --tojson                output as json. Set indent to 0 to print json in one line.
       --unwrapScalar          unwrap scalar, print the value with no quotes, colors or comments (default true)
   -v, --verbose               verbose mode
   -V, --version               Print version information and quit
 
 Use "yq [command] --help" for more information about a command.
 ```
-
-Simple Example:
-
-```bash
-yq e '.a.b | length' f1.yml f2.yml 
-```
-
 ## Known Issues / Missing Features
 - `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details)
 - Powershell has its own...opinions: https://mikefarah.gitbook.io/yq/usage/tips-and-tricks#quotes-in-windows-powershell

acceptance_tests/leading-seperator.sh

@@ -296,6 +296,35 @@ EOM
   assertEquals "$expected" "$X"
 }
 
+# https://github.com/mikefarah/yq/issues/919
+testLeadingSeparatorDoesNotBreakCommentsOnOtherFiles() {
+  cat >test.yml <<EOL
+# a1
+a: 1
+# a2
+EOL
+
+cat >test2.yml <<EOL
+# b1
+b: 2
+# b2
+EOL
+
+  read -r -d '' expected << EOM
+# a1
+a: 1
+# a2
+
+# b1
+b: 2
+# b2
+EOM
+
+
+  X=$(./yq ea 'select(fi == 0) * select(fi == 1)' test.yml test2.yml)
+  assertEquals "$expected" "$X"
+}
+
 testLeadingSeperatorMultiDocEvalCommentsStripComments() {
   cat >test.yml <<EOL
 ---

acceptance_tests/output-format.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+
+testOutputJsonDeprecated() {
+  cat >test.yml <<EOL
+a: {b: ["cat"]}
+EOL
+
+  read -r -d '' expected << EOM
+{
+  "a": {
+    "b": [
+      "cat"
+    ]
+  }
+}
+EOM
+
+  X=$(./yq e -j test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -j test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testOutputJson() {
+  cat >test.yml <<EOL
+a: {b: ["cat"]}
+EOL
+
+  read -r -d '' expected << EOM
+{
+  "a": {
+    "b": [
+      "cat"
+    ]
+  }
+}
+EOM
+
+  X=$(./yq e --output-format=json test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --output-format=json test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testOutputJsonShort() {
+  cat >test.yml <<EOL
+a: {b: ["cat"]}
+EOL
+
+  read -r -d '' expected << EOM
+{
+  "a": {
+    "b": [
+      "cat"
+    ]
+  }
+}
+EOM
+
+  X=$(./yq e -o=j test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -o=j test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testOutputProperties() {
+  cat >test.yml <<EOL
+a: {b: {c: ["cat"]}}
+EOL
+
+  read -r -d '' expected << EOM
+a.b.c.0 = cat
+EOM
+
+  X=$(./yq e --output-format=props test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --output-format=props test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testOutputPropertiesShort() {
+  cat >test.yml <<EOL
+a: {b: {c: ["cat"]}}
+EOL
+
+  read -r -d '' expected << EOM
+a.b.c.0 = cat
+EOM
+
+  X=$(./yq e -o=p test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -o=p test.yml)
+  assertEquals "$expected" "$X"
+}
+
+source ./scripts/shunit2

acceptance_tests/pretty-print.sh

@@ -0,0 +1,173 @@
+#!/bin/bash
+
+
+testPrettyPrintWithBooleans() {
+  cat >test.yml <<EOL
+leaveUnquoted: [yes, no, on, off, y, n, true, false]
+leaveQuoted: ["yes", "no", "on", "off", "y", "n", "true", "false"]
+
+EOL
+
+  read -r -d '' expected << EOM
+leaveUnquoted:
+  - yes
+  - no
+  - on
+  - off
+  - y
+  - n
+  - true
+  - false
+leaveQuoted:
+  - "yes"
+  - "no"
+  - "on"
+  - "off"
+  - "y"
+  - "n"
+  - "true"
+  - "false"
+EOM
+
+  X=$(./yq e --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testPrettyPrintWithBooleansCapitals() {
+  cat >test.yml <<EOL
+leaveUnquoted: [YES, NO, ON, OFF, Y, N, TRUE, FALSE]
+leaveQuoted: ["YES", "NO", "ON", "OFF", "Y", "N", "TRUE", "FALSE"]
+EOL
+
+  read -r -d '' expected << EOM
+leaveUnquoted:
+  - YES
+  - NO
+  - ON
+  - OFF
+  - Y
+  - N
+  - TRUE
+  - FALSE
+leaveQuoted:
+  - "YES"
+  - "NO"
+  - "ON"
+  - "OFF"
+  - "Y"
+  - "N"
+  - "TRUE"
+  - "FALSE"
+EOM
+
+  X=$(./yq e --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testPrettyPrintOtherStringValues() {
+  cat >test.yml <<EOL
+leaveUnquoted: [yesSir, hellno, bonapite]
+makeUnquoted: ["yesSir", "hellno", "bonapite"]
+EOL
+
+  read -r -d '' expected << EOM
+leaveUnquoted:
+  - yesSir
+  - hellno
+  - bonapite
+makeUnquoted:
+  - yesSir
+  - hellno
+  - bonapite
+EOM
+
+  X=$(./yq e --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testPrettyPrintKeys() {
+  cat >test.yml <<EOL
+"removeQuotes": "please"
+EOL
+
+  read -r -d '' expected << EOM
+removeQuotes: please
+EOM
+
+  X=$(./yq e --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testPrettyPrintOtherStringValues() {
+  cat >test.yml <<EOL
+leaveUnquoted: [yesSir, hellno, bonapite]
+makeUnquoted: ["yesSir", "hellno", "bonapite"]
+EOL
+
+  read -r -d '' expected << EOM
+leaveUnquoted:
+  - yesSir
+  - hellno
+  - bonapite
+makeUnquoted:
+  - yesSir
+  - hellno
+  - bonapite
+EOM
+
+  X=$(./yq e --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testPrettyPrintStringBlocks() {
+  cat >test.yml <<EOL
+"removeQuotes": | 
+  "please"
+EOL
+
+  read -r -d '' expected << EOM
+removeQuotes: |
+  "please"
+EOM
+
+  X=$(./yq e --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testPrettyPrintWithExpression() {
+  cat >test.yml <<EOL
+a: {b: {c: ["cat"]}}
+EOL
+
+  read -r -d '' expected << EOM
+b:
+  c:
+    - cat
+EOM
+
+  X=$(./yq e '.a' --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea '.a' --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+}
+
+source ./scripts/shunit2

cmd/constant.go

@@ -5,6 +5,7 @@ var unwrapScalar = true
 
 var writeInplace = false
 var outputToJSON = false
+var outputFormat = "yaml"
 
 var exitStatus = false
 var forceColor = false

cmd/evaluate_all_command.go

@@ -24,8 +24,8 @@ yq ea -i 'select(fi == 0) * select(fi == 1)' f1.yml f2.yml
 # Merge all given files
 yq ea '. as $item ireduce ({}; . * $item )' file1.yml file2.yml ...
 
-# Read from STDIN
-## use '-' as a filename to read from STDIN
+# Pipe from STDIN
+## use '-' as a filename to pipe from STDIN
 cat file2.yml | yq ea '.a.b' file1.yml - file3.yml
 `,
 		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
@@ -34,8 +34,7 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.
 ## Evaluate All ##
 This command loads _all_ yaml documents of _all_ yaml files and runs expression once
 Useful when you need to run an expression across several yaml documents or files (like merge).
-If you're just merging entire multiple files together, you may want to consider eval-reduce as it's faster.
-Note that it consumes more memory than eval and eval-reduce.
+Note that it consumes more memory than eval.
 `,
 		RunE: evaluateAll,
 	}
@@ -87,8 +86,17 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 	if nullInput && len(args) > 1 {
 		return errors.New("Cannot pass files in when using null-input flag")
 	}
+	// backwards compatibilty
+	if outputToJSON {
+		outputFormat = "json"
+	}
 
-	printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+	format, err := yqlib.OutputFormatFromString(outputFormat)
+	if err != nil {
+		return err
+	}
+
+	printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
 
 	if frontMatter != "" {
 		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])

cmd/evaluate_reduce_command.go

@@ -1,112 +0,0 @@
-package cmd
-
-import (
-	"errors"
-	"fmt"
-	"os"
-
-	"github.com/mikefarah/yq/v4/pkg/yqlib"
-	"github.com/spf13/cobra"
-)
-
-func createEvaluateReduceCommand() *cobra.Command {
-	var cmdEvalReduce = &cobra.Command{
-		Use:     "eval-reduce [reduce expression] [yaml_file1]...",
-		Aliases: []string{"er"},
-		Short:   "Runs a reduce expression sequentially against each document of each file given. More memory efficient than using eval-all if you can get away with it.",
-		Example: `
-# Merge f2.yml into f1.yml (inplace)
-yq eval-reduce --inplace '{} ; . * $doc' f1.yml f2.yml
-`,
-		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
-See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.
-
-## Evaluate Reduce ##
-This command runs the reduce expression against each document of each file given, accumulating the results.
-It is most useful when merging multiple files (but isn't as flexible as using eval-all with ireduce, as you
-can only merge the top level nodes).
-`,
-		RunE: evaluateReduce,
-	}
-	return cmdEvalReduce
-}
-func evaluateReduce(cmd *cobra.Command, args []string) error {
-	cmd.SilenceUsage = true
-	// 2+ args, [0] = expression, file the rest
-
-	var err error
-
-	out := cmd.OutOrStdout()
-
-	fileInfo, _ := os.Stdout.Stat()
-
-	if forceColor || (!forceNoColor && (fileInfo.Mode()&os.ModeCharDevice) != 0) {
-		colorsEnabled = true
-	}
-
-	firstFileIndex := -1
-	if !nullInput && len(args) == 1 {
-		firstFileIndex = 0
-	} else if len(args) > 1 {
-		firstFileIndex = 1
-	}
-
-	if writeInplace && (firstFileIndex == -1) {
-		return fmt.Errorf("Write inplace flag only applicable when giving an expression and at least one file")
-	}
-
-	if writeInplace {
-		// only use colors if its forced
-		colorsEnabled = forceColor
-		writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[firstFileIndex])
-		out, err = writeInPlaceHandler.CreateTempFile()
-		if err != nil {
-			return err
-		}
-		// need to indirectly call the function so  that completedSuccessfully is
-		// passed when we finish execution as opposed to now
-		defer func() { writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully) }()
-	}
-
-	if nullInput && len(args) > 1 {
-		return errors.New("Cannot pass files in when using null-input flag")
-	}
-
-	printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
-
-	if frontMatter != "" {
-		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])
-		err = frontMatterHandler.Split()
-		if err != nil {
-			return err
-		}
-		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
-
-		if frontMatter == "process" {
-			reader := frontMatterHandler.GetContentReader()
-			printer.SetAppendix(reader)
-			defer yqlib.SafelyCloseReader(reader)
-		}
-		defer frontMatterHandler.CleanUp()
-	}
-
-	reduceEvaluator := yqlib.NewReduceEvaluator()
-	switch len(args) {
-	case 0:
-		cmd.Println(cmd.UsageString())
-		return nil
-	case 1:
-		cmd.Println(cmd.UsageString())
-		return nil
-	default:
-		err = reduceEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing)
-	}
-
-	completedSuccessfully = err == nil
-
-	if err == nil && exitStatus && !printer.PrintedAnything() {
-		return errors.New("no matches found")
-	}
-
-	return err
-}

cmd/evalute_sequence_command.go

@@ -21,8 +21,8 @@ yq e '.a.b' f1.yml f2.yml
 # Prints out the file
 yq e sample.yaml 
 
-# Read from STDIN
-## use '-' as a filename to read from STDIN
+# Pipe from STDIN
+## use '-' as a filename to pipe from STDIN
 cat file2.yml | yq e '.a.b' file1.yml - file3.yml
 
 # Creates a new yaml document
@@ -44,10 +44,11 @@ expression and prints the result in sequence.`,
 }
 
 func processExpression(expression string) string {
+	var prettyPrintExp = `(... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not))  ) style=""`
 	if prettyPrint && expression == "" {
-		return `... style=""`
+		return prettyPrintExp
 	} else if prettyPrint {
-		return fmt.Sprintf("%v | ... style= \"\"", expression)
+		return fmt.Sprintf("%v | %v", expression, prettyPrintExp)
 	}
 	return expression
 }
@@ -95,7 +96,17 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 		defer func() { writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully) }()
 	}
 
-	printer := yqlib.NewPrinter(out, outputToJSON, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+	// backwards compatibilty
+	if outputToJSON {
+		outputFormat = "json"
+	}
+
+	format, err := yqlib.OutputFormatFromString(outputFormat)
+	if err != nil {
+		return err
+	}
+
+	printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
 
 	streamEvaluator := yqlib.NewStreamEvaluator()
 

cmd/root.go

@@ -41,7 +41,14 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 	}
 
 	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "verbose mode")
-	rootCmd.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "output as json. Set indent to 0 to print json in one line.")
+
+	rootCmd.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "(deprecated) output as json. Set indent to 0 to print json in one line.")
+	err := rootCmd.PersistentFlags().MarkDeprecated("tojson", "please use -o=json instead")
+	if err != nil {
+		panic(err)
+	}
+
+	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "yaml", "[yaml|y|json|j|props|p] output format type.")
 	rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.")
 	rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)")
 
@@ -55,11 +62,10 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 	rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")
 	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
 	rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
-	rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and seperators before processing expression. This is a workaround for go-yaml to persist header content. This flag will be removed once this feature has been out in the wild for a while.")
+	rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression. This is a workaround for go-yaml to persist header content properly.")
 	rootCmd.AddCommand(
 		createEvaluateSequenceCommand(),
 		createEvaluateAllCommand(),
-		createEvaluateReduceCommand(),
 		completionCmd,
 	)
 	return rootCmd

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.11.1"
+	Version = "4.13.4"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

debian/changelog

@@ -1,3 +1,19 @@
+yq (4.13.0) focal; urgency=medium
+
+  * New `with` operator for making multiple changes to a given path
+  * New `contains` operator, works like the `jq` equivalent
+  * Subtract operator now supports subtracting elements from arrays!
+  * Fixed Swapping values using variables #934
+  * Github Action now properly supports multiline output #936, thanks @pjxiao
+  * Fixed missing closing bracket validation #932
+  * Fix processing of hex numbers #929
+  * Fixed alternative and union operator issues #930
+  * Can now convert yaml to properties properties format (`-o=props`), See [docs](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties) for more info.
+  * Fixed document header/footer comment handling when merging (https://github.com/mikefarah/yq/issues/919)
+  * pretty print yaml 1.1 compatibility  (https://github.com/mikefarah/yq/issues/914)
+
+ -- Roberto Mier Escandon <rmescandon@gmail.com>  Thu, 16 Sep 2021 20:58:30 +0200
+
 yq (4.9.6) focal; urgency=medium
 
   * Added darwin/arm64 build, thanks @alecthomas

debian/rules

@@ -45,7 +45,9 @@ override_dh_auto_build:
 	(cd ${SRCDIR} && go install -buildmode=pie ./...)
 	
 override_dh_auto_test:
+ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
 	(cd ${SRCDIR} && go test -v ./...)
+endif
 
 override_dh_auto_install:
 	cp ${GOBIN}/yq ${DESTDIR}/${BINDIR}

examples/data1.yaml

@@ -1,3 +1,5 @@
-a: apple
----
-a2: fish
+Resources:
+  S3Bucket:
+    Type: AWS::CloudFormation::Stack
+    Properties:
+      BucketName: !Ref MyBucketNameA

examples/data2.yaml

@@ -1 +1,3 @@
-b: doc2
+#b1 
+b: 2
+#b2

github-action/Dockerfile

@@ -1,4 +1,4 @@
-FROM mikefarah/yq:4.11.1
+FROM mikefarah/yq:4.13.4
 
 COPY entrypoint.sh /entrypoint.sh
 

github-action/entrypoint.sh

@@ -1,6 +1,9 @@
 #!/bin/sh -l
-
+set -e
 echo "::debug::\$cmd: $1"
 RESULT=$(eval "$1")
+RESULT="${RESULT//'%'/'%25'}"
+RESULT="${RESULT//$'\n'/'%0A'}"
+RESULT="${RESULT//$'\r'/'%0D'}"
 echo "::debug::\$RESULT: $RESULT"
 echo ::set-output name=result::"$RESULT"

go.mod

@@ -5,12 +5,21 @@ require (
 	github.com/fatih/color v1.10.0
 	github.com/goccy/go-yaml v1.8.9
 	github.com/jinzhu/copier v0.2.8
+	github.com/magiconair/properties v1.8.5
 	github.com/spf13/cobra v1.1.3
-	github.com/timtadh/data-structures v0.5.3 // indirect
 	github.com/timtadh/lexmachine v0.2.2
-	golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )
 
-go 1.15
+require (
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/mattn/go-colorable v0.1.8 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/timtadh/data-structures v0.5.3 // indirect
+	golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+)
+
+go 1.17

go.sum

@@ -119,6 +119,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls=
+github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
@@ -256,8 +258,8 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE=
-golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA=
+golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=

pkg/yqlib/all_at_once_evaluator.go

@@ -52,7 +52,7 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
 
 	var allDocuments *list.List = list.New()
 	for _, filename := range filenames {
-		reader, leadingContent, err := readStream(filename, leadingContentPreProcessing)
+		reader, leadingContent, err := readStream(filename, fileIndex == 0 && leadingContentPreProcessing)
 		if err != nil {
 			return err
 		}

pkg/yqlib/context.go

@@ -2,8 +2,10 @@ package yqlib
 
 import (
 	"container/list"
+	"fmt"
 
 	"github.com/jinzhu/copier"
+	logging "gopkg.in/op/go-logging.v1"
 )
 
 type Context struct {
@@ -30,7 +32,6 @@ func (n *Context) GetVariable(name string) *list.List {
 	if n.Variables == nil {
 		return nil
 	}
-	log.Debug("GetVariable - %v to %v", name, NodesToString(n.Variables[name]))
 	return n.Variables[name]
 }
 
@@ -38,24 +39,56 @@ func (n *Context) SetVariable(name string, value *list.List) {
 	if n.Variables == nil {
 		n.Variables = make(map[string]*list.List)
 	}
-	log.Debug("SetVariable - %v to %v", name, NodesToString(value))
 	n.Variables[name] = value
 }
 
 func (n *Context) ChildContext(results *list.List) Context {
+	clone := Context{DontAutoCreate: n.DontAutoCreate}
+	clone.Variables = make(map[string]*list.List)
+	if len(n.Variables) > 0 {
+		err := copier.Copy(&clone.Variables, n.Variables)
+		if err != nil {
+			log.Error("Error cloning context :(")
+			panic(err)
+		}
+	}
+	clone.MatchingNodes = results
+	return clone
+}
+
+func (n *Context) ToString() string {
+	if !log.IsEnabledFor(logging.DEBUG) {
+		return ""
+	}
+	result := fmt.Sprintf("Context\nDontAutoCreate: %v\n", n.DontAutoCreate)
+	return result + NodesToString(n.MatchingNodes)
+}
+
+func (n *Context) DeepClone() Context {
 	clone := Context{}
 	err := copier.Copy(&clone, n)
+	// copier doesn't do lists properly for some reason
+	clone.MatchingNodes = list.New()
+	for el := n.MatchingNodes.Front(); el != nil; el = el.Next() {
+		clonedNode, err := el.Value.(*CandidateNode).Copy()
+		if err != nil {
+			log.Error("Error cloning context :(")
+			panic(err)
+		}
+		clone.MatchingNodes.PushBack(clonedNode)
+	}
+
 	if err != nil {
 		log.Error("Error cloning context :(")
 		panic(err)
 	}
-	clone.MatchingNodes = results
 	return clone
 }
 
 func (n *Context) Clone() Context {
 	clone := Context{}
 	err := copier.Copy(&clone, n)
+
 	if err != nil {
 		log.Error("Error cloning context :(")
 		panic(err)

pkg/yqlib/doc/Contains.md

@@ -0,0 +1,85 @@
+
+## Array contains array
+Array is equal or subset of
+
+Given a sample.yml file of:
+```yaml
+- foobar
+- foobaz
+- blarp
+```
+then
+```bash
+yq eval 'contains(["baz", "bar"])' sample.yml
+```
+will output
+```yaml
+true
+```
+
+## Object included in array
+Given a sample.yml file of:
+```yaml
+"foo": 12
+"bar":
+  - 1
+  - 2
+  - "barp": 12
+    "blip": 13
+```
+then
+```bash
+yq eval 'contains({"bar": [{"barp": 12}]})' sample.yml
+```
+will output
+```yaml
+true
+```
+
+## Object not included in array
+Given a sample.yml file of:
+```yaml
+"foo": 12
+"bar":
+  - 1
+  - 2
+  - "barp": 12
+    "blip": 13
+```
+then
+```bash
+yq eval 'contains({"foo": 12, "bar": [{"barp": 15}]})' sample.yml
+```
+will output
+```yaml
+false
+```
+
+## String contains substring
+Given a sample.yml file of:
+```yaml
+foobar
+```
+then
+```bash
+yq eval 'contains("bar")' sample.yml
+```
+will output
+```yaml
+true
+```
+
+## String equals string
+Given a sample.yml file of:
+```yaml
+meow
+```
+then
+```bash
+yq eval 'contains("meow")' sample.yml
+```
+will output
+```yaml
+true
+```
+

pkg/yqlib/doc/Multiply (Merge).md

@@ -357,7 +357,7 @@ yq eval '.foobar * .foobarList' sample.yml
 will output
 ```yaml
 c: foobarList_c
-<<:
+!!merge <<:
   - *foo
   - *bar
 thing: foobar_thing

pkg/yqlib/doc/String Operators.md

@@ -3,6 +3,45 @@
 ## RegEx
 This uses golangs native regex functions under the hood - See https://github.com/google/re2/wiki/Syntax for the supported syntax.
 
+
+# String blocks, bash and newlines
+Bash is notorious for chomping on precious trailing newline characters, making it tricky to set strings with newlines properly. In particular, the `$( exp )` _will trim trailing newlines_.
+
+For instance to get this yaml:
+
+```
+a: |
+  cat
+```
+
+Using `$( exp )` wont work, as it will trim the trailing new line.
+
+```
+m=$(echo "cat\n") yq e -n '.a = strenv(m)'
+a: cat
+```
+
+However, using printf works:
+```
+printf -v m "cat\n" ; m="$m" yq e -n '.a = strenv(m)'
+a: |
+  cat
+```
+
+As well as having multiline expressions:
+```
+m="cat
+"  yq e -n '.a = strenv(m)'
+a: |
+  cat
+```
+
+Similarly, if you're trying to set the content from a file, and want a trailing new line:
+
+```
+IFS= read -rd '' output < <(cat my_file)
+output=$output ./yq e '.data.values = strenv(output)' first.yml
+```
 ## Join strings
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/Style.md

@@ -18,8 +18,6 @@ a:
 ```
 
 ## Update and set style of a particular node using path variables
-You can use a variable to re-use a path
-
 Given a sample.yml file of:
 ```yaml
 a:
@@ -28,7 +26,7 @@ a:
 ```
 then
 ```bash
-yq eval '.a.b as $x | $x = "new" | $x style="double"' sample.yml
+yq eval 'with(.a.b ; . = "new" | . style="double")' sample.yml
 ```
 will output
 ```yaml

pkg/yqlib/doc/Subtract.md

@@ -1,4 +1,43 @@
 
+## Array subtraction
+Running
+```bash
+yq eval --null-input '[1,2] - [2,3]'
+```
+will output
+```yaml
+- 1
+```
+
+## Array subtraction with nested array
+Running
+```bash
+yq eval --null-input '[[1], 1, 2] - [[1], 3]'
+```
+will output
+```yaml
+- 1
+- 2
+```
+
+## Array subtraction with nested object
+Note that order of the keys does not matter
+
+Given a sample.yml file of:
+```yaml
+- a: b
+  c: d
+- a: b
+```
+then
+```bash
+yq eval '. - [{"c": "d", "a": "b"}]' sample.yml
+```
+will output
+```yaml
+- a: b
+```
+
 ## Number subtraction - float
 If the lhs or rhs are floats then the expression will be calculated with floats.
 

pkg/yqlib/doc/Variable Operators.md

@@ -1,4 +1,6 @@
-For more complex scenarios, variables can be used to hold values of expression to be used in other expressions.
+Like the `jq` equivalents, variables are sometimes required for the more complex expressions (or swapping values between fields).
+
+Note that there is also an additional `ref` operator that holds a reference (instead of a copy) of the path, allowing you to make multiple changes to the same path.
 
 ## Single value variable
 Given a sample.yml file of:
@@ -56,3 +58,39 @@ title: A well-written article
 author: Person McPherson
 ```
 
+## Using variables to swap values
+Given a sample.yml file of:
+```yaml
+a: a_value
+b: b_value
+```
+then
+```bash
+yq eval '.a as $x  | .b as $y | .b = $x | .a = $y' sample.yml
+```
+will output
+```yaml
+a: b_value
+b: a_value
+```
+
+## Use ref to reference a path repeatedly
+Note: You may find the `with` operator more useful.
+
+Given a sample.yml file of:
+```yaml
+a:
+  b: thing
+  c: something
+```
+then
+```bash
+yq eval '.a.b ref $x | $x = "new" | $x style="double"' sample.yml
+```
+will output
+```yaml
+a:
+  b: "new"
+  c: something
+```
+

pkg/yqlib/doc/With.md

@@ -0,0 +1,62 @@
+Use the `with` operator to conveniently make multiple updates to a deeply nested path, or to update array elements relatively to each other. The first argument expression sets the root context, and the second expression runs against that root context.
+
+## Update and style
+Given a sample.yml file of:
+```yaml
+a:
+  deeply:
+    nested: value
+```
+then
+```bash
+yq eval 'with(.a.deeply.nested; . = "newValue" | . style="single")' sample.yml
+```
+will output
+```yaml
+a:
+  deeply:
+    nested: 'newValue'
+```
+
+## Update multiple deeply nested properties
+Given a sample.yml file of:
+```yaml
+a:
+  deeply:
+    nested: value
+    other: thing
+```
+then
+```bash
+yq eval 'with(.a.deeply; .nested = "newValue" | .other= "newThing")' sample.yml
+```
+will output
+```yaml
+a:
+  deeply:
+    nested: newValue
+    other: newThing
+```
+
+## Update array elements relatively
+The second expression runs with each element of the array as it's contextual root. This allows you to make updates relative to the element.
+
+Given a sample.yml file of:
+```yaml
+myArray:
+  - a: apple
+  - a: banana
+```
+then
+```bash
+yq eval 'with(.myArray[]; .b = .a + " yum")' sample.yml
+```
+will output
+```yaml
+myArray:
+  - a: apple
+    b: apple yum
+  - a: banana
+    b: banana yum
+```
+

pkg/yqlib/doc/aa.md

@@ -1,5 +1,3 @@
-# Operators
-
 In `yq` expressions are made up of operators and pipes. A context of nodes is passed through the expression and each operation takes the context as input and returns a new context as output. That output is piped in as input for the next operation in the expression. To begin with, the context is set to the first yaml document of the first yaml file (if processing in sequence using eval).
 
 Lets look at a couple of examples.

pkg/yqlib/doc/headers/Main.md

@@ -0,0 +1,61 @@
+# NAME
+  *yq* is a portable command-line YAML processor
+
+# SYNOPSIS 
+
+yq [eval/eval-all] [expression] files..
+
+eval/e  - Apply the expression to each document in each yaml file in sequence
+
+eval-all/ea - Loads all yaml documents of all yaml files and runs expression once
+
+# DESCRIPTION
+
+a lightweight and portable command-line YAML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.
+
+This documentation is also available at https://mikefarah.gitbook.io/yq/
+# QUICK GUIDE 
+
+## Read a value:
+```bash
+yq e '.a.b[0].c' file.yaml
+```
+
+## Pipe from STDIN:
+```bash
+cat file.yaml | yq e '.a.b[0].c' -
+```
+
+## Update a yaml file, inplace
+```bash
+yq e -i '.a.b[0].c = "cool"' file.yaml
+```
+
+## Update using environment variables
+```bash
+NAME=mike yq e -i '.a.b[0].c = strenv(NAME)' file.yaml
+```
+
+## Merge multiple files
+```
+yq ea '. as $item ireduce ({}; . * $item )' path/to/*.yml
+```
+
+## Multiple updates to a yaml file
+```bash
+yq e -i '
+  .a.b[0].c = "cool" |
+  .x.y.z = "foobar" |
+  .person.name = strenv(NAME)
+' file.yaml
+```
+
+See the [documentation](https://mikefarah.gitbook.io/yq/) for more.
+
+# KNOWN ISSUES / MISSING FEATURES
+- `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details)
+- Powershell has its own...opinions: https://mikefarah.gitbook.io/yq/usage/tips-and-tricks#quotes-in-windows-powershell
+
+# BUGS / ISSUES / FEATURE REQUESTS
+
+Please visit the GitHub page https://github.com/mikefarah/yq/.

pkg/yqlib/doc/headers/String Operators.md

@@ -2,3 +2,43 @@
 
 ## RegEx
 This uses golangs native regex functions under the hood - See https://github.com/google/re2/wiki/Syntax for the supported syntax.
+
+
+# String blocks, bash and newlines
+Bash is notorious for chomping on precious trailing newline characters, making it tricky to set strings with newlines properly. In particular, the `$( exp )` _will trim trailing newlines_.
+
+For instance to get this yaml:
+
+```
+a: |
+  cat
+```
+
+Using `$( exp )` wont work, as it will trim the trailing new line.
+
+```
+m=$(echo "cat\n") yq e -n '.a = strenv(m)'
+a: cat
+```
+
+However, using printf works:
+```
+printf -v m "cat\n" ; m="$m" yq e -n '.a = strenv(m)'
+a: |
+  cat
+```
+
+As well as having multiline expressions:
+```
+m="cat
+"  yq e -n '.a = strenv(m)'
+a: |
+  cat
+```
+
+Similarly, if you're trying to set the content from a file, and want a trailing new line:
+
+```
+IFS= read -rd '' output < <(cat my_file)
+output=$output ./yq e '.data.values = strenv(output)' first.yml
+```

pkg/yqlib/doc/headers/Variable Operators.md

@@ -1 +1,3 @@
-For more complex scenarios, variables can be used to hold values of expression to be used in other expressions.
+Like the `jq` equivalents, variables are sometimes required for the more complex expressions (or swapping values between fields).
+
+Note that there is also an additional `ref` operator that holds a reference (instead of a copy) of the path, allowing you to make multiple changes to the same path.

pkg/yqlib/doc/headers/With.md

@@ -0,0 +1 @@
+Use the `with` operator to conveniently make multiple updates to a deeply nested path, or to update array elements relatively to each other. The first argument expression sets the root context, and the second expression runs against that root context.

pkg/yqlib/encoder_properties.go

@@ -0,0 +1,80 @@
+package yqlib
+
+import (
+	"fmt"
+	"io"
+	"strings"
+
+	"github.com/magiconair/properties"
+	yaml "gopkg.in/yaml.v3"
+)
+
+type propertiesEncoder struct {
+	destination io.Writer
+}
+
+func NewPropertiesEncoder(destination io.Writer) Encoder {
+	return &propertiesEncoder{destination}
+}
+
+func (pe *propertiesEncoder) Encode(node *yaml.Node) error {
+	mapKeysToStrings(node)
+	p := properties.NewProperties()
+	err := pe.doEncode(p, node, "")
+	if err != nil {
+		return err
+	}
+
+	_, err = p.WriteComment(pe.destination, "#", properties.UTF8)
+	return err
+}
+
+func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yaml.Node, path string) error {
+	p.SetComment(path,
+		strings.Replace(node.HeadComment, "#", "", 1)+
+			strings.Replace(node.LineComment, "#", "", 1))
+	switch node.Kind {
+	case yaml.ScalarNode:
+		_, _, err := p.Set(path, node.Value)
+		return err
+	case yaml.DocumentNode:
+		return pe.doEncode(p, node.Content[0], path)
+	case yaml.SequenceNode:
+		return pe.encodeArray(p, node.Content, path)
+	case yaml.MappingNode:
+		return pe.encodeMap(p, node.Content, path)
+	case yaml.AliasNode:
+		return pe.doEncode(p, node.Alias, path)
+	default:
+		return fmt.Errorf("Unsupported node %v", node.Tag)
+	}
+}
+
+func (pe *propertiesEncoder) appendPath(path string, key interface{}) string {
+	if path == "" {
+		return fmt.Sprintf("%v", key)
+	}
+	return fmt.Sprintf("%v.%v", path, key)
+}
+
+func (pe *propertiesEncoder) encodeArray(p *properties.Properties, kids []*yaml.Node, path string) error {
+	for index, child := range kids {
+		err := pe.doEncode(p, child, pe.appendPath(path, index))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (pe *propertiesEncoder) encodeMap(p *properties.Properties, kids []*yaml.Node, path string) error {
+	for index := 0; index < len(kids); index = index + 2 {
+		key := kids[index]
+		value := kids[index+1]
+		err := pe.doEncode(p, value, pe.appendPath(path, key.Value))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/yqlib/encoder_properties_test.go

@@ -0,0 +1,78 @@
+package yqlib
+
+import (
+	"bufio"
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+func yamlToProps(sampleYaml string) string {
+	var output bytes.Buffer
+	writer := bufio.NewWriter(&output)
+
+	var propsEncoder = NewPropertiesEncoder(writer)
+	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0)
+	if err != nil {
+		panic(err)
+	}
+	node := inputs.Front().Value.(*CandidateNode).Node
+	err = propsEncoder.Encode(node)
+	if err != nil {
+		panic(err)
+	}
+	writer.Flush()
+
+	return strings.TrimSuffix(output.String(), "\n")
+}
+
+func TestPropertiesEncoderSimple(t *testing.T) {
+	var sampleYaml = `a: 'bob cool'`
+
+	var expectedJson = `a = bob cool`
+	var actualProps = yamlToProps(sampleYaml)
+	test.AssertResult(t, expectedJson, actualProps)
+}
+
+func TestPropertiesEncoderSimpleWithComments(t *testing.T) {
+	var sampleYaml = `a: 'bob cool' # line`
+
+	var expectedJson = `# line
+a = bob cool`
+	var actualProps = yamlToProps(sampleYaml)
+	test.AssertResult(t, expectedJson, actualProps)
+}
+
+func TestPropertiesEncoderDeep(t *testing.T) {
+	var sampleYaml = `a: 
+  b: "bob cool"
+`
+
+	var expectedJson = `a.b = bob cool`
+	var actualProps = yamlToProps(sampleYaml)
+	test.AssertResult(t, expectedJson, actualProps)
+}
+
+func TestPropertiesEncoderDeepWithComments(t *testing.T) {
+	var sampleYaml = `a:  # a thing
+  b: "bob cool" # b thing
+`
+
+	var expectedJson = `# b thing
+a.b = bob cool`
+	var actualProps = yamlToProps(sampleYaml)
+	test.AssertResult(t, expectedJson, actualProps)
+}
+
+func TestPropertiesEncoderArray(t *testing.T) {
+	var sampleYaml = `a: 
+  b: [{c: dog}, {c: cat}]
+`
+
+	var expectedJson = `a.b.0.c = dog
+a.b.1.c = cat`
+	var actualProps = yamlToProps(sampleYaml)
+	test.AssertResult(t, expectedJson, actualProps)
+}

pkg/yqlib/expression_parser_test.go

@@ -6,37 +6,66 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
-func TestPathTreeNoArgsForTwoArgOp(t *testing.T) {
+func TestParserNoMatchingCloseCollect(t *testing.T) {
+	_, err := NewExpressionParser().ParseExpression("[1,2")
+	test.AssertResultComplex(t, "Bad expression, could not find matching `]`", err.Error())
+}
+func TestParserNoMatchingCloseObjectInCollect(t *testing.T) {
+	_, err := NewExpressionParser().ParseExpression(`[{"b": "c"]`)
+	test.AssertResultComplex(t, "Bad expression, could not find matching `}`", err.Error())
+}
+
+func TestParserNoMatchingCloseInCollect(t *testing.T) {
+	_, err := NewExpressionParser().ParseExpression(`[(.a]`)
+	test.AssertResultComplex(t, "Bad expression, could not find matching `)`", err.Error())
+}
+
+func TestParserNoMatchingCloseCollectObject(t *testing.T) {
+	_, err := NewExpressionParser().ParseExpression(`{"a": "b"`)
+	test.AssertResultComplex(t, "Bad expression, could not find matching `}`", err.Error())
+}
+
+func TestParserNoMatchingCloseCollectInCollectObject(t *testing.T) {
+	_, err := NewExpressionParser().ParseExpression(`{"b": [1}`)
+	test.AssertResultComplex(t, "Bad expression, could not find matching `]`", err.Error())
+}
+
+func TestParserNoMatchingCloseBracketInCollectObject(t *testing.T) {
+	_, err := NewExpressionParser().ParseExpression(`{"b": (1}`)
+	test.AssertResultComplex(t, "Bad expression, could not find matching `)`", err.Error())
+}
+
+func TestParserNoArgsForTwoArgOp(t *testing.T) {
 	_, err := NewExpressionParser().ParseExpression("=")
 	test.AssertResultComplex(t, "'=' expects 2 args but there is 0", err.Error())
 }
 
-func TestPathTreeOneLhsArgsForTwoArgOp(t *testing.T) {
+func TestParserOneLhsArgsForTwoArgOp(t *testing.T) {
 	_, err := NewExpressionParser().ParseExpression(".a =")
 	test.AssertResultComplex(t, "'=' expects 2 args but there is 1", err.Error())
 }
 
-func TestPathTreeOneRhsArgsForTwoArgOp(t *testing.T) {
+func TestParserOneRhsArgsForTwoArgOp(t *testing.T) {
 	_, err := NewExpressionParser().ParseExpression("= .a")
 	test.AssertResultComplex(t, "'=' expects 2 args but there is 1", err.Error())
 }
 
-func TestPathTreeTwoArgsForTwoArgOp(t *testing.T) {
+func TestParserTwoArgsForTwoArgOp(t *testing.T) {
 	_, err := NewExpressionParser().ParseExpression(".a = .b")
 	test.AssertResultComplex(t, nil, err)
 }
 
-func TestPathTreeNoArgsForOneArgOp(t *testing.T) {
+func TestParserNoArgsForOneArgOp(t *testing.T) {
 	_, err := NewExpressionParser().ParseExpression("explode")
 	test.AssertResultComplex(t, "'explode' expects 1 arg but received none", err.Error())
 }
 
-func TestPathTreeOneArgForOneArgOp(t *testing.T) {
+func TestParserOneArgForOneArgOp(t *testing.T) {
 	_, err := NewExpressionParser().ParseExpression("explode(.)")
 	test.AssertResultComplex(t, nil, err)
 }
 
-func TestPathTreeExtraArgs(t *testing.T) {
+func TestParserExtraArgs(t *testing.T) {
 	_, err := NewExpressionParser().ParseExpression("sortKeys(.) explode(.)")
 	test.AssertResultComplex(t, "Bad expression, please check expression syntax", err.Error())
 }

pkg/yqlib/expression_postfix.go

@@ -2,6 +2,7 @@ package yqlib
 
 import (
 	"errors"
+	"fmt"
 
 	logging "gopkg.in/op/go-logging.v1"
 )
@@ -24,6 +25,17 @@ func popOpToResult(opStack []*token, result []*Operation) ([]*token, []*Operatio
 	return opStack, append(result, newOp.Operation)
 }
 
+func validateNoOpenTokens(token *token) error {
+	if token.TokenType == openCollect {
+		return fmt.Errorf(("Bad expression, could not find matching `]`"))
+	} else if token.TokenType == openCollectObject {
+		return fmt.Errorf(("Bad expression, could not find matching `}`"))
+	} else if token.TokenType == openBracket {
+		return fmt.Errorf(("Bad expression, could not find matching `)`"))
+	}
+	return nil
+}
+
 func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Operation, error) {
 	var result []*Operation
 	// surround the whole thing with brackets
@@ -45,6 +57,10 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
 			}
 
 			for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != opener {
+				missingClosingTokenErr := validateNoOpenTokens(opStack[len(opStack)-1])
+				if missingClosingTokenErr != nil {
+					return nil, missingClosingTokenErr
+				}
 				opStack, result = popOpToResult(opStack, result)
 			}
 			if len(opStack) == 0 {
@@ -76,6 +92,11 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
 
 		case closeBracket:
 			for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != openBracket {
+				missingClosingTokenErr := validateNoOpenTokens(opStack[len(opStack)-1])
+				if missingClosingTokenErr != nil {
+					return nil, missingClosingTokenErr
+				}
+
 				opStack, result = popOpToResult(opStack, result)
 			}
 			if len(opStack) == 0 {
@@ -98,6 +119,8 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
 		}
 	}
 
+	log.Debugf("opstackLen: %v", len(opStack))
+
 	if log.IsEnabledFor(logging.DEBUG) {
 		log.Debugf("PostFix Result:")
 		for _, currentToken := range result {

pkg/yqlib/expression_processing_test.go

@@ -15,6 +15,31 @@ var pathTests = []struct {
 	expectedTokens  []interface{}
 	expectedPostFix []interface{}
 }{
+	{
+		`.["cat"].["dog"]`,
+		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "cat (string)", "]", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "dog (string)", "]"),
+		append(make([]interface{}, 0), "SELF", "cat (string)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SELF", "dog (string)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
+	},
+	{
+		`.["cat"]`,
+		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "cat (string)", "]"),
+		append(make([]interface{}, 0), "SELF", "cat (string)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+	},
+	{
+		"with(.a;.=3)",
+		append(make([]interface{}, 0), "WITH", "(", "a", "BLOCK", "SELF", "ASSIGN", "3 (int64)", ")"),
+		append(make([]interface{}, 0), "a", "SELF", "3 (int64)", "ASSIGN", "BLOCK", "WITH"),
+	},
+	{
+		"0x12",
+		append(make([]interface{}, 0), "18 (int64)"),
+		append(make([]interface{}, 0), "18 (int64)"),
+	},
+	{
+		"0X12",
+		append(make([]interface{}, 0), "18 (int64)"),
+		append(make([]interface{}, 0), "18 (int64)"),
+	},
 	{
 		".a\n",
 		append(make([]interface{}, 0), "a"),
@@ -112,8 +137,8 @@ var pathTests = []struct {
 	},
 	{
 		`.a.[]`,
-		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
-		append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+		append(make([]interface{}, 0), "a", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
+		append(make([]interface{}, 0), "a", "SELF", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
 	},
 	{
 		`.a[0]`,
@@ -127,8 +152,8 @@ var pathTests = []struct {
 	},
 	{
 		`.a.[0]`,
-		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
-		append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+		append(make([]interface{}, 0), "a", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
+		append(make([]interface{}, 0), "a", "SELF", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
 	},
 	{
 		`.a[].c`,
@@ -202,8 +227,8 @@ var pathTests = []struct {
 	},
 	{
 		`{.a: .c, .b.[]: .f.g[]}`,
-		append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "c", "UNION", "b", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "CREATE_MAP", "f", "SHORT_PIPE", "g", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "}"),
-		append(make([]interface{}, 0), "a", "c", "CREATE_MAP", "b", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "f", "g", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE", "CREATE_MAP", "UNION", "COLLECT_OBJECT", "SHORT_PIPE"),
+		append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "c", "UNION", "b", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "CREATE_MAP", "f", "SHORT_PIPE", "g", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "}"),
+		append(make([]interface{}, 0), "a", "c", "CREATE_MAP", "b", "SELF", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE", "f", "g", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE", "CREATE_MAP", "UNION", "COLLECT_OBJECT", "SHORT_PIPE"),
 	},
 	{
 		`explode(.a.b)`,

pkg/yqlib/expression_tokeniser.go

@@ -111,6 +111,7 @@ func multiplyWithPrefs() lex.Action {
 		if strings.Contains(options, "d") {
 			prefs.DeepMergeArrays = true
 		}
+		prefs.TraversePrefs.DontFollowAlias = true
 		op := &Operation{OperationType: multiplyOpType, Value: multiplyOpType.Type, StringValue: options, Preferences: prefs}
 		return &token{TokenType: operationToken, Operation: op}, nil
 	}
@@ -166,6 +167,20 @@ func numberValue() lex.Action {
 	}
 }
 
+func hexValue() lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		var originalString = string(m.Bytes)
+		var numberString = originalString[2:]
+		log.Debugf("numberString: %v", numberString)
+		var number, errParsingInt = strconv.ParseInt(numberString, 16, 64) // nolint
+		if errParsingInt != nil {
+			return nil, errParsingInt
+		}
+
+		return &token{TokenType: operationToken, Operation: createValueOperation(number, originalString)}, nil
+	}
+}
+
 func floatValue() lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		var numberString = string(m.Bytes)
@@ -284,6 +299,7 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`any_c`), opToken(anyConditionOpType))
 	lexer.Add([]byte(`all`), opToken(allOpType))
 	lexer.Add([]byte(`all_c`), opToken(allConditionOpType))
+	lexer.Add([]byte(`contains`), opToken(containsOpType))
 
 	lexer.Add([]byte(`split`), opToken(splitStringOpType))
 	lexer.Add([]byte(`keys`), opToken(keysOpType))
@@ -301,6 +317,8 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`from_entries`), opToken(fromEntriesOpType))
 	lexer.Add([]byte(`with_entries`), opToken(withEntriesOpType))
 
+	lexer.Add([]byte(`with`), opToken(withOpType))
+
 	lexer.Add([]byte(`lineComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}))
 
 	lexer.Add([]byte(`headComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{HeadComment: true}))
@@ -323,11 +341,12 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte("( |\t|\n|\r)+"), skip)
 
 	lexer.Add([]byte(`\."[^ "]+"\??`), pathToken(true))
-	lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=\n]+\??`), pathToken(false))
+	lexer.Add([]byte(`\.[^ ;\}\{\:\[\],\|\.\[\(\)=\n]+\??`), pathToken(false))
 	lexer.Add([]byte(`\.`), selfToken())
 
 	lexer.Add([]byte(`\|`), opToken(pipeOpType))
 
+	lexer.Add([]byte(`0[xX][0-9A-Fa-f]+`), hexValue())
 	lexer.Add([]byte(`-?\d+(\.\d+)`), floatValue())
 	lexer.Add([]byte(`-?[1-9](\.\d+)?[Ee][-+]?\d+`), floatValue())
 	lexer.Add([]byte(`-?\d+`), numberValue())
@@ -352,7 +371,8 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`\-`), opToken(subtractOpType))
 	lexer.Add([]byte(`\-=`), opToken(subtractAssignOpType))
 	lexer.Add([]byte(`\$[a-zA-Z_-0-9]+`), getVariableOpToken())
-	lexer.Add([]byte(`as`), opToken(assignVariableOpType))
+	lexer.Add([]byte(`as`), opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{}))
+	lexer.Add([]byte(`ref`), opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{IsReference: true}))
 
 	err := lexer.CompileNFA()
 	if err != nil {
@@ -417,17 +437,16 @@ func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postPr
 	log.Debug("processing %v", currentToken.toString(true))
 
 	if currentToken.TokenType == traverseArrayCollect {
+		// `.[exp]`` works by creating a traversal array of [self, exp] and piping that into the traverse array operator
 		//need to put a traverse array then a collect currentToken
 		// do this by adding traverse then converting currentToken to collect
 
-		if index == 0 || tokens[index-1].TokenType != operationToken ||
-			tokens[index-1].Operation.OperationType != traversePathOpType {
-			log.Debug("  adding self")
-			op := &Operation{OperationType: selfReferenceOpType, StringValue: "SELF"}
-			postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
-		}
+		log.Debug("  adding self")
+		op := &Operation{OperationType: selfReferenceOpType, StringValue: "SELF"}
+		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
+
 		log.Debug("  adding traverse array")
-		op := &Operation{OperationType: traverseArrayOpType, StringValue: "TRAVERSE_ARRAY"}
+		op = &Operation{OperationType: traverseArrayOpType, StringValue: "TRAVERSE_ARRAY"}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 
 		currentToken = &token{TokenType: openCollect}
@@ -455,20 +474,15 @@ func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postPr
 	}
 
 	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
-		tokens[index+1].TokenType == operationToken &&
-		tokens[index+1].Operation.OperationType == traversePathOpType {
+		((tokens[index+1].TokenType == operationToken && (tokens[index+1].Operation.OperationType == traversePathOpType)) ||
+			(tokens[index+1].TokenType == traverseArrayCollect)) {
 		log.Debug("  adding pipe because the next thing is traverse")
-		op := &Operation{OperationType: shortPipeOpType, Value: "PIPE"}
+		op := &Operation{OperationType: shortPipeOpType, Value: "PIPE", StringValue: "."}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
 	}
 	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
 		tokens[index+1].TokenType == openCollect {
 
-		// if tokens[index].TokenType == closeCollect {
-		// 	log.Debug("  adding pipe because next is opencollect")
-		// 	op := &Operation{OperationType: shortPipeOpType, Value: "PIPE"}
-		// 	postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
-		// }
 		log.Debug("  adding traverArray because next is opencollect")
 		op := &Operation{OperationType: traverseArrayOpType}
 		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})

pkg/yqlib/lib.go

@@ -6,6 +6,8 @@ import (
 	"bytes"
 	"container/list"
 	"fmt"
+	"strconv"
+	"strings"
 
 	logging "gopkg.in/op/go-logging.v1"
 	yaml "gopkg.in/yaml.v3"
@@ -60,6 +62,7 @@ var collectOpType = &operationType{Type: "COLLECT", NumArgs: 0, Precedence: 50,
 
 var anyOpType = &operationType{Type: "ANY", NumArgs: 0, Precedence: 50, Handler: anyOperator}
 var allOpType = &operationType{Type: "ALL", NumArgs: 0, Precedence: 50, Handler: allOperator}
+var containsOpType = &operationType{Type: "CONTAINS", NumArgs: 1, Precedence: 50, Handler: containsOperator}
 var anyConditionOpType = &operationType{Type: "ANY_CONDITION", NumArgs: 1, Precedence: 50, Handler: anyOperator}
 var allConditionOpType = &operationType{Type: "ALL_CONDITION", NumArgs: 1, Precedence: 50, Handler: allOperator}
 
@@ -67,6 +70,8 @@ var toEntriesOpType = &operationType{Type: "TO_ENTRIES", NumArgs: 0, Precedence:
 var fromEntriesOpType = &operationType{Type: "FROM_ENTRIES", NumArgs: 0, Precedence: 50, Handler: fromEntriesOperator}
 var withEntriesOpType = &operationType{Type: "WITH_ENTRIES", NumArgs: 1, Precedence: 50, Handler: withEntriesOperator}
 
+var withOpType = &operationType{Type: "WITH", NumArgs: 1, Precedence: 50, Handler: withOperator}
+
 var splitDocumentOpType = &operationType{Type: "SPLIT_DOC", NumArgs: 0, Precedence: 50, Handler: splitDocumentOperator}
 var getVariableOpType = &operationType{Type: "GET_VARIABLE", NumArgs: 0, Precedence: 55, Handler: getVariableOperator}
 var getStyleOpType = &operationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: getStyleOperator}
@@ -112,11 +117,80 @@ type Operation struct {
 	OperationType *operationType
 	Value         interface{}
 	StringValue   string
-	ValueNodes    *list.List // used for Value Path elements
+	CandidateNode *CandidateNode // used for Value Path elements
 	Preferences   interface{}
 	UpdateAssign  bool // used for assign ops, when true it means we evaluate the rhs given the lhs
 }
 
+func recurseNodeArrayEqual(lhs *yaml.Node, rhs *yaml.Node) bool {
+	if len(lhs.Content) != len(rhs.Content) {
+		return false
+	}
+
+	for index := 0; index < len(lhs.Content); index = index + 1 {
+		if !recursiveNodeEqual(lhs.Content[index], rhs.Content[index]) {
+			return false
+		}
+	}
+	return true
+}
+
+func findInArray(array *yaml.Node, item *yaml.Node) int {
+
+	for index := 0; index < len(array.Content); index = index + 1 {
+		if recursiveNodeEqual(array.Content[index], item) {
+			return index
+		}
+	}
+	return -1
+}
+
+func recurseNodeObjectEqual(lhs *yaml.Node, rhs *yaml.Node) bool {
+	if len(lhs.Content) != len(rhs.Content) {
+		return false
+	}
+
+	for index := 0; index < len(lhs.Content); index = index + 2 {
+		key := lhs.Content[index]
+		value := lhs.Content[index+1]
+
+		indexInRhs := findInArray(rhs, key)
+
+		if indexInRhs == -1 || !recursiveNodeEqual(value, rhs.Content[indexInRhs+1]) {
+			return false
+		}
+	}
+	return true
+}
+
+func recursiveNodeEqual(lhs *yaml.Node, rhs *yaml.Node) bool {
+	if lhs.Kind != rhs.Kind || lhs.Tag != rhs.Tag {
+		return false
+	} else if lhs.Tag == "!!null" {
+		return true
+
+	} else if lhs.Kind == yaml.ScalarNode {
+		return lhs.Value == rhs.Value
+	} else if lhs.Kind == yaml.SequenceNode {
+		return recurseNodeArrayEqual(lhs, rhs)
+	} else if lhs.Kind == yaml.MappingNode {
+		return recurseNodeObjectEqual(lhs, rhs)
+	}
+	return false
+
+}
+
+// yaml numbers can be hex encoded...
+func parseInt(numberString string) (string, int64, error) {
+	if strings.HasPrefix(numberString, "0x") ||
+		strings.HasPrefix(numberString, "0X") {
+		num, err := strconv.ParseInt(numberString[2:], 16, 64) // nolint
+		return "0x%X", num, err
+	}
+	num, err := strconv.ParseInt(numberString, 10, 64) // nolint
+	return "%v", num, err
+}
+
 func createScalarNode(value interface{}, stringValue string) *yaml.Node {
 	var node = &yaml.Node{Kind: yaml.ScalarNode}
 	node.Value = stringValue
@@ -138,18 +212,20 @@ func createScalarNode(value interface{}, stringValue string) *yaml.Node {
 
 func createValueOperation(value interface{}, stringValue string) *Operation {
 	var node *yaml.Node = createScalarNode(value, stringValue)
-	list := list.New()
-	list.PushBack(&CandidateNode{Node: node})
+
 	return &Operation{
 		OperationType: valueOpType,
 		Value:         value,
 		StringValue:   stringValue,
-		ValueNodes:    list,
+		CandidateNode: &CandidateNode{Node: node},
 	}
 }
 
 // debugging purposes only
 func (p *Operation) toString() string {
+	if p == nil {
+		return "OP IS NIL"
+	}
 	if p.OperationType == traversePathOpType {
 		return fmt.Sprintf("%v", p.Value)
 	} else if p.OperationType == selfReferenceOpType {
@@ -167,7 +243,7 @@ func NodesToString(collection *list.List) string {
 		return ""
 	}
 
-	result := ""
+	result := fmt.Sprintf("%v results\n", collection.Len())
 	for el := collection.Front(); el != nil; el = el.Next() {
 		result = result + "\n" + NodeToString(el.Value.(*CandidateNode))
 	}

pkg/yqlib/matchKeyString.go

@@ -9,26 +9,50 @@ func matchKey(name string, pattern string) (matched bool) {
 		log.Debug("wild!")
 		return true
 	}
-	return deepMatch([]rune(name), []rune(pattern))
+	return deepMatch(name, pattern)
 }
 
-func deepMatch(str, pattern []rune) bool {
-	for len(pattern) > 0 {
-		switch pattern[0] {
-		default:
-			if len(str) == 0 || str[0] != pattern[0] {
-				return false
+// deepMatch reports whether the name matches the pattern in linear time.
+// Source https://research.swtch.com/glob
+func deepMatch(name, pattern string) bool {
+	px := 0
+	nx := 0
+	nextPx := 0
+	nextNx := 0
+	for px < len(pattern) || nx < len(name) {
+		if px < len(pattern) {
+			c := pattern[px]
+			switch c {
+			default: // ordinary character
+				if nx < len(name) && name[nx] == c {
+					px++
+					nx++
+					continue
+				}
+			case '?': // single-character wildcard
+				if nx < len(name) {
+					px++
+					nx++
+					continue
+				}
+			case '*': // zero-or-more-character wildcard
+				// Try to match at nx.
+				// If that doesn't work out,
+				// restart at nx+1 next.
+				nextPx = px
+				nextNx = nx + 1
+				px++
+				continue
 			}
-		case '?':
-			if len(str) == 0 {
-				return false
-			}
-		case '*':
-			return deepMatch(str, pattern[1:]) ||
-				(len(str) > 0 && deepMatch(str[1:], pattern))
 		}
-		str = str[1:]
-		pattern = pattern[1:]
+		// Mismatch. Maybe restart.
+		if 0 < nextNx && nextNx <= len(name) {
+			px = nextPx
+			nx = nextNx
+			continue
+		}
+		return false
 	}
-	return len(str) == 0 && len(pattern) == 0
+	// Matched all of pattern to all of name. Success.
+	return true
 }

pkg/yqlib/matchKeyString_test.go

@@ -0,0 +1,40 @@
+package yqlib
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestDeepMatch(t *testing.T) {
+	var tests = []struct {
+		name    string
+		pattern string
+		ok      bool
+	}{
+		{"", "", true},
+		{"", "x", false},
+		{"x", "", false},
+		{"abc", "abc", true},
+		{"abc", "*", true},
+		{"abc", "*c", true},
+		{"abc", "*b", false},
+		{"abc", "a*", true},
+		{"abc", "b*", false},
+		{"a", "a*", true},
+		{"a", "*a", true},
+		{"axbxcxdxe", "a*b*c*d*e*", true},
+		{"axbxcxdxexxx", "a*b*c*d*e*", true},
+		{"abxbbxdbxebxczzx", "a*b?c*x", true},
+		{"abxbbxdbxebxczzy", "a*b?c*x", false},
+		{strings.Repeat("a", 100), "a*a*a*a*b", false},
+		{"xxx", "*x", true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name+" "+tt.pattern, func(t *testing.T) {
+			if want, got := tt.ok, deepMatch(tt.name, tt.pattern); want != got {
+				t.Errorf("Expected %v got %v", want, got)
+			}
+		})
+	}
+}

pkg/yqlib/operator_add.go

@@ -76,17 +76,17 @@ func addScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*Candida
 		target.Node.Tag = "!!str"
 		target.Node.Value = lhs.Value + rhs.Value
 	} else if lhs.Tag == "!!int" && rhs.Tag == "!!int" {
-		lhsNum, err := strconv.Atoi(lhs.Value)
+		format, lhsNum, err := parseInt(lhs.Value)
 		if err != nil {
 			return nil, err
 		}
-		rhsNum, err := strconv.Atoi(rhs.Value)
+		_, rhsNum, err := parseInt(rhs.Value)
 		if err != nil {
 			return nil, err
 		}
 		sum := lhsNum + rhsNum
 		target.Node.Tag = "!!int"
-		target.Node.Value = fmt.Sprintf("%v", sum)
+		target.Node.Value = fmt.Sprintf(format, sum)
 	} else if (lhs.Tag == "!!int" || lhs.Tag == "!!float") && (rhs.Tag == "!!int" || rhs.Tag == "!!float") {
 		lhsNum, err := strconv.ParseFloat(lhs.Value, 64)
 		if err != nil {

pkg/yqlib/operator_alternative.go

@@ -2,7 +2,7 @@ package yqlib
 
 func alternativeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- alternative")
-	return crossFunction(d, context.ReadOnlyClone(), expressionNode, alternativeFunc, true)
+	return crossFunction(d, context, expressionNode, alternativeFunc, true)
 }
 
 func alternativeFunc(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {

pkg/yqlib/operator_alternative_test.go

@@ -6,10 +6,13 @@ import (
 
 var alternativeOperatorScenarios = []expressionScenario{
 	{
+		// to match jq - we do not use a readonly clone context on the LHS.
 		skipDoc:    true,
 		expression: `.b // .c`,
 		document:   `a: bridge`,
-		expected:   []string{},
+		expected: []string{
+			"D0, P[c], (!!null)::null\n",
+		},
 	},
 	{
 		skipDoc:    true,

pkg/yqlib/operator_anchors_aliases.go

@@ -141,6 +141,48 @@ func explodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expr
 	return context, nil
 }
 
+func reconstructAliasedMap(node *yaml.Node, context Context) error {
+	var newContent = list.New()
+	// can I short cut here by prechecking if there's an anchor in the map?
+	// no it needs to recurse in overrideEntry.
+
+	for index := 0; index < len(node.Content); index = index + 2 {
+		keyNode := node.Content[index]
+		valueNode := node.Content[index+1]
+		log.Debugf("traversing %v", keyNode.Value)
+		if keyNode.Value != "<<" {
+			err := overrideEntry(node, keyNode, valueNode, index, context.ChildContext(newContent))
+			if err != nil {
+				return err
+			}
+		} else {
+			if valueNode.Kind == yaml.SequenceNode {
+				log.Debugf("an alias merge list!")
+				for index := len(valueNode.Content) - 1; index >= 0; index = index - 1 {
+					aliasNode := valueNode.Content[index]
+					err := applyAlias(node, aliasNode.Alias, index, context.ChildContext(newContent))
+					if err != nil {
+						return err
+					}
+				}
+			} else {
+				log.Debugf("an alias merge!")
+				err := applyAlias(node, valueNode.Alias, index, context.ChildContext(newContent))
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+	node.Content = make([]*yaml.Node, newContent.Len())
+	index := 0
+	for newEl := newContent.Front(); newEl != nil; newEl = newEl.Next() {
+		node.Content[index] = newEl.Value.(*yaml.Node)
+		index++
+	}
+	return nil
+}
+
 func explodeNode(node *yaml.Node, context Context) error {
 	node.Anchor = ""
 	switch node.Kind {
@@ -165,43 +207,35 @@ func explodeNode(node *yaml.Node, context Context) error {
 		}
 		return nil
 	case yaml.MappingNode:
-		var newContent = list.New()
+		// //check the map has an alias in it
+		hasAlias := false
 		for index := 0; index < len(node.Content); index = index + 2 {
 			keyNode := node.Content[index]
-			valueNode := node.Content[index+1]
-			log.Debugf("traversing %v", keyNode.Value)
-			if keyNode.Value != "<<" {
-				err := overrideEntry(node, keyNode, valueNode, index, context.ChildContext(newContent))
+			if keyNode.Value == "<<" {
+				hasAlias = true
+				break
+			}
+		}
+
+		if hasAlias {
+			// this is a slow op, which is why we want to check before running it.
+			return reconstructAliasedMap(node, context)
+		} else {
+			// this map has no aliases, but it's kids might
+			for index := 0; index < len(node.Content); index = index + 2 {
+				keyNode := node.Content[index]
+				valueNode := node.Content[index+1]
+				err := explodeNode(keyNode, context)
 				if err != nil {
 					return err
 				}
-			} else {
-				if valueNode.Kind == yaml.SequenceNode {
-					log.Debugf("an alias merge list!")
-					for index := len(valueNode.Content) - 1; index >= 0; index = index - 1 {
-						aliasNode := valueNode.Content[index]
-						err := applyAlias(node, aliasNode.Alias, index, context.ChildContext(newContent))
-						if err != nil {
-							return err
-						}
-					}
-				} else {
-					log.Debugf("an alias merge!")
-					err := applyAlias(node, valueNode.Alias, index, context.ChildContext(newContent))
-					if err != nil {
-						return err
-					}
+				err = explodeNode(valueNode, context)
+				if err != nil {
+					return err
 				}
 			}
+			return nil
 		}
-		node.Content = make([]*yaml.Node, newContent.Len())
-		index := 0
-		for newEl := newContent.Front(); newEl != nil; newEl = newEl.Next() {
-			node.Content[index] = newEl.Value.(*yaml.Node)
-			index++
-		}
-
-		return nil
 	default:
 		return nil
 	}

pkg/yqlib/operator_collect_object.go

@@ -20,8 +20,7 @@ import (
 func collectObjectOperator(d *dataTreeNavigator, originalContext Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- collectObjectOperation")
 
-	context := originalContext.Clone()
-	context.DontAutoCreate = false
+	context := originalContext.WritableClone()
 
 	if context.MatchingNodes.Len() == 0 {
 		node := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map", Value: "{}"}

pkg/yqlib/operator_contains.go

@@ -0,0 +1,111 @@
+package yqlib
+
+import (
+	"fmt"
+	"strings"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+func containsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, containsWithNodes, false)
+}
+
+func containsArrayElement(array *yaml.Node, item *yaml.Node) (bool, error) {
+	for index := 0; index < len(array.Content); index = index + 1 {
+		containedInArray, err := contains(array.Content[index], item)
+		if err != nil {
+			return false, err
+		}
+		if containedInArray {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func containsArray(lhs *yaml.Node, rhs *yaml.Node) (bool, error) {
+	if rhs.Kind != yaml.SequenceNode {
+		return containsArrayElement(lhs, rhs)
+	}
+	for index := 0; index < len(rhs.Content); index = index + 1 {
+		itemInArray, err := containsArrayElement(lhs, rhs.Content[index])
+		if err != nil {
+			return false, err
+		}
+		if !itemInArray {
+			return false, nil
+		}
+	}
+	return true, nil
+}
+
+func containsObject(lhs *yaml.Node, rhs *yaml.Node) (bool, error) {
+	if rhs.Kind != yaml.MappingNode {
+		return false, nil
+	}
+	for index := 0; index < len(rhs.Content); index = index + 2 {
+		rhsKey := rhs.Content[index]
+		rhsValue := rhs.Content[index+1]
+		log.Debugf("Looking for %v in the lhs", rhsKey.Value)
+		lhsKeyIndex := findInArray(lhs, rhsKey)
+		log.Debugf("index is %v", lhsKeyIndex)
+		if lhsKeyIndex < 0 || lhsKeyIndex%2 != 0 {
+			return false, nil
+		}
+		lhsValue := lhs.Content[lhsKeyIndex+1]
+		log.Debugf("lhsValue is %v", lhsValue.Value)
+
+		itemInArray, err := contains(lhsValue, rhsValue)
+		log.Debugf("rhsValue is %v", rhsValue.Value)
+		if err != nil {
+			return false, err
+		}
+		if !itemInArray {
+			return false, nil
+		}
+	}
+	return true, nil
+}
+
+func containsScalars(lhs *yaml.Node, rhs *yaml.Node) (bool, error) {
+	if lhs.Tag == "!!str" {
+		return strings.Contains(lhs.Value, rhs.Value), nil
+	}
+	return lhs.Value == rhs.Value, nil
+}
+
+func contains(lhs *yaml.Node, rhs *yaml.Node) (bool, error) {
+	switch lhs.Kind {
+	case yaml.MappingNode:
+		return containsObject(lhs, rhs)
+	case yaml.SequenceNode:
+		return containsArray(lhs, rhs)
+	case yaml.ScalarNode:
+		if rhs.Kind != yaml.ScalarNode || lhs.Tag != rhs.Tag {
+			return false, nil
+		}
+		if lhs.Tag == "!!null" {
+			return rhs.Tag == "!!null", nil
+		}
+		return containsScalars(lhs, rhs)
+	}
+
+	return false, fmt.Errorf("%v not yet supported for contains", lhs.Tag)
+}
+
+func containsWithNodes(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	lhs.Node = unwrapDoc(lhs.Node)
+	rhs.Node = unwrapDoc(rhs.Node)
+
+	if lhs.Node.Kind != rhs.Node.Kind {
+		return nil, fmt.Errorf("%v cannot check contained in %v", rhs.Node.Tag, lhs.Node.Tag)
+	}
+
+	result, err := contains(lhs.Node, rhs.Node)
+	if err != nil {
+		return nil, err
+	}
+
+	return createBooleanCandidate(lhs, result), nil
+}

pkg/yqlib/operator_contains_test.go

@@ -0,0 +1,82 @@
+package yqlib
+
+import "testing"
+
+var containsOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		expression: `null | contains(~)`,
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `3 | contains(3)`,
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `3 | contains(32)`,
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		description:    "Array contains array",
+		subdescription: "Array is equal or subset of",
+		document:       `["foobar", "foobaz", "blarp"]`,
+		expression:     `contains(["baz", "bar"])`,
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `["dog", "cat", "giraffe"] | contains(["camel"])`,
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		description: "Object included in array",
+		document:    `{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}`,
+		expression:  `contains({"bar": [{"barp": 12}]})`,
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		description: "Object not included in array",
+		document:    `{"foo": 12, "bar":[1,2,{"barp":12, "blip":13}]}`,
+		expression:  `contains({"foo": 12, "bar": [{"barp": 15}]})`,
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		description: "String contains substring",
+		document:    `"foobar"`,
+		expression:  `contains("bar")`,
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		description: "String equals string",
+		document:    `"meow"`,
+		expression:  `contains("meow")`,
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+}
+
+func TestContainsOperatorScenarios(t *testing.T) {
+	for _, tt := range containsOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "Contains", containsOperatorScenarios)
+}

pkg/yqlib/operator_env.go

@@ -13,8 +13,7 @@ type envOpPreferences struct {
 }
 
 func envOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	envNameNode := expressionNode.Operation.ValueNodes.Front().Value.(*CandidateNode)
-	envName := envNameNode.Node.Value
+	envName := expressionNode.Operation.CandidateNode.Node.Value
 	log.Debug("EnvOperator, env name:", envName)
 
 	rawValue := os.Getenv(envName)

pkg/yqlib/operator_multiply.go

@@ -6,6 +6,7 @@ import (
 
 	"container/list"
 
+	"github.com/jinzhu/copier"
 	yaml "gopkg.in/yaml.v3"
 )
 
@@ -20,8 +21,29 @@ func multiplyOperator(d *dataTreeNavigator, context Context, expressionNode *Exp
 	return crossFunction(d, context, expressionNode, multiply(expressionNode.Operation.Preferences.(multiplyPreferences)), false)
 }
 
+func getNewBlankNode(lhs *yaml.Node, rhs *yaml.Node) *yaml.Node {
+
+	blankNode := &yaml.Node{}
+
+	if lhs.HeadComment != "" {
+		blankNode.HeadComment = lhs.HeadComment
+	} else if rhs.HeadComment != "" {
+		blankNode.HeadComment = rhs.HeadComment
+	}
+
+	if lhs.FootComment != "" {
+		blankNode.FootComment = lhs.FootComment
+	} else if rhs.FootComment != "" {
+		blankNode.FootComment = rhs.FootComment
+	}
+
+	return blankNode
+}
+
 func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+		// need to do this before unWrapping the potential document node
+		newBlankNode := getNewBlankNode(lhs.Node, rhs.Node)
 		lhs.Node = unwrapDoc(lhs.Node)
 		rhs.Node = unwrapDoc(rhs.Node)
 		log.Debugf("Multipling LHS: %v", lhs.Node.Tag)
@@ -29,14 +51,21 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
 
 		if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode ||
 			(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
-
-			var newBlank = lhs.CreateChild(nil, &yaml.Node{})
-			log.Debugf("merge - merge lhs into blank")
-			var newThing, err = mergeObjects(d, context.WritableClone(), newBlank, lhs, multiplyPreferences{})
+			var newBlank = CandidateNode{}
+			err := copier.CopyWithOption(&newBlank, lhs, copier.Option{IgnoreEmpty: true, DeepCopy: true})
 			if err != nil {
 				return nil, err
 			}
-			return mergeObjects(d, context.WritableClone(), newThing, rhs, preferences)
+			newBlank.Node.HeadComment = newBlankNode.HeadComment
+			newBlank.Node.FootComment = newBlankNode.FootComment
+
+			// var newBlank = lhs.CreateChild(nil, newBlankNode)
+			// log.Debugf("merge - merge lhs into blank")
+			// var newThing, err = mergeObjects(d, context.WritableClone(), newBlank, lhs, multiplyPreferences{})
+			// if err != nil {
+			// 	return nil, err
+			// }
+			return mergeObjects(d, context.WritableClone(), &newBlank, rhs, preferences)
 		} else if lhs.Node.Tag == "!!int" && rhs.Node.Tag == "!!int" {
 			return multiplyIntegers(lhs, rhs)
 		} else if (lhs.Node.Tag == "!!int" || lhs.Node.Tag == "!!float") && (rhs.Node.Tag == "!!int" || rhs.Node.Tag == "!!float") {
@@ -70,15 +99,15 @@ func multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, e
 	target.Node.Style = lhs.Node.Style
 	target.Node.Tag = "!!int"
 
-	lhsNum, err := strconv.Atoi(lhs.Node.Value)
+	format, lhsNum, err := parseInt(lhs.Node.Value)
 	if err != nil {
 		return nil, err
 	}
-	rhsNum, err := strconv.Atoi(rhs.Node.Value)
+	_, rhsNum, err := parseInt(rhs.Node.Value)
 	if err != nil {
 		return nil, err
 	}
-	target.Node.Value = fmt.Sprintf("%v", lhsNum*rhsNum)
+	target.Node.Value = fmt.Sprintf(format, lhsNum*rhsNum)
 	return target, nil
 }
 
@@ -136,9 +165,7 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom
 	} else {
 		log.Debugf("merge - assignmentOp := &Operation{OperationType: assignAttributesOpType}")
 	}
-	valueNodes := list.New()
-	valueNodes.PushBack(rhs)
-	rhsOp := &Operation{OperationType: valueOpType, ValueNodes: valueNodes}
+	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs}
 
 	assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath, preferences.TraversePrefs, rhs.IsMapKey), Rhs: &ExpressionNode{Operation: rhsOp}}
 

pkg/yqlib/operator_multiply_test.go

@@ -35,6 +35,38 @@ We then need to update the first array. We will use the relative update (|=) bec
 We set the current element of the first array as $cur. Now we multiply (merge) $cur with the matching entry in $two, by passing $two through a select filter.
 `
 
+var docWithHeader = `
+# here
+
+a: apple
+`
+
+var nodeWithHeader = `
+# here
+a: apple
+`
+
+var docNoComments = `
+b: banana
+`
+
+var docWithFooter = `
+a: apple
+
+# footer
+`
+
+var nodeWithFooter = `
+a: apple
+# footer`
+
+var document = `
+a: &cat {name: cat}
+b: {name: dog}
+c: 
+  <<: *cat
+`
+
 var multiplyOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
@@ -44,6 +76,69 @@ var multiplyOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!map)::sample:\n    - &a\n    - !!merge <<: *a\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   docWithHeader,
+		document2:  docNoComments,
+		expression: `select(fi == 0) * select(fi == 1)`,
+		expected: []string{
+			"D0, P[], (!!map)::# here\na: apple\nb: banana\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   nodeWithHeader,
+		document2:  docNoComments,
+		expression: `select(fi == 0) * select(fi == 1)`,
+		expected: []string{
+			"D0, P[], (!!map)::# here\na: apple\nb: banana\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   docNoComments,
+		document2:  docWithHeader,
+		expression: `select(fi == 0) * select(fi == 1)`,
+		expected: []string{
+			"D0, P[], (!!map)::# here\nb: banana\na: apple\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   docNoComments,
+		document2:  nodeWithHeader,
+		expression: `select(fi == 0) * select(fi == 1)`,
+		expected: []string{
+			"D0, P[], (!!map)::b: banana\n# here\na: apple\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   docWithFooter,
+		document2:  docNoComments,
+		expression: `select(fi == 0) * select(fi == 1)`,
+		expected: []string{
+			"D0, P[], (!!map)::a: apple\nb: banana\n\n# footer\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   nodeWithFooter,
+		document2:  docNoComments,
+		expression: `select(fi == 0) * select(fi == 1)`,
+		expected: []string{ // not sure why there's an extra newline *shrug*
+			"D0, P[], (!!map)::a: apple\n# footer\n\nb: banana\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   docNoComments,
+		document2:  docWithFooter,
+		expression: `select(fi == 0) * select(fi == 1)`,
+		expected: []string{
+			"D0, P[], (!!map)::b: banana\na: apple\n\n# footer\n",
+		},
+	},
 	{
 		description: "Multiply integers",
 		expression:  `3 * 4`,
@@ -317,7 +412,15 @@ var multiplyOperatorScenarios = []expressionScenario{
 		document:    mergeDocSample,
 		expression:  `.foobar * .foobarList`,
 		expected: []string{
-			"D0, P[foobar], (!!map)::c: foobarList_c\n<<: [*foo, *bar]\nthing: foobar_thing\nb: foobarList_b\n",
+			"D0, P[foobar], (!!map)::c: foobarList_c\n!!merge <<: [*foo, *bar]\nthing: foobar_thing\nb: foobarList_b\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   document,
+		expression: `.b * .c`,
+		expected: []string{
+			"D0, P[b], (!!map)::{name: dog, <<: *cat}\n",
 		},
 	},
 }

pkg/yqlib/operator_reduce.go

@@ -24,7 +24,7 @@ func reduceOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 	arrayExpNode := expressionNode.Lhs.Lhs
 	array, err := d.GetMatchingNodes(context, arrayExpNode)
 
-	log.Debugf("reducing %v", NodesToString(array.MatchingNodes))
+	log.Debugf("array of %v things", array.MatchingNodes.Len())
 
 	if err != nil {
 		return Context{}, err
@@ -39,8 +39,6 @@ func reduceOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 		return Context{}, err
 	}
 
-	log.Debugf("initialised with %v", NodesToString(accum.MatchingNodes))
-
 	log.Debugf("with variable %v", variableName)
 
 	blockExp := expressionNode.Rhs.Rhs

pkg/yqlib/operator_select.go

@@ -17,22 +17,25 @@ func selectOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 			return Context{}, err
 		}
 
-		// grab the first value
-		first := rhs.MatchingNodes.Front()
+		// find any truthy node
+		var errDecoding error
+		includeResult := false
 
-		if first != nil {
-			result := first.Value.(*CandidateNode)
-			log.Debugf("result %v", NodeToString(result))
-			includeResult, errDecoding := isTruthy(result)
+		for resultEl := rhs.MatchingNodes.Front(); resultEl != nil; resultEl = resultEl.Next() {
+			result := resultEl.Value.(*CandidateNode)
+			includeResult, errDecoding = isTruthy(result)
 			log.Debugf("isTruthy %v", includeResult)
 			if errDecoding != nil {
 				return Context{}, errDecoding
 			}
-
 			if includeResult {
-				results.PushBack(candidate)
+				break
 			}
 		}
+
+		if includeResult {
+			results.PushBack(candidate)
+		}
 	}
 	return context.ChildContext(results), nil
 }

pkg/yqlib/operator_select_test.go

@@ -5,6 +5,28 @@ import (
 )
 
 var selectOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   `cat`,
+		expression: `select(false, true)`,
+		expected: []string{
+			"D0, P[], (doc)::cat\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `cat`,
+		expression: `select(true, false)`,
+		expected: []string{
+			"D0, P[], (doc)::cat\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `cat`,
+		expression: `select(false)`,
+		expected:   []string{},
+	},
 	{
 		description: "Select elements from array",
 		document:    `[cat,goat,dog]`,

pkg/yqlib/operator_style_test.go

@@ -14,10 +14,9 @@ var styleOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description:    "Update and set style of a particular node using path variables",
-		subdescription: "You can use a variable to re-use a path",
-		document:       `a: {b: thing, c: something}`,
-		expression:     `.a.b as $x | $x = "new" | $x style="double"`,
+		description: "Update and set style of a particular node using path variables",
+		document:    `a: {b: thing, c: something}`,
+		expression:  `with(.a.b ; . = "new" | . style="double")`,
 		expected: []string{
 			"D0, P[], (doc)::a: {b: \"new\", c: something}\n",
 		},

pkg/yqlib/operator_subtract.go

@@ -5,7 +5,7 @@ import (
 
 	"strconv"
 
-	yaml "gopkg.in/yaml.v3"
+	"gopkg.in/yaml.v3"
 )
 
 func createSubtractOp(lhs *ExpressionNode, rhs *ExpressionNode) *ExpressionNode {
@@ -24,6 +24,24 @@ func subtractOperator(d *dataTreeNavigator, context Context, expressionNode *Exp
 	return crossFunction(d, context.ReadOnlyClone(), expressionNode, subtract, false)
 }
 
+func subtractArray(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	newLhsArray := make([]*yaml.Node, 0)
+
+	for lindex := 0; lindex < len(lhs.Node.Content); lindex = lindex + 1 {
+		shouldInclude := true
+		for rindex := 0; rindex < len(rhs.Node.Content) && shouldInclude; rindex = rindex + 1 {
+			if recursiveNodeEqual(lhs.Node.Content[lindex], rhs.Node.Content[rindex]) {
+				shouldInclude = false
+			}
+		}
+		if shouldInclude {
+			newLhsArray = append(newLhsArray, lhs.Node.Content[lindex])
+		}
+	}
+	lhs.Node.Content = newLhsArray
+	return lhs, nil
+}
+
 func subtract(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	lhs.Node = unwrapDoc(lhs.Node)
 	rhs.Node = unwrapDoc(rhs.Node)
@@ -40,14 +58,13 @@ func subtract(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Ca
 	case yaml.MappingNode:
 		return nil, fmt.Errorf("Maps not yet supported for subtraction")
 	case yaml.SequenceNode:
-		return nil, fmt.Errorf("Sequences not yet supported for subtraction")
-		// target.Node.Kind = yaml.SequenceNode
-		// target.Node.Style = lhsNode.Style
-		// target.Node.Tag = "!!seq"
-		// target.Node.Content = append(lhsNode.Content, toNodes(rhs)...)
+		if rhs.Node.Kind != yaml.SequenceNode {
+			return nil, fmt.Errorf("%v (%v) cannot be subtracted from %v", rhs.Node.Tag, rhs.Path, lhsNode.Tag)
+		}
+		return subtractArray(d, context, lhs, rhs)
 	case yaml.ScalarNode:
 		if rhs.Node.Kind != yaml.ScalarNode {
-			return nil, fmt.Errorf("%v (%v) cannot be added to a %v", rhs.Node.Tag, rhs.Path, lhsNode.Tag)
+			return nil, fmt.Errorf("%v (%v) cannot be subtracted from %v", rhs.Node.Tag, rhs.Path, lhsNode.Tag)
 		}
 		target.Node.Kind = yaml.ScalarNode
 		target.Node.Style = lhsNode.Style
@@ -62,17 +79,17 @@ func subtractScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*Ca
 	if lhs.Tag == "!!str" {
 		return nil, fmt.Errorf("strings cannot be subtracted")
 	} else if lhs.Tag == "!!int" && rhs.Tag == "!!int" {
-		lhsNum, err := strconv.Atoi(lhs.Value)
+		format, lhsNum, err := parseInt(lhs.Value)
 		if err != nil {
 			return nil, err
 		}
-		rhsNum, err := strconv.Atoi(rhs.Value)
+		_, rhsNum, err := parseInt(rhs.Value)
 		if err != nil {
 			return nil, err
 		}
 		result := lhsNum - rhsNum
 		target.Node.Tag = "!!int"
-		target.Node.Value = fmt.Sprintf("%v", result)
+		target.Node.Value = fmt.Sprintf(format, result)
 	} else if (lhs.Tag == "!!int" || lhs.Tag == "!!float") && (rhs.Tag == "!!int" || rhs.Tag == "!!float") {
 		lhsNum, err := strconv.ParseFloat(lhs.Value, 64)
 		if err != nil {

pkg/yqlib/operator_subtract_test.go

@@ -13,6 +13,51 @@ var subtractOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{}\n",
 		},
 	},
+	{
+		description: "Array subtraction",
+		expression:  `[1,2] - [2,3]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- 1\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `[2,1,2,2] - [2,3]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- 1\n",
+		},
+	},
+	{
+		description: "Array subtraction with nested array",
+		expression:  `[[1], 1, 2] - [[1], 3]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- 1\n- 2\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `[[1], 1, [[[2]]]] - [[1], [[[3]]]]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- 1\n- - - - 2\n",
+		},
+	},
+	{
+		description:    "Array subtraction with nested object",
+		subdescription: `Note that order of the keys does not matter`,
+		document:       `[{a: b, c: d}, {a: b}]`,
+		expression:     `. - [{"c": "d", "a": "b"}]`,
+		expected: []string{
+			"D0, P[], (!!seq)::[{a: b}]\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{a: [1], c: d}, {a: [2], c: d}, {a: b}]`,
+		expression: `. - [{"c": "d", "a": [1]}]`,
+		expected: []string{
+			"D0, P[], (!!seq)::[{a: [2], c: d}, {a: b}]\n",
+		},
+	},
 	{
 		description:    "Number subtraction - float",
 		subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.",

pkg/yqlib/operator_union.go

@@ -1,17 +1,36 @@
 package yqlib
 
 func unionOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	log.Debug("unionOperator")
+	log.Debug("context: %v", NodesToString(context.MatchingNodes))
 	lhs, err := d.GetMatchingNodes(context, expressionNode.Lhs)
 	if err != nil {
 		return Context{}, err
 	}
+	log.Debug("lhs: %v", NodesToString(lhs.MatchingNodes))
+	log.Debug("rhs input: %v", NodesToString(context.MatchingNodes))
+	log.Debug("rhs: %v", expressionNode.Rhs.Operation.toString())
 	rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
+
 	if err != nil {
 		return Context{}, err
 	}
-	for el := rhs.MatchingNodes.Front(); el != nil; el = el.Next() {
-		node := el.Value.(*CandidateNode)
-		lhs.MatchingNodes.PushBack(node)
+	log.Debug("lhs: %v", lhs.ToString())
+	log.Debug("rhs: %v", rhs.ToString())
+
+	// this can happen when both expressions modify the context
+	// instead of creating their own.
+	/// (.foo = "bar"), (.thing = "cat")
+
+	if rhs.MatchingNodes != lhs.MatchingNodes {
+
+		for el := rhs.MatchingNodes.Front(); el != nil; el = el.Next() {
+			node := el.Value.(*CandidateNode)
+			log.Debug("processing %v", NodeToString(node))
+
+			lhs.MatchingNodes.PushBack(node)
+		}
 	}
+	log.Debug("all together: %v", lhs.ToString())
 	return lhs, nil
 }

pkg/yqlib/operator_union_test.go

@@ -5,20 +5,21 @@ import (
 )
 
 var unionOperatorScenarios = []expressionScenario{
-	// {
-	// 	skipDoc:    true,
-	// 	document:   "{}",
-	// 	expression: `(.a, .b.c) as $x`,
-	// 	expected: []string{
-	// 		"D0, P[], (doc)::{}\n",
-	// 	},
-	// },
-	// {
-	// 	skipDoc:    true,
-	// 	document:   "{}",
-	// 	expression: `(.a, .b.c)`,
-	// 	expected:   []string{},
-	// },
+	{
+		skipDoc:    true,
+		document:   "{}",
+		expression: `(.a, .b.c) as $x`,
+		expected: []string{
+			"D0, P[], (doc)::{}\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `(.foo = "bar"), (.toe = "jam")`,
+		expected: []string{
+			"D0, P[], ()::foo: bar\ntoe: jam\n",
+		},
+	},
 	{
 		description: "Combine scalars",
 		expression:  `1, true, "cat"`,

pkg/yqlib/operator_value.go

@@ -1,5 +1,6 @@
 package yqlib
 
 func valueOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	return context.ChildContext(expressionNode.Operation.ValueNodes), nil
+	log.Debug("value = %v", expressionNode.Operation.CandidateNode.Node.Value)
+	return context.SingleChildContext(expressionNode.Operation.CandidateNode), nil
 }

pkg/yqlib/operator_value_test.go

@@ -12,6 +12,76 @@ var valueOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!int)::1\n",
 		},
 	},
+	{
+		document:   ``,
+		expression: `0x9f`,
+		expected: []string{
+			"D0, P[], (!!int)::0x9f\n",
+		},
+	},
+	{
+		document:   ``,
+		expression: `0x1A`,
+		expected: []string{
+			"D0, P[], (!!int)::0x1A\n",
+		},
+	},
+	{
+		document:   ``,
+		expression: `0x1A + 2`,
+		expected: []string{
+			"D0, P[], (!!int)::0x1C\n",
+		},
+	},
+	{
+		document:   ``,
+		expression: `0x12 * 2`,
+		expected: []string{
+			"D0, P[], (!!int)::0x24\n",
+		},
+	},
+	{
+		document:   ``,
+		expression: `0xF - 1`,
+		expected: []string{
+			"D0, P[], (!!int)::0xE\n",
+		},
+	},
+	{
+		document:   ``,
+		expression: `12`,
+		expected: []string{
+			"D0, P[], (!!int)::12\n",
+		},
+	},
+	{
+		document:   ``,
+		expression: `12 + 2`,
+		expected: []string{
+			"D0, P[], (!!int)::14\n",
+		},
+	},
+	{
+		document:   ``,
+		expression: `12 * 2`,
+		expected: []string{
+			"D0, P[], (!!int)::24\n",
+		},
+	},
+	{
+		document:   ``,
+		expression: `12 - 2`,
+		expected: []string{
+			"D0, P[], (!!int)::10\n",
+		},
+	},
+	{
+		document:   ``,
+		expression: `0X12`,
+		expected: []string{
+			"D0, P[], (!!int)::0X12\n",
+		},
+	},
 	{
 		document:   ``,
 		expression: `-1`,

pkg/yqlib/operator_variables.go

@@ -15,6 +15,10 @@ func getVariableOperator(d *dataTreeNavigator, context Context, expressionNode *
 	return context.ChildContext(result), nil
 }
 
+type assignVarPreferences struct {
+	IsReference bool
+}
+
 func assignVariableOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	lhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Lhs)
 	if err != nil {
@@ -24,6 +28,15 @@ func assignVariableOperator(d *dataTreeNavigator, context Context, expressionNod
 		return Context{}, fmt.Errorf("RHS of 'as' operator must be a variable name e.g. $foo")
 	}
 	variableName := expressionNode.Rhs.Operation.StringValue
-	context.SetVariable(variableName, lhs.MatchingNodes)
+
+	prefs := expressionNode.Operation.Preferences.(assignVarPreferences)
+
+	var variableValue *list.List
+	if prefs.IsReference {
+		variableValue = lhs.MatchingNodes
+	} else {
+		variableValue = lhs.DeepClone().MatchingNodes
+	}
+	context.SetVariable(variableName, variableValue)
 	return context, nil
 }

pkg/yqlib/operator_variables_test.go

@@ -34,15 +34,32 @@ var variableOperatorScenarios = []expressionScenario{
 		description:    "Using variables as a lookup",
 		subdescription: "Example taken from [jq](https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|...)",
 		document: `{"posts": [{"title": "Frist psot", "author": "anon"},
-		{"title": "A well-written article", "author": "person1"}],
-"realnames": {"anon": "Anonymous Coward",
-				"person1": "Person McPherson"}}`,
+			{"title": "A well-written article", "author": "person1"}],
+	"realnames": {"anon": "Anonymous Coward",
+					"person1": "Person McPherson"}}`,
 		expression: `.realnames as $names | .posts[] | {"title":.title, "author": $names[.author]}`,
 		expected: []string{
 			"D0, P[], (!!map)::title: \"Frist psot\"\nauthor: \"Anonymous Coward\"\n",
 			"D0, P[], (!!map)::title: \"A well-written article\"\nauthor: \"Person McPherson\"\n",
 		},
 	},
+	{
+		description: "Using variables to swap values",
+		document:    "a: a_value\nb: b_value",
+		expression:  `.a as $x  | .b as $y | .b = $x | .a = $y`,
+		expected: []string{
+			"D0, P[], (doc)::a: b_value\nb: a_value\n",
+		},
+	},
+	{
+		description:    "Use ref to reference a path repeatedly",
+		subdescription: "Note: You may find the `with` operator more useful.",
+		document:       `a: {b: thing, c: something}`,
+		expression:     `.a.b ref $x | $x = "new" | $x style="double"`,
+		expected: []string{
+			"D0, P[], (doc)::a: {b: \"new\", c: something}\n",
+		},
+	},
 }
 
 func TestVariableOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_with.go

@@ -0,0 +1,30 @@
+package yqlib
+
+import "fmt"
+
+func withOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	log.Debugf("-- withOperator")
+	// with(path, exp)
+
+	if expressionNode.Rhs.Operation.OperationType != blockOpType {
+		return Context{}, fmt.Errorf("with must be given a block, got %v instead", expressionNode.Rhs.Operation.OperationType.Type)
+	}
+
+	pathExp := expressionNode.Rhs.Lhs
+
+	updateContext, err := d.GetMatchingNodes(context, pathExp)
+
+	if err != nil {
+		return Context{}, err
+	}
+
+	updateExp := expressionNode.Rhs.Rhs
+
+	_, err = d.GetMatchingNodes(updateContext, updateExp)
+	if err != nil {
+		return Context{}, err
+	}
+
+	return context, nil
+
+}

pkg/yqlib/operator_with_test.go

@@ -0,0 +1,38 @@
+package yqlib
+
+import "testing"
+
+var withOperatorScenarios = []expressionScenario{
+	{
+		description: "Update and style",
+		document:    `a: {deeply: {nested: value}}`,
+		expression:  `with(.a.deeply.nested; . = "newValue" | . style="single")`,
+		expected: []string{
+			"D0, P[], (doc)::a: {deeply: {nested: 'newValue'}}\n",
+		},
+	},
+	{
+		description: "Update multiple deeply nested properties",
+		document:    `a: {deeply: {nested: value, other: thing}}`,
+		expression:  `with(.a.deeply; .nested = "newValue" | .other= "newThing")`,
+		expected: []string{
+			"D0, P[], (doc)::a: {deeply: {nested: newValue, other: newThing}}\n",
+		},
+	},
+	{
+		description:    "Update array elements relatively",
+		subdescription: "The second expression runs with each element of the array as it's contextual root. This allows you to make updates relative to the element.",
+		document:       `myArray: [{a: apple},{a: banana}]`,
+		expression:     `with(.myArray[]; .b = .a + " yum")`,
+		expected: []string{
+			"D0, P[], (doc)::myArray: [{a: apple, b: apple yum}, {a: banana, b: banana yum}]\n",
+		},
+	},
+}
+
+func TestWithOperatorScenarios(t *testing.T) {
+	for _, tt := range withOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "With", withOperatorScenarios)
+}

pkg/yqlib/operators.go

@@ -23,9 +23,7 @@ func compoundAssignFunction(d *dataTreeNavigator, context Context, expressionNod
 
 	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
-		valueNodes := list.New()
-		valueNodes.PushBack(candidate)
-		valueOp.ValueNodes = valueNodes
+		valueOp.CandidateNode = candidate
 		valueExpression := &ExpressionNode{Operation: valueOp}
 
 		assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: valueExpression, Rhs: calculation(valueExpression, expressionNode.Rhs)}
@@ -85,7 +83,7 @@ func doCrossFunc(d *dataTreeNavigator, context Context, expressionNode *Expressi
 	if err != nil {
 		return Context{}, err
 	}
-	log.Debugf("crossFunction LHS %v", NodesToString(lhs.MatchingNodes))
+	log.Debugf("crossFunction LHS len: %v", lhs.MatchingNodes.Len())
 
 	rhs, err := d.GetMatchingNodes(context, expressionNode.Rhs)
 

pkg/yqlib/operators_test.go

@@ -107,7 +107,7 @@ func copyFromHeader(title string, out *os.File) error {
 
 func formatYaml(yaml string, filename string) string {
 	var output bytes.Buffer
-	printer := NewPrinter(bufio.NewWriter(&output), false, true, false, 2, true)
+	printer := NewPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(".. style= \"\"")
 	if err != nil {
@@ -216,7 +216,7 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
 func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formattedDoc string, formattedDoc2 string) {
 	var output bytes.Buffer
 	var err error
-	printer := NewPrinter(bufio.NewWriter(&output), false, true, false, 2, true)
+	printer := NewPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(s.expression)
 	if err != nil {

pkg/yqlib/printer.go

@@ -3,6 +3,7 @@ package yqlib
 import (
 	"bufio"
 	"container/list"
+	"fmt"
 	"io"
 	"strings"
 
@@ -16,8 +17,29 @@ type Printer interface {
 	SetAppendix(reader io.Reader)
 }
 
+type PrinterOutputFormat uint32
+
+const (
+	YamlOutputFormat = 1 << iota
+	JsonOutputFormat
+	PropsOutputFormat
+)
+
+func OutputFormatFromString(format string) (PrinterOutputFormat, error) {
+	switch format {
+	case "yaml", "y":
+		return YamlOutputFormat, nil
+	case "json", "j":
+		return JsonOutputFormat, nil
+	case "props", "p":
+		return PropsOutputFormat, nil
+	default:
+		return 0, fmt.Errorf("Unknown fromat '%v' please use [yaml|json|props]", format)
+	}
+}
+
 type resultsPrinter struct {
-	outputToJSON       bool
+	outputFormat       PrinterOutputFormat
 	unwrapScalar       bool
 	colorsEnabled      bool
 	indent             int
@@ -31,14 +53,14 @@ type resultsPrinter struct {
 	appendixReader     io.Reader
 }
 
-func NewPrinter(writer io.Writer, outputToJSON bool, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
+func NewPrinter(writer io.Writer, outputFormat PrinterOutputFormat, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer {
 	return &resultsPrinter{
 		writer:             writer,
-		outputToJSON:       outputToJSON,
+		outputFormat:       outputFormat,
 		unwrapScalar:       unwrapScalar,
 		colorsEnabled:      colorsEnabled,
 		indent:             indent,
-		printDocSeparators: !outputToJSON && printDocSeparators,
+		printDocSeparators: outputFormat == YamlOutputFormat && printDocSeparators,
 		firstTimePrinting:  true,
 		treeNavigator:      NewDataTreeNavigator(),
 	}
@@ -57,11 +79,14 @@ func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error {
 		(node.Tag != "!!bool" || node.Value != "false"))
 
 	var encoder Encoder
-	if node.Kind == yaml.ScalarNode && p.unwrapScalar && !p.outputToJSON {
+	if node.Kind == yaml.ScalarNode && p.unwrapScalar && p.outputFormat == YamlOutputFormat {
 		return p.writeString(writer, node.Value+"\n")
 	}
-	if p.outputToJSON {
+
+	if p.outputFormat == JsonOutputFormat {
 		encoder = NewJsonEncoder(writer, p.indent)
+	} else if p.outputFormat == PropsOutputFormat {
+		encoder = NewPropertiesEncoder(writer)
 	} else {
 		encoder = NewYamlEncoder(writer, p.indent, p.colorsEnabled)
 	}
@@ -82,7 +107,7 @@ func (p *resultsPrinter) safelyFlush(writer *bufio.Writer) {
 
 func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 	log.Debug("PrintResults for %v matches", matchingNodes.Len())
-	if p.outputToJSON {
+	if p.outputFormat != YamlOutputFormat {
 		explodeOp := Operation{OperationType: explodeOpType}
 		explodeNode := ExpressionNode{Operation: &explodeOp}
 		context, err := p.treeNavigator.GetMatchingNodes(Context{MatchingNodes: matchingNodes}, &explodeNode)
@@ -140,7 +165,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 							return err
 						}
 					}
-				} else if !p.outputToJSON {
+				} else if p.outputFormat == YamlOutputFormat {
 					if err := p.writeString(bufferedWriter, readline); err != nil {
 						return err
 					}
@@ -158,6 +183,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 			}
 
 		}
+
 		if err := p.printNode(mappedDoc.Node, bufferedWriter); err != nil {
 			return err
 		}
@@ -165,7 +191,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
 		p.previousDocIndex = mappedDoc.Document
 	}
 
-	if p.appendixReader != nil && !p.outputToJSON {
+	if p.appendixReader != nil && p.outputFormat == YamlOutputFormat {
 		log.Debug("Piping appendix reader...")
 		betterReader := bufio.NewReader(p.appendixReader)
 		_, err := io.Copy(bufferedWriter, betterReader)

pkg/yqlib/printer_test.go

@@ -36,7 +36,7 @@ func nodeToList(candidate *CandidateNode) *list.List {
 func TestPrinterMultipleDocsInSequence(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, false, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -74,7 +74,7 @@ func TestPrinterMultipleDocsInSequence(t *testing.T) {
 func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, false, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -116,7 +116,7 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) {
 func TestPrinterMultipleFilesInSequence(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, false, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -163,7 +163,7 @@ func TestPrinterMultipleFilesInSequence(t *testing.T) {
 func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, false, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -213,7 +213,7 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) {
 func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, false, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -232,7 +232,7 @@ func TestPrinterMultipleDocsInSinglePrint(t *testing.T) {
 func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, false, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -261,7 +261,7 @@ a: coconut
 func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, false, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {
@@ -287,7 +287,7 @@ a: coconut
 func TestPrinterScalarWithLeadingCont(t *testing.T) {
 	var output bytes.Buffer
 	var writer = bufio.NewWriter(&output)
-	printer := NewPrinter(writer, false, true, false, 2, true)
+	printer := NewPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
 	node, err := NewExpressionParser().ParseExpression(".a")
 	if err != nil {
@@ -314,7 +314,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	// note printDocSeparators is true, it should still not print document separators
 	// when outputing JSON.
-	printer := NewPrinter(writer, true, true, false, 0, true)
+	printer := NewPrinter(writer, JsonOutputFormat, true, false, 0, true)
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0)
 	if err != nil {

pkg/yqlib/reduce_evaluator.go

@@ -1,142 +0,0 @@
-package yqlib
-
-import (
-	"container/list"
-	"fmt"
-	"io"
-	"os"
-
-	yaml "gopkg.in/yaml.v3"
-)
-
-type ReduceEvaluator interface {
-	EvaluateFiles(reduceExpression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error
-}
-
-type reduceEvaluator struct {
-	treeNavigator DataTreeNavigator
-	treeCreator   ExpressionParser
-	reduceLhs     *ExpressionNode
-	fileIndex     int
-}
-
-func NewReduceEvaluator() ReduceEvaluator {
-	treeCreator := NewExpressionParser()
-	reduceLhs, err := treeCreator.ParseExpression(". as $doc")
-	if err != nil {
-		panic(err)
-	}
-	return &reduceEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: treeCreator, reduceLhs: reduceLhs}
-}
-
-func (r *reduceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error {
-
-	node, err := r.treeCreator.ParseExpression(expression)
-	if err != nil {
-		return err
-	}
-	log.Debug("node %v", node.Operation.toString())
-
-	if node.Operation.OperationType != blockOpType {
-		return fmt.Errorf("Invalid reduce expression - expected '<initialValue>; <block that uses $doc>' got '%v'", expression)
-	}
-
-	currentValue := node.Lhs
-	reduceExp := node.Rhs
-	firstLeadingContent := ""
-
-	log.Debug("initialValue %v", currentValue.Operation.toString())
-
-	log.Debug("reduce Exp %v", reduceExp.Operation.toString())
-
-	for index, filename := range filenames {
-		reader, leadingContent, err := readStream(filename, leadingContentPreProcessing)
-
-		if index == 0 {
-			firstLeadingContent = leadingContent
-		}
-
-		if err != nil {
-			return err
-		}
-
-		currentValue, err = r.ReduceFile(filename, leadingContent, reader, currentValue, reduceExp)
-		if err != nil {
-			return err
-		}
-
-		switch reader := reader.(type) {
-		case *os.File:
-			safelyCloseFile(reader)
-		}
-	}
-
-	result := currentValue.Operation.ValueNodes
-
-	if result.Len() > 0 {
-		result.Front().Value.(*CandidateNode).Node.HeadComment = firstLeadingContent
-	}
-
-	printer.PrintResults(result)
-
-	return nil
-}
-
-func (r *reduceEvaluator) createReduceOp(initialValue *ExpressionNode, reduceExp *ExpressionNode) *ExpressionNode {
-	reduceBlock := &ExpressionNode{
-		Operation: &Operation{OperationType: blockOpType},
-		Lhs:       initialValue,
-		Rhs:       reduceExp,
-	}
-
-	return &ExpressionNode{
-		Operation: &Operation{OperationType: reduceOpType},
-		Lhs:       r.reduceLhs,
-		Rhs:       reduceBlock,
-	}
-}
-
-func (r *reduceEvaluator) ReduceFile(filename string, leadingContent string, reader io.Reader, initialValue *ExpressionNode, reduceExp *ExpressionNode) (*ExpressionNode, error) {
-
-	var currentIndex uint
-	var currentValue = initialValue
-	decoder := yaml.NewDecoder(reader)
-	for {
-		var dataBucket yaml.Node
-		errorReading := decoder.Decode(&dataBucket)
-
-		if errorReading == io.EOF {
-			r.fileIndex = r.fileIndex + 1
-			return currentValue, nil
-		} else if errorReading != nil {
-			return currentValue, errorReading
-		}
-		candidateNode := &CandidateNode{
-			Document:  currentIndex,
-			Filename:  filename,
-			Node:      &dataBucket,
-			FileIndex: r.fileIndex,
-		}
-		inputList := list.New()
-		inputList.PushBack(candidateNode)
-
-		reduceOp := r.createReduceOp(currentValue, reduceExp)
-		// log.Debug("reduce - currentValueBefore: %v", NodesToString(currentValue.Operation.ValueNodes))
-
-		result, errorParsing := r.treeNavigator.GetMatchingNodes(Context{MatchingNodes: inputList}, reduceOp)
-		if errorParsing != nil {
-			return currentValue, errorParsing
-		}
-
-		currentValue = &ExpressionNode{
-			Operation: &Operation{
-				OperationType: valueOpType,
-				ValueNodes:    result.MatchingNodes,
-			},
-		}
-
-		log.Debug("reduce - currentValueAfter: %v", NodesToString(currentValue.Operation.ValueNodes))
-
-		currentIndex = currentIndex + 1
-	}
-}

release_instructions.txt

@@ -1,3 +1,4 @@
+- update release notes
 - increment version in version.go
 - increment version in snapcraft.yaml
 - increment version in github-action/Dockerfile

release_notes.txt

@@ -0,0 +1,46 @@
+4.13.4:
+  - Fixed select bug (#958)
+  - Improved performance of `explode` (this will also speed up json conversion)
+  - Improved performance of `merge` (significantly if your merging a small file into a big one)
+
+4.13.3:
+
+- Updated go compiler to 1.17 to fix CVE (#944)
+
+4.13.2:
+
+- Fixing Docker build timeout issues when attempting to release
+
+4.13.1:
+
+- Update to `with` operator, allow for no leading space on the `;`.
+
+4.13.0:
+
+BREAKING CHANGE - the `as` variable operator (e.g. `.a as $x`) now makes a _copy_ of the node(s) at the 
+path rather than a reference. This is in order to make it work more like the `jq` equivalent. 
+
+This means any updates made to that variable do not update the original.
+
+There's a new operator `ref` that will make a reference (and allow multiple updates to the original path by referencing the variable).
+Sorry for any inconvenience caused!.
+
+
+- New `with` operator for making multiple changes to a given path
+- New `contains` operator, works like the `jq` equivalent
+- Subtract operator now supports subtracting elements from arrays!
+- Fixed Swapping values using variables #934
+- Github Action now properly supports multiline output #936, thanks @pjxiao
+- Fixed missing closing bracket validation #932
+
+4.12.2:
+- Fix processing of hex numbers #929
+- Fixed alternative and union operator issues #930
+
+4.12.1:
+ - Merge comment fix #919
+
+4.12.0:
+- Can now convert yaml to properties properties format (`-o=props`), See [docs](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties) for more info.
+- Fixed document header/footer comment handling when merging (https://github.com/mikefarah/yq/issues/919)
+- pretty print yaml 1.1 compatibility  (https://github.com/mikefarah/yq/issues/914)

scripts/devtools.sh

@@ -2,4 +2,4 @@
 set -ex
 go get golang.org/x/tools/cmd/goimports
 wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.37.1
-#  wget -O- -nv https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.6.1
+#  wget -O- -nv https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.8.1

scripts/generate-man-page-md.sh

@@ -0,0 +1,19 @@
+#! /bin/bash
+set -e
+
+# note that this reqires pandoc to be installed.
+
+cat ./pkg/yqlib/doc/headers/Main.md > man.md
+printf "\n# HOW IT WORKS\n" >> man.md
+cat ./pkg/yqlib/doc/aa.md >> man.md
+
+for f in ./pkg/yqlib/doc/*.md; do 
+  docNameWithExt="${f##*/}"
+  docName="${docNameWithExt%.*}"
+  docNameCap=$(echo $docName | tr [a-z] [A-Z])
+  if [ "$docName" != "aa" ]; then
+    printf "\n\n# ${docNameCap}\n" >> man.md
+    cat "$f" >> man.md
+  fi
+  
+done

scripts/generate-man-page.sh

@@ -0,0 +1,11 @@
+#! /bin/bash
+set -e
+
+# note that this reqires pandoc to be installed.
+
+pandoc \
+  --variable=title:"YQ" \
+  --variable=section:"1" \
+  --variable=author:"Mike Farah" \
+  --variable=header:"${MAN_HEADER}" \
+  --standalone --to man man.md -o yq.1

scripts/install-man-page.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+cp yq.1 /usr/local/share/man/man1/.

scripts/xcompile.sh

@@ -8,7 +8,8 @@ CGO_ENABLED=0 gox -ldflags "${LDFLAGS}" -output="build/yq_{{.OS}}_{{.Arch}}" --o
 
 cd build
 
-find . -executable -type f | xargs -I {} tar czvf {}.tar.gz {}
+find . -executable -type f | xargs -I {} tar czvf {}.tar.gz {} yq.1 -C ../scripts install-man-page.sh
+tar czvf yq_man_page_only.tar.gz yq.1 -C ../scripts install-man-page.sh
 
 rhash -r -a . -o checksums
 
@@ -17,6 +18,7 @@ rhash --list-hashes > checksums_hashes_order
 # just in case find thinks this is executable...
 rm -f checksums_hashes_order.tar.gz 
 rm -f checksums.tar.gz
+rm yq.1 
 
 rm yq_windows_386.exe.tar.gz
 rm yq_windows_amd64.exe.tar.gz

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.11.1'
+version: '4.13.4'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.