Using new lexer by alecthomas!

* adding support for --wrapScalar=false in properties encoder

* altering encoder and decoder tests somewhat

* adding .idea

* Revert "altering encoder and decoder tests somewhat"

This reverts commit e3655130e2.

* adding test scenario for encoding with wrapped scalars

.github/workflows/codeql.yml

@@ -0,0 +1,70 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '24 3 * * 1'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2

.github/workflows/docker-release.yml

@@ -0,0 +1,62 @@
+name: Release Docker
+
+on:
+  release:
+    types: [released]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  publishDocker:
+    environment: dockerhub
+    env:
+      IMAGE_NAME: mikefarah/yq
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: all
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          version: latest
+
+      - name: Available platforms
+        run: echo ${{ steps.buildx.outputs.platforms }} && docker version
+
+      - name: Build and push image
+        run: |
+          echo "GithubRef: ${GITHUB_REF}"
+          VERSION=${GITHUB_REF##*/}
+          echo "VERSION: ${VERSION}"
+          IMAGE_VERSION=${VERSION:1}
+          echo "IMAGE_VERSION: ${IMAGE_VERSION}"
+
+          PLATFORMS="linux/amd64,linux/ppc64le,linux/arm64"
+
+          echo "Building and pushing version ${IMAGE_VERSION} of image ${IMAGE_NAME}"
+          echo '${{ secrets.DOCKER_PASSWORD }}' | docker login -u '${{ secrets.DOCKER_USERNAME }}' --password-stdin
+          docker buildx build \
+            --label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
+            --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
+            --label "org.opencontainers.image.description=yq is a portable command-line YAML processor" \
+            --label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
+            --label "org.opencontainers.image.licenses=MIT" \
+            --label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \
+            --label "org.opencontainers.image.source=https://github.com/mikefarah/yq" \
+            --label "org.opencontainers.image.title=yq" \
+            --label "org.opencontainers.image.url=https://mikefarah.gitbook.io/yq/" \
+            --label "org.opencontainers.image.version=${IMAGE_VERSION}" \
+            --platform "${PLATFORMS}" \
+            --pull \
+            --push \
+            -t "${IMAGE_NAME}:${IMAGE_VERSION}" \
+            -t "${IMAGE_NAME}:4" \
+            -t "${IMAGE_NAME}:latest" \
+            .

.github/workflows/go.yml

@@ -1,5 +1,8 @@
 name: Build
 on: [push, pull_request]
+permissions:
+  contents: read
+
 jobs:
 
   build:
@@ -7,10 +10,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.17
-      uses: actions/setup-go@v2
+    - name: Set up Go 1.18
+      uses: actions/setup-go@v3
       with:
-        go-version: 1.17
+        go-version: 1.18
       id: go
 
     - name: Check out code into the Go module directory

.github/workflows/release.yml

@@ -3,15 +3,16 @@ on:
   push:
     tags:
       - 'v*'
+      - 'draft-*'
 
 jobs:
   publishGitRelease:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
-          go-version: '^1.17.5'
+          go-version: '^1.18'
       - name: Compile man page markup
         id: gen-man-page-md
         run: |
@@ -39,72 +40,14 @@ jobs:
       - name: Cross compile
         run: |
           sudo apt-get install rhash -y
-          go get github.com/mitchellh/gox
+          go install github.com/mitchellh/gox@v1.0.1
           mkdir -p build
           cp yq.1 build/yq.1
           ./scripts/xcompile.sh
 
-      - name: Create Release
-        id: create_release
-        uses: actions/create-release@v1.1.4
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Release
+        uses: softprops/action-gh-release@v1
         with:
-          tag_name: ${{ github.ref }}
-          release_name: ${{ github.ref }}
+          files: build/*
           draft: true
-          prerelease: false
-
-      - uses: shogo82148/actions-upload-release-asset@v1
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: build/*
-
-  publishDocker:
-    environment: dockerhub
-    env:
-      IMAGE_NAME: mikefarah/yq
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-        with:
-          platforms: all
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          version: latest
-
-      - name: Available platforms
-        run: echo ${{ steps.buildx.outputs.platforms }} && docker version
-
-      - name: Build and push image
-        run: |
-          IMAGE_V_VERSION="$(git describe --tags --abbrev=0)"
-          IMAGE_VERSION=${IMAGE_V_VERSION:1}
-          PLATFORMS="linux/amd64,linux/ppc64le,linux/arm64"
-
-          echo "Building and pushing version ${IMAGE_VERSION} of image ${IMAGE_NAME}"
-          echo '${{ secrets.DOCKER_PASSWORD }}' | docker login -u '${{ secrets.DOCKER_USERNAME }}' --password-stdin
-          docker buildx build \
-            --label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
-            --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
-            --label "org.opencontainers.image.description=yq is a portable command-line YAML processor" \
-            --label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
-            --label "org.opencontainers.image.licenses=MIT" \
-            --label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \
-            --label "org.opencontainers.image.source=https://github.com/mikefarah/yq" \
-            --label "org.opencontainers.image.title=yq" \
-            --label "org.opencontainers.image.url=https://mikefarah.gitbook.io/yq/" \
-            --label "org.opencontainers.image.version=${IMAGE_VERSION}" \
-            --platform "${PLATFORMS}" \
-            --pull \
-            --push \
-            -t "${IMAGE_NAME}:${IMAGE_VERSION}" \
-            -t "${IMAGE_NAME}:4" \
-            -t "${IMAGE_NAME}:latest" \
-            .
+          fail_on_unmatched_files: true

.github/workflows/test-yq.yml

@@ -1,5 +1,8 @@
 name: Test Yq Action
 on: [push]
+permissions:
+  contents: read
+
 jobs:
 
   build:

.gitignore

@@ -41,6 +41,7 @@ yq*.snap
 
 test.yml
 test*.yml
+test*.yaml
 0.yml
 1.yml
 2.yml
@@ -53,3 +54,6 @@ yq.1
 # debian pkg
 _build
 debian/files
+
+# intellij
+/.idea

CONTRIBUTING.md

@@ -1,3 +1,5 @@
+# Development
+
 1. Install (golang)[https://golang.org/]
 1. Run  `scripts/devtools.sh` to install the required devtools
 2. Run `make [local] vendor` to install the vendor dependencies
@@ -6,3 +8,33 @@
 4. Make the code changes.
 5. `make [local] test` to lint code and run tests
 6. Profit! ok no profit, but raise a PR and get kudos :)
+
+
+# Documentation
+
+The documentation is a bit of a mixed bag (sorry in advanced, I do plan on simplifying it...) - with some parts automatically generated and stiched together and some statically defined.
+
+Documentation is written in markdown, and is published in the 'gitbook' branch.
+
+The various operator documentation (e.g. 'strings') are generated from the 'master' branch, and have a statically defined header (e.g. `pkg/yqlib/doc/operators/headers/add.md`) and the bulk of the docs are generated from the unit tests e.g. `pkg/yqlib/operator_add_test.go`.
+
+The pipeline will run the tests and automatically concatenate the files together, and put them under 
+`pkg/qylib/doc/add.md`. These files are checked in the master branch (and are copied to the gitbook branch as part of the release process).
+
+## How to contribute
+
+The first step is to find if what you want is automatically generated or not - start by looking in the master branch. 
+
+### Updating dynamic documentation from master
+- Search for the documentation you want to update. If you find matches in a `*_test.go` file - update that, as that will automatically update the matching `*.md` file 
+- Assuming you are updating a `*_test.go` file, once updated, run the test to regenerated the docs. E.g. for the 'Add' test generated docs, from the pkg/yqlib folder run:
+`go test -run TestAddOperatorScenarios` which will run that test defined in the `operator_add_test.go` file.
+- Ensure the tests still pass, and check the generated documentation have your update.
+- Note: If the documentation is only in a `headers/*.md` file, then just update that directly
+- Raise a PR to merge the changes into master!
+
+### Updating static documentation from the gitbook branch
+If you haven't found what you want to update in the master branch, then check the gitbook branch directly as there are a few pages in there that are not in master.
+
+- Update the `*.md` files
+- Raise a PR to merge the changes into gitbook.

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.18.0 as builder
+FROM golang:1.18.3 as builder
 
 WORKDIR /go/src/mikefarah/yq
 

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM golang:1.18.0
+FROM golang:1.18.3
 
 COPY scripts/devtools.sh /opt/devtools.sh
 

README.md

@@ -1,6 +1,6 @@
 # yq
 
-![Build](https://github.com/mikefarah/yq/workflows/Build/badge.svg)  ![Docker Pulls](https://img.shields.io/docker/pulls/mikefarah/yq.svg) ![Github Releases (by Release)](https://img.shields.io/github/downloads/mikefarah/yq/total.svg) ![Go Report](https://goreportcard.com/badge/github.com/mikefarah/yq)
+![Build](https://github.com/mikefarah/yq/workflows/Build/badge.svg)  ![Docker Pulls](https://img.shields.io/docker/pulls/mikefarah/yq.svg) ![Github Releases (by Release)](https://img.shields.io/github/downloads/mikefarah/yq/total.svg) ![Go Report](https://goreportcard.com/badge/github.com/mikefarah/yq) ![CodeQL](https://github.com/mikefarah/yq/workflows/CodeQL/badge.svg)
 
 
 a lightweight and portable command-line YAML, JSON and XML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json and xml. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.
@@ -18,12 +18,12 @@ TLDR:
 
 Prior to 4.18.1 
 ```bash
-cat file.yaml | yq e '.cool' -
+yq e '.cool' - < file.yaml
 ```
 
 4.18+ 
 ```bash
-cat file.yaml | yq '.cool'
+yq '.cool' < file.yaml
 ```
 
 When merging multiple files together, `eval-all/ea` is still required to tell `yq` to run the expression against all the document at once.
@@ -37,7 +37,7 @@ yq '.a.b[0].c' file.yaml
 
 Pipe from STDIN:
 ```bash
-cat file.yaml | yq '.a.b[0].c'
+yq '.a.b[0].c' < file.yaml
 ```
 
 Update a yaml file, inplace
@@ -143,11 +143,11 @@ podman run --rm -v "${PWD}":/workdir mikefarah/yq [command] [flags] [expression
 You'll need to pass the `-i\--interactive` flag to docker:
 
 ```bash
-cat myfile.yml | docker run -i --rm mikefarah/yq '.this.thing'
+docker run -i --rm mikefarah/yq '.this.thing' < myfile.yml
 ```
 
 ```bash
-cat myfile.yml | podman run -i --rm mikefarah/yq '.this.thing'
+podman run -i --rm mikefarah/yq '.this.thing' < myfile.yml
 ```
 
 #### Run commands interactively:
@@ -217,9 +217,9 @@ USER yq
 
 See https://mikefarah.gitbook.io/yq/usage/github-action for more.
 
-### Go Get:
+### Go Install:
 ```
-GO111MODULE=on go get github.com/mikefarah/yq/v4
+go install github.com/mikefarah/yq/v4@latest
 ```
 
 ## Community Supported Installation methods
@@ -310,7 +310,7 @@ Usage:
 Examples:
 
 # yq defaults to 'eval' command if no command is specified. See "yq eval --help" for more examples.
-cat myfile.yml | yq '.stuff' # outputs the data at the "stuff" node from "myfile.yml"
+yq '.stuff' < myfile.yml # outputs the data at the "stuff" node from "myfile.yml"
 
 yq -i '.stuff = "foo"' myfile.yml # update myfile.yml inplace
 

acceptance_tests/basic.sh

@@ -12,6 +12,62 @@ testBasicEvalRoundTrip() {
   assertEquals 123 "$X"
 }
 
+testBasicTrailingContent() {
+  cat >test-trailing.yml <<EOL
+test:
+# this comment will be removed
+EOL
+
+  read -r -d '' expected << EOM
+test:
+# this comment will be removed
+EOM
+  X=$(./yq test-trailing.yml -P)
+  assertEquals "$expected" "$X"
+}
+
+testBasicTrailingContent() {
+  cat >test-trailing.yml <<EOL
+test:
+# this comment will be removed
+EOL
+
+  read -r -d '' expected << EOM
+test:
+# hi
+EOM
+  X=$(./yq '. footComment = "hi"' test-trailing.yml)
+  assertEquals "$expected" "$X"
+}
+
+testBasicTrailingContentEvalAll() {
+  cat >test-trailing.yml <<EOL
+test:
+# this comment will be removed
+EOL
+
+  read -r -d '' expected << EOM
+test:
+# this comment will be removed
+EOM
+  X=$(./yq ea test-trailing.yml -P)
+  assertEquals "$expected" "$X"
+}
+
+testBasicTrailingContentEvalAll() {
+  cat >test-trailing.yml <<EOL
+test:
+# this comment will be removed
+EOL
+
+  read -r -d '' expected << EOM
+test:
+# hi
+EOM
+  X=$(./yq ea '. footComment = "hi"' test-trailing.yml)
+  assertEquals "$expected" "$X"
+}
+
 testBasicPipeWithDot() {
   ./yq -n ".a = 123" > test.yml
   X=$(cat test.yml | ./yq '.')
@@ -86,6 +142,12 @@ testBasicCatWithFilesNoDash() {
   assertEquals "$Y" "$X"
 }
 
+# when the nullinput flag is used
+# dont automatically read STDIN (this breaks github actions)
+testBasicCreateFileGithubAction() {
+  cat /dev/null | ./yq -n ".a = 123" > test.yml
+}
+
 testBasicEvalAllCatWithFilesNoDash() {
   ./yq -n ".a = 123" > test.yml
   ./yq -n ".a = 124" > test2.yml
@@ -188,6 +250,40 @@ EOL
   assertEquals "10" "$X"
 }
 
+testBasicUpdateInPlaceMultipleFilesNoExpressionEval() {
+  cat >test.yml <<EOL
+a: 0
+EOL
+  cat >test2.yml <<EOL
+a: 1
+EOL
+read -r -d '' expected << EOM
+0
+---
+1
+EOM
+  ./yq -i test.yml test2.yml
+  X=$(./yq e '.a' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testBasicUpdateInPlaceMultipleFilesNoExpressionEvalAll() {
+  cat >test.yml <<EOL
+a: 0
+EOL
+  cat >test2.yml <<EOL
+a: 1
+EOL
+read -r -d '' expected << EOM
+0
+---
+1
+EOM
+  ./yq -i ea test.yml test2.yml
+  X=$(./yq e '.a' test.yml)
+  assertEquals "$expected" "$X"
+}
+
 testBasicNoExitStatus() {
   echo "a: cat" > test.yml
   X=$(./yq e '.z' test.yml)
@@ -235,4 +331,26 @@ EOM
   assertEquals "$expected" "$X"
 }
 
+testBasicMultiplyAssignMultiDoc() {
+      cat >test.yml <<EOL
+a: 1
+---
+b: 2
+EOL
+
+read -r -d '' expected << EOM
+a: 1
+c: 3
+---
+b: 2
+c: 3
+EOM
+
+
+  X=$(./yq '. *= {"c":3}' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+
+
 source ./scripts/shunit2

acceptance_tests/inputs-format.sh

@@ -58,6 +58,48 @@ EOM
   assertEquals "$expected" "$X"
 }
 
+testInputXmlNamespaces() {
+  cat >test.yml <<EOL
+<?xml version="1.0"?>
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
+</map>
+EOL
+
+  read -r -d '' expected << EOM
+map:
+  +xmlns: some-namespace
+  +xmlns:xsi: some-instance
+  +xsi:schemaLocation: some-url
+EOM
+
+  X=$(./yq e -p=xml test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -p=xml test.yml)
+  assertEquals "$expected" "$X"
+}
+
+
+testInputXmlStrict() {
+  cat >test.yml <<EOL
+<?xml version="1.0"?>
+<!DOCTYPE root [
+<!ENTITY writer "Catherine.">
+<!ENTITY copyright "(r) Great">
+]>
+<root>
+    <item>&writer;&copyright;</item>
+</root>
+EOL
+
+  X=$(./yq -p=xml --xml-strict-mode test.yml 2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: bad file 'test.yml': XML syntax error on line 7: invalid character entity &writer;" "$X"
+
+  X=$(./yq ea -p=xml --xml-strict-mode test.yml 2>&1)
+  assertEquals "Error: bad file 'test.yml': XML syntax error on line 7: invalid character entity &writer;" "$X"
+}
+
 testInputXmlGithubAction() {
   cat >test.yml <<EOL
 <cat legs="4">BiBi</cat>

acceptance_tests/split-printer.sh

@@ -25,6 +25,56 @@ EOM
   assertEquals "$expectedDoc2" "$doc2"
 }
 
+testBasicSplitWithNameCustomExtension() {
+  rm test*.yaml || true
+  cat >test.yml <<EOL
+a: test_doc1
+--- 
+a: test_doc2
+EOL
+
+  ./yq e test.yml -s '.a + ".yaml"'
+
+  doc1=$(cat test_doc1.yaml)
+  
+  assertEquals "a: test_doc1" "$doc1"
+
+  doc2=$(cat test_doc2.yaml)
+  read -r -d '' expectedDoc2 << EOM
+---
+a: test_doc2
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+}
+
+
+
+
+testSplitFromFile() {
+  cat >test.yml <<EOL
+a: test_doc1
+--- 
+a: test_doc2
+EOL
+
+cat >test_splitExp.yml <<EOL
+.a
+EOL
+
+  ./yq test.yml --split-exp-file test_splitExp.yml
+
+  doc1=$(cat test_doc1.yml)
+  
+  assertEquals "a: test_doc1" "$doc1"
+
+  doc2=$(cat test_doc2.yml)
+  read -r -d '' expectedDoc2 << EOM
+---
+a: test_doc2
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+}
+
 testBasicSplitWithNameEvalAll() {
   cat >test.yml <<EOL
 a: test_doc1

cmd/constant.go

@@ -10,6 +10,9 @@ var inputFormat = "yaml"
 
 var xmlAttributePrefix = "+"
 var xmlContentName = "+content"
+var xmlStrictMode = false
+var xmlKeepNamespace = true
+var xmlUseRawToken = true
 
 var exitStatus = false
 var forceColor = false
@@ -26,6 +29,7 @@ var prettyPrint = false
 var frontMatter = ""
 
 var splitFileExp = ""
+var splitFileExpFile = ""
 
 var completedSuccessfully = false
 

cmd/evaluate_all_command.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"errors"
-	"os"
 
 	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"github.com/spf13/cobra"
@@ -47,36 +46,17 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 
 	var err error
 
-	firstFileIndex, err := initCommand(cmd, args)
+	expression, args, err := initCommand(cmd, args)
 	if err != nil {
 		return err
 	}
 
-	stat, _ := os.Stdin.Stat()
-	pipingStdIn := (stat.Mode() & os.ModeCharDevice) == 0
-	yqlib.GetLogger().Debug("pipingStdIn: %v", pipingStdIn)
-
-	yqlib.GetLogger().Debug("stat.Mode(): %v", stat.Mode())
-	yqlib.GetLogger().Debug("ModeDir: %v", stat.Mode()&os.ModeDir)
-	yqlib.GetLogger().Debug("ModeAppend: %v", stat.Mode()&os.ModeAppend)
-	yqlib.GetLogger().Debug("ModeExclusive: %v", stat.Mode()&os.ModeExclusive)
-	yqlib.GetLogger().Debug("ModeTemporary: %v", stat.Mode()&os.ModeTemporary)
-	yqlib.GetLogger().Debug("ModeSymlink: %v", stat.Mode()&os.ModeSymlink)
-	yqlib.GetLogger().Debug("ModeDevice: %v", stat.Mode()&os.ModeDevice)
-	yqlib.GetLogger().Debug("ModeNamedPipe: %v", stat.Mode()&os.ModeNamedPipe)
-	yqlib.GetLogger().Debug("ModeSocket: %v", stat.Mode()&os.ModeSocket)
-	yqlib.GetLogger().Debug("ModeSetuid: %v", stat.Mode()&os.ModeSetuid)
-	yqlib.GetLogger().Debug("ModeSetgid: %v", stat.Mode()&os.ModeSetgid)
-	yqlib.GetLogger().Debug("ModeCharDevice: %v", stat.Mode()&os.ModeCharDevice)
-	yqlib.GetLogger().Debug("ModeSticky: %v", stat.Mode()&os.ModeSticky)
-	yqlib.GetLogger().Debug("ModeIrregular: %v", stat.Mode()&os.ModeIrregular)
-
 	out := cmd.OutOrStdout()
 
 	if writeInplace {
 		// only use colors if its forced
 		colorsEnabled = forceColor
-		writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[firstFileIndex])
+		writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[0])
 		out, err = writeInPlaceHandler.CreateTempFile()
 		if err != nil {
 			return err
@@ -109,12 +89,12 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 	printer := yqlib.NewPrinter(encoder, printerWriter)
 
 	if frontMatter != "" {
-		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])
+		frontMatterHandler := yqlib.NewFrontMatterHandler(args[0])
 		err = frontMatterHandler.Split()
 		if err != nil {
 			return err
 		}
-		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
+		args[0] = frontMatterHandler.GetYamlFrontMatterFilename()
 
 		if frontMatter == "process" {
 			reader := frontMatterHandler.GetContentReader()
@@ -126,12 +106,6 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 
 	allAtOnceEvaluator := yqlib.NewAllAtOnceEvaluator()
 
-	expression, args, err := processArgs(pipingStdIn, args)
-	if err != nil {
-		return err
-	}
-	yqlib.GetLogger().Debugf("processed args: %v", args)
-
 	switch len(args) {
 	case 0:
 		if nullInput {

cmd/evalute_sequence_command.go

@@ -3,7 +3,6 @@ package cmd
 import (
 	"errors"
 	"fmt"
-	"os"
 
 	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"github.com/spf13/cobra"
@@ -59,42 +58,19 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 	// 1 arg, read file in sequence
 	// 2+ args, [0] = expression, file the rest
 
+	out := cmd.OutOrStdout()
+
 	var err error
-	firstFileIndex, err := initCommand(cmd, args)
+
+	expression, args, err := initCommand(cmd, args)
 	if err != nil {
 		return err
 	}
 
-	stat, _ := os.Stdin.Stat()
-	pipingStdIn := (stat.Mode() & os.ModeCharDevice) == 0
-	yqlib.GetLogger().Debug("pipingStdIn: %v", pipingStdIn)
-
-	yqlib.GetLogger().Debug("stat.Mode(): %v", stat.Mode())
-	yqlib.GetLogger().Debug("ModeDir: %v", stat.Mode()&os.ModeDir)
-	yqlib.GetLogger().Debug("ModeAppend: %v", stat.Mode()&os.ModeAppend)
-	yqlib.GetLogger().Debug("ModeExclusive: %v", stat.Mode()&os.ModeExclusive)
-	yqlib.GetLogger().Debug("ModeTemporary: %v", stat.Mode()&os.ModeTemporary)
-	yqlib.GetLogger().Debug("ModeSymlink: %v", stat.Mode()&os.ModeSymlink)
-	yqlib.GetLogger().Debug("ModeDevice: %v", stat.Mode()&os.ModeDevice)
-	yqlib.GetLogger().Debug("ModeNamedPipe: %v", stat.Mode()&os.ModeNamedPipe)
-	yqlib.GetLogger().Debug("ModeSocket: %v", stat.Mode()&os.ModeSocket)
-	yqlib.GetLogger().Debug("ModeSetuid: %v", stat.Mode()&os.ModeSetuid)
-	yqlib.GetLogger().Debug("ModeSetgid: %v", stat.Mode()&os.ModeSetgid)
-	yqlib.GetLogger().Debug("ModeCharDevice: %v", stat.Mode()&os.ModeCharDevice)
-	yqlib.GetLogger().Debug("ModeSticky: %v", stat.Mode()&os.ModeSticky)
-	yqlib.GetLogger().Debug("ModeIrregular: %v", stat.Mode()&os.ModeIrregular)
-
-	// Mask for the type bits. For regular files, none will be set.
-	yqlib.GetLogger().Debug("ModeType: %v", stat.Mode()&os.ModeType)
-
-	yqlib.GetLogger().Debug("ModePerm: %v", stat.Mode()&os.ModePerm)
-
-	out := cmd.OutOrStdout()
-
 	if writeInplace {
 		// only use colors if its forced
 		colorsEnabled = forceColor
-		writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[firstFileIndex])
+		writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[0])
 		out, err = writeInPlaceHandler.CreateTempFile()
 		if err != nil {
 			return err
@@ -129,12 +105,12 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 
 	if frontMatter != "" {
 		yqlib.GetLogger().Debug("using front matter handler")
-		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])
+		frontMatterHandler := yqlib.NewFrontMatterHandler(args[0])
 		err = frontMatterHandler.Split()
 		if err != nil {
 			return err
 		}
-		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
+		args[0] = frontMatterHandler.GetYamlFrontMatterFilename()
 
 		if frontMatter == "process" {
 			reader := frontMatterHandler.GetContentReader()
@@ -143,10 +119,6 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 		}
 		defer frontMatterHandler.CleanUp()
 	}
-	expression, args, err := processArgs(pipingStdIn, args)
-	if err != nil {
-		return err
-	}
 
 	switch len(args) {
 	case 0:

cmd/root.go

@@ -18,7 +18,7 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 # yq defaults to 'eval' command if no command is specified. See "yq eval --help" for more examples.
 
 # read the "stuff" node from "myfile.yml"
-cat myfile.yml | yq '.stuff' 
+yq '.stuff' < myfile.yml
 
 # update myfile.yml in place
 yq -i '.stuff = "foo"' myfile.yml
@@ -54,6 +54,7 @@ yq -P sample.json
 			yqlib.InitExpressionParser()
 			yqlib.XMLPreferences.AttributePrefix = xmlAttributePrefix
 			yqlib.XMLPreferences.ContentName = xmlContentName
+			yqlib.XMLPreferences.StrictMode = xmlStrictMode
 		},
 	}
 
@@ -70,6 +71,9 @@ yq -P sample.json
 
 	rootCmd.PersistentFlags().StringVar(&xmlAttributePrefix, "xml-attribute-prefix", "+", "prefix for xml attributes")
 	rootCmd.PersistentFlags().StringVar(&xmlContentName, "xml-content-name", "+content", "name for xml content (if no attribute name is present).")
+	rootCmd.PersistentFlags().BoolVar(&xmlStrictMode, "xml-strict-mode", false, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
+	rootCmd.PersistentFlags().BoolVar(&xmlKeepNamespace, "xml-keep-namespace", true, "enables keeping namespace after parsing attributes")
+	rootCmd.PersistentFlags().BoolVar(&xmlUseRawToken, "xml-raw-token", true, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")
 
 	rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.")
 	rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)")
@@ -77,7 +81,7 @@ yq -P sample.json
 	rootCmd.PersistentFlags().IntVarP(&indent, "indent", "I", 2, "sets indent level for output")
 	rootCmd.Flags().BoolVarP(&version, "version", "V", false, "Print version information and quit")
 	rootCmd.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the file inplace of first file given.")
-	rootCmd.PersistentFlags().BoolVarP(&unwrapScalar, "unwrapScalar", "", true, "unwrap scalar, print the value with no quotes, colors or comments")
+	rootCmd.PersistentFlags().BoolVarP(&unwrapScalar, "unwrapScalar", "r", true, "unwrap scalar, print the value with no quotes, colors or comments")
 	rootCmd.PersistentFlags().BoolVarP(&prettyPrint, "prettyPrint", "P", false, "pretty print, shorthand for '... style = \"\"'")
 	rootCmd.PersistentFlags().BoolVarP(&exitStatus, "exit-status", "e", false, "set exit status if there are no matches or null or false is returned")
 
@@ -88,6 +92,7 @@ yq -P sample.json
 	rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.")
 
 	rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter.")
+	rootCmd.PersistentFlags().StringVarP(&splitFileExpFile, "split-exp-file", "", "", "Use a file to specify the split-exp expression.")
 
 	rootCmd.PersistentFlags().StringVarP(&expressionFile, "from-file", "", "", "Load expression from specified file.")
 

cmd/utils.go

@@ -10,7 +10,7 @@ import (
 	"gopkg.in/op/go-logging.v1"
 )
 
-func initCommand(cmd *cobra.Command, args []string) (firstFileIndex int, err error) {
+func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 	cmd.SilenceUsage = true
 
 	fileInfo, _ := os.Stdout.Stat()
@@ -19,11 +19,17 @@ func initCommand(cmd *cobra.Command, args []string) (firstFileIndex int, err err
 		colorsEnabled = true
 	}
 
-	firstFileIndex = -1
-	if !nullInput && len(args) == 1 {
-		firstFileIndex = 0
-	} else if len(args) > 1 {
-		firstFileIndex = 1
+	expression, args, err := processArgs(args)
+	if err != nil {
+		return "", nil, err
+	}
+
+	if splitFileExpFile != "" {
+		splitExpressionBytes, err := os.ReadFile(splitFileExpFile)
+		if err != nil {
+			return "", nil, err
+		}
+		splitFileExp = string(splitExpressionBytes)
 	}
 
 	// backwards compatibility
@@ -31,19 +37,23 @@ func initCommand(cmd *cobra.Command, args []string) (firstFileIndex int, err err
 		outputFormat = "json"
 	}
 
-	if writeInplace && (firstFileIndex == -1) {
-		return 0, fmt.Errorf("write inplace flag only applicable when giving an expression and at least one file")
+	if writeInplace && (len(args) == 0 || args[0] == "-") {
+		return "", nil, fmt.Errorf("write inplace flag only applicable when giving an expression and at least one file")
+	}
+
+	if frontMatter != "" && len(args) == 0 {
+		return "", nil, fmt.Errorf("front matter flag only applicable when giving an expression and at least one file")
 	}
 
 	if writeInplace && splitFileExp != "" {
-		return 0, fmt.Errorf("write inplace cannot be used with split file")
+		return "", nil, fmt.Errorf("write inplace cannot be used with split file")
 	}
 
-	if nullInput && len(args) > 1 {
-		return 0, fmt.Errorf("cannot pass files in when using null-input flag")
+	if nullInput && len(args) > 0 {
+		return "", nil, fmt.Errorf("cannot pass files in when using null-input flag")
 	}
 
-	return firstFileIndex, nil
+	return expression, args, nil
 }
 
 func configureDecoder() (yqlib.Decoder, error) {
@@ -53,7 +63,7 @@ func configureDecoder() (yqlib.Decoder, error) {
 	}
 	switch yqlibInputFormat {
 	case yqlib.XMLInputFormat:
-		return yqlib.NewXMLDecoder(xmlAttributePrefix, xmlContentName), nil
+		return yqlib.NewXMLDecoder(xmlAttributePrefix, xmlContentName, xmlStrictMode, xmlKeepNamespace, xmlUseRawToken), nil
 	case yqlib.PropertiesInputFormat:
 		return yqlib.NewPropertiesDecoder(), nil
 	}
@@ -81,9 +91,9 @@ func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yq
 func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
 	switch format {
 	case yqlib.JSONOutputFormat:
-		return yqlib.NewJONEncoder(indent)
+		return yqlib.NewJONEncoder(indent, colorsEnabled)
 	case yqlib.PropsOutputFormat:
-		return yqlib.NewPropertiesEncoder()
+		return yqlib.NewPropertiesEncoder(unwrapScalar)
 	case yqlib.CSVOutputFormat:
 		return yqlib.NewCsvEncoder(',')
 	case yqlib.TSVOutputFormat:
@@ -116,12 +126,15 @@ func maybeFile(str string) bool {
 	return result
 }
 
-func processStdInArgs(pipingStdin bool, args []string) []string {
+func processStdInArgs(args []string) []string {
+	stat, _ := os.Stdin.Stat()
+	pipingStdin := (stat.Mode() & os.ModeCharDevice) == 0
+
 	// if we've been given a file, don't automatically
 	// read from stdin.
 	// this happens if there is more than one argument
 	// or only one argument and its a file
-	if !pipingStdin || len(args) > 1 || (len(args) > 0 && maybeFile(args[0])) {
+	if nullInput || !pipingStdin || len(args) > 1 || (len(args) > 0 && maybeFile(args[0])) {
 		return args
 	}
 
@@ -137,7 +150,7 @@ func processStdInArgs(pipingStdin bool, args []string) []string {
 	return append(args, "-")
 }
 
-func processArgs(pipingStdin bool, originalArgs []string) (string, []string, error) {
+func processArgs(originalArgs []string) (string, []string, error) {
 	expression := forceExpression
 	if expressionFile != "" {
 		expressionBytes, err := os.ReadFile(expressionFile)
@@ -147,7 +160,7 @@ func processArgs(pipingStdin bool, originalArgs []string) (string, []string, err
 		expression = string(expressionBytes)
 	}
 
-	args := processStdInArgs(pipingStdin, originalArgs)
+	args := processStdInArgs(originalArgs)
 	yqlib.GetLogger().Debugf("processed args: %v", args)
 	if expression == "" && len(args) > 0 && args[0] != "-" && !maybeFile(args[0]) {
 		yqlib.GetLogger().Debug("assuming expression is '%v'", args[0])

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.23.1"
+	Version = "4.25.3"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

examples/data1.yaml

@@ -1,2 +1,12 @@
-a: mike
-b: [t, f]
+# --------------------------------------------------
+# It's a test with comment
+# --------------------------------------------------
+test:
+  - name: a
+  - name: b
+  - name: c
+groups:
+  - name: a
+  - name: b
+  - name: c
+  - name: c

examples/data2.yaml

@@ -1,2 +1,5 @@
-c:
-  d: hamster
+# --------------------------------------------------
+# It's a test with comment
+# --------------------------------------------------
+groups:
+  - name: d

go.mod

@@ -2,17 +2,18 @@ module github.com/mikefarah/yq/v4
 
 require (
 	github.com/a8m/envsubst v1.3.0
+	github.com/alecthomas/participle/v2 v2.0.0-beta.4
+	github.com/alecthomas/repr v0.1.0
 	github.com/elliotchance/orderedmap v1.4.0
 	github.com/fatih/color v1.13.0
 	github.com/goccy/go-yaml v1.9.5
 	github.com/jinzhu/copier v0.3.5
 	github.com/magiconair/properties v1.8.6
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
-	github.com/spf13/cobra v1.4.0
-	github.com/timtadh/lexmachine v0.2.2
-	golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
+	github.com/spf13/cobra v1.5.0
+	golang.org/x/net v0.0.0-20220708220712-1185a9018129
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -20,10 +21,9 @@ require (
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/timtadh/data-structures v0.5.3 // indirect
-	golang.org/x/sys v0.0.0-20211205182925-97ca703d548d // indirect
+	golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 )
 
-go 1.17
+go 1.18

go.sum

@@ -1,6 +1,11 @@
 github.com/a8m/envsubst v1.3.0 h1:GmXKmVssap0YtlU3E230W98RWtWCyIZzjtf1apWWyAg=
 github.com/a8m/envsubst v1.3.0/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
-github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/alecthomas/assert/v2 v2.0.3 h1:WKqJODfOiQG0nEJKFKzDIG3E29CN2/4zR9XGJzKIkbg=
+github.com/alecthomas/participle/v2 v2.0.0-beta.4 h1:ublfGBm+x+p2j7KotHhrUMbKtejT7M0Gv1Mt1u3absw=
+github.com/alecthomas/participle/v2 v2.0.0-beta.4/go.mod h1:RC764t6n4L8D8ITAJv0qdokritYSNR3wV5cVwmIEaMM=
+github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE=
+github.com/alecthomas/repr v0.1.0/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/elliotchance/orderedmap v1.4.0 h1:wZtfeEONCbx6in1CZyE6bELEt/vFayMvsxqI5SgsR+A=
@@ -14,6 +19,7 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/goccy/go-yaml v1.9.5 h1:Eh/+3uk9kLxG4koCX6lRMAPS1OaMSAi+FJcya0INdB0=
 github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg=
@@ -33,42 +39,35 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q=
-github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g=
+github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
+github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/timtadh/data-structures v0.5.3 h1:F2tEjoG9qWIyUjbvXVgJqEOGJPMIiYn7U5W5mE+i/vQ=
-github.com/timtadh/data-structures v0.5.3/go.mod h1:9R4XODhJ8JdWFEI8P/HJKqxuJctfBQw6fDibMQny2oU=
-github.com/timtadh/lexmachine v0.2.2 h1:g55RnjdYazm5wnKv59pwFcBJHOyvTPfDEoz21s4PHmY=
-github.com/timtadh/lexmachine v0.2.2/go.mod h1:GBJvD5OAfRn/gnp92zb9KTgHLB7akKyxmVivoYCcjQI=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c=
-golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220708220712-1185a9018129 h1:vucSRfWwTsoXro7P+3Cjlr6flUMtzCwzlvkxEQtHHB0=
+golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211205182925-97ca703d548d h1:FjkYO/PPp4Wi0EAUOVLxePm7qVW4r4ctbWpURyuOD0E=
-golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e h1:NHvCuwuS43lGnYhten69ZWqi2QOj/CiDNcKbVqwVoew=
+golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0=
+golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW8s2qTSe3wGBtvo0MbVQG/c5k8RE=
@@ -76,5 +75,5 @@ gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

how-it-works.md

@@ -19,7 +19,7 @@ with an expression:
 .a = .b
 ```
 
-Like math expression - operator precedence is important. 
+Like math expressions - operator precedence is important. 
 
 The `=` operator takes two arguments, a `lhs` expression, which in this case is `.a` and `rhs` expression which is `.b`. 
 
@@ -47,7 +47,7 @@ b: dog
 
 ## Complex assignment, operator precedence rules
 
-Just like math expression - `yq` expression have an order of precedence. The pipe `|` operator has a low order of precedence, so operators with higher precedence will get evalated first. 
+Just like math expressions - `yq` expressions have an order of precedence. The pipe `|` operator has a low order of precedence, so operators with higher precedence will get evaluated first. 
 
 Most of the time, this is intuitively what you'd want, for instance `.a = "cat" | .b = "dog"` is effectively: `(.a = "cat") | (.b = "dog")`.
 
@@ -66,7 +66,7 @@ Lets say you had:
 Lets say you wanted to update the `sally` entry to have fruit: 'mango'. The _incorrect_ way to do that is:
 `.[] | select(.name == "sally") | .fruit = "mango"`.
 
-Becasue `|` has a low operator precedence, this will be evaluated (_incorrectly_) as : `(.[]) | (select(.name == "sally")) | (.fruit = "mango")`. What you'll see is only the updated segment returned:
+Because `|` has a low operator precedence, this will be evaluated (_incorrectly_) as : `(.[]) | (select(.name == "sally")) | (.fruit = "mango")`. What you'll see is only the updated segment returned:
 
 ```yaml
 name: sally

pkg/yqlib/candidate_node.go

@@ -14,14 +14,15 @@ type CandidateNode struct {
 	Parent *CandidateNode // parent node
 	Key    *yaml.Node     // node key, if this is a value from a map (or index in an array)
 
-	LeadingContent string
+	LeadingContent  string
+	TrailingContent string
 
 	Path      []interface{} /// the path we took to get to this node
 	Document  uint          // the document index of this node
 	Filename  string
 	FileIndex int
 	// when performing op against all nodes given, this will treat all the nodes as one
-	// (e.g. top level cross document merge). This property does not propegate to child nodes.
+	// (e.g. top level cross document merge). This property does not propagate to child nodes.
 	EvaluateTogether bool
 	IsMapKey         bool
 }
@@ -96,6 +97,13 @@ func (n *CandidateNode) CreateReplacement(node *yaml.Node) *CandidateNode {
 	}
 }
 
+func (n *CandidateNode) CreateReplacementWithDocWrappers(node *yaml.Node) *CandidateNode {
+	replacement := n.CreateReplacement(node)
+	replacement.LeadingContent = n.LeadingContent
+	replacement.TrailingContent = n.TrailingContent
+	return replacement
+}
+
 func (n *CandidateNode) createChildPath(path interface{}) []interface{} {
 	if path == nil {
 		newPath := make([]interface{}, len(n.Path))
@@ -169,6 +177,9 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignP
 	if other.Node.FootComment != "" {
 		n.Node.FootComment = other.Node.FootComment
 	}
+	if other.TrailingContent != "" {
+		n.TrailingContent = other.TrailingContent
+	}
 	if other.Node.HeadComment != "" {
 		n.Node.HeadComment = other.Node.HeadComment
 	}

pkg/yqlib/compare_operators.go

@@ -0,0 +1,133 @@
+package yqlib
+
+import (
+	"fmt"
+	"strconv"
+	"time"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+type compareTypePref struct {
+	OrEqual bool
+	Greater bool
+}
+
+func compareOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	log.Debugf("-- compareOperator")
+	prefs := expressionNode.Operation.Preferences.(compareTypePref)
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, compare(prefs), true)
+}
+
+func compare(prefs compareTypePref) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+		log.Debugf("-- compare cross function")
+		if lhs == nil && rhs == nil {
+			owner := &CandidateNode{}
+			return createBooleanCandidate(owner, prefs.OrEqual), nil
+		} else if lhs == nil {
+			log.Debugf("lhs nil, but rhs is not")
+			return createBooleanCandidate(rhs, false), nil
+		} else if rhs == nil {
+			log.Debugf("rhs nil, but rhs is not")
+			return createBooleanCandidate(lhs, false), nil
+		}
+
+		lhs.Node = unwrapDoc(lhs.Node)
+		rhs.Node = unwrapDoc(rhs.Node)
+
+		switch lhs.Node.Kind {
+		case yaml.MappingNode:
+			return nil, fmt.Errorf("maps not yet supported for comparison")
+		case yaml.SequenceNode:
+			return nil, fmt.Errorf("arrays not yet supported for comparison")
+		default:
+			if rhs.Node.Kind != yaml.ScalarNode {
+				return nil, fmt.Errorf("%v (%v) cannot be subtracted from %v", rhs.Node.Tag, rhs.Path, lhs.Node.Tag)
+			}
+			target := lhs.CreateReplacement(&yaml.Node{})
+			boolV, err := compareScalars(context, prefs, lhs.Node, rhs.Node)
+
+			return createBooleanCandidate(target, boolV), err
+		}
+	}
+}
+
+func compareDateTime(layout string, prefs compareTypePref, lhs *yaml.Node, rhs *yaml.Node) (bool, error) {
+	lhsTime, err := time.Parse(layout, lhs.Value)
+	if err != nil {
+		return false, err
+	}
+
+	rhsTime, err := time.Parse(layout, rhs.Value)
+	if err != nil {
+		return false, err
+	}
+
+	if prefs.OrEqual && lhsTime.Equal(rhsTime) {
+		return true, nil
+	}
+	if prefs.Greater {
+		return lhsTime.After(rhsTime), nil
+	}
+	return lhsTime.Before(rhsTime), nil
+
+}
+
+func compareScalars(context Context, prefs compareTypePref, lhs *yaml.Node, rhs *yaml.Node) (bool, error) {
+	lhsTag := guessTagFromCustomType(lhs)
+	rhsTag := guessTagFromCustomType(rhs)
+
+	isDateTime := lhs.Tag == "!!timestamp"
+	// if the lhs is a string, it might be a timestamp in a custom format.
+	if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
+		_, err := time.Parse(context.GetDateTimeLayout(), lhs.Value)
+		isDateTime = err == nil
+	}
+	if isDateTime {
+		return compareDateTime(context.GetDateTimeLayout(), prefs, lhs, rhs)
+	} else if lhsTag == "!!int" && rhsTag == "!!int" {
+		_, lhsNum, err := parseInt64(lhs.Value)
+		if err != nil {
+			return false, err
+		}
+		_, rhsNum, err := parseInt64(rhs.Value)
+		if err != nil {
+			return false, err
+		}
+
+		if prefs.OrEqual && lhsNum == rhsNum {
+			return true, nil
+		}
+		if prefs.Greater {
+			return lhsNum > rhsNum, nil
+		}
+		return lhsNum < rhsNum, nil
+	} else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") {
+		lhsNum, err := strconv.ParseFloat(lhs.Value, 64)
+		if err != nil {
+			return false, err
+		}
+		rhsNum, err := strconv.ParseFloat(rhs.Value, 64)
+		if err != nil {
+			return false, err
+		}
+		if prefs.OrEqual && lhsNum == rhsNum {
+			return true, nil
+		}
+		if prefs.Greater {
+			return lhsNum > rhsNum, nil
+		}
+		return lhsNum < rhsNum, nil
+	} else if lhsTag == "!!str" && rhsTag == "!!str" {
+		if prefs.OrEqual && lhs.Value == rhs.Value {
+			return true, nil
+		}
+		if prefs.Greater {
+			return lhs.Value > rhs.Value, nil
+		}
+		return lhs.Value < rhs.Value, nil
+	}
+
+	return false, fmt.Errorf("%v not yet supported for comparison", lhs.Tag)
+}

pkg/yqlib/compare_operators_test.go

@@ -0,0 +1,371 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var compareOperatorScenarios = []expressionScenario{
+	// ints, not equal
+	{
+		description: "Compare numbers (>)",
+		document:    "a: 5\nb: 4",
+		expression:  ".a > .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 5\nb: 4",
+		expression: ".a < .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:     true,
+		description: "Compare integers (>=)",
+		document:    "a: 5\nb: 4",
+		expression:  ".a >= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 5\nb: 4",
+		expression: ".a <= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+
+	// ints, equal
+	{
+		skipDoc:     true,
+		description: "Compare equal numbers (>)",
+		document:    "a: 5\nb: 5",
+		expression:  ".a > .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 5\nb: 5",
+		expression: ".a < .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		description: "Compare equal numbers (>=)",
+		document:    "a: 5\nb: 5",
+		expression:  ".a >= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 5\nb: 5",
+		expression: ".a <= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+
+	// floats, not equal
+	{
+		skipDoc:    true,
+		document:   "a: 5.2\nb: 4.1",
+		expression: ".a > .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 5.2\nb: 4.1",
+		expression: ".a < .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 5.2\nb: 4.1",
+		expression: ".a >= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 5.5\nb: 4.1",
+		expression: ".a <= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+
+	// floats, equal
+	{
+		skipDoc:    true,
+		document:   "a: 5.5\nb: 5.5",
+		expression: ".a > .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 5.5\nb: 5.5",
+		expression: ".a < .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 5.1\nb: 5.1",
+		expression: ".a >= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 5.1\nb: 5.1",
+		expression: ".a <= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+
+	// strings, not equal
+	{
+		description:    "Compare strings",
+		subdescription: "Compares strings by their bytecode.",
+		document:       "a: zoo\nb: apple",
+		expression:     ".a > .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: zoo\nb: apple",
+		expression: ".a < .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: zoo\nb: apple",
+		expression: ".a >= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: zoo\nb: apple",
+		expression: ".a <= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+
+	// strings, equal
+	{
+		skipDoc:    true,
+		document:   "a: cat\nb: cat",
+		expression: ".a > .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: cat\nb: cat",
+		expression: ".a < .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: cat\nb: cat",
+		expression: ".a >= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: cat\nb: cat",
+		expression: ".a <= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+
+	// datetime, not equal
+	{
+		description:    "Compare date times",
+		subdescription: "You can compare date times. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
+		document:       "a: 2021-01-01T03:10:00Z\nb: 2020-01-01T03:10:00Z",
+		expression:     ".a > .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 2021-01-01T03:10:00Z\nb: 2020-01-01T03:10:00Z",
+		expression: ".a < .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 2021-01-01T03:10:00Z\nb: 2020-01-01T03:10:00Z",
+		expression: ".a >= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 2021-01-01T03:10:00Z\nb: 2020-01-01T03:10:00Z",
+		expression: ".a <= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+
+	// datetime, equal
+	{
+		skipDoc:    true,
+		document:   "a: 2021-01-01T03:10:00Z\nb: 2021-01-01T03:10:00Z",
+		expression: ".a > .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 2021-01-01T03:10:00Z\nb: 2021-01-01T03:10:00Z",
+		expression: ".a < .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 2021-01-01T03:10:00Z\nb: 2021-01-01T03:10:00Z",
+		expression: ".a >= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 2021-01-01T03:10:00Z\nb: 2021-01-01T03:10:00Z",
+		expression: ".a <= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	// both null
+	{
+		description: "Both sides are null: > is false",
+		expression:  ".a > .b",
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: ".a < .b",
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		description: "Both sides are null: >= is true",
+		expression:  ".a >= .b",
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: ".a <= .b",
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+
+	// one null
+	{
+		skipDoc:     true,
+		description: "One side is null: > is false",
+		document:    `a: 5`,
+		expression:  ".a > .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: 5`,
+		expression: ".a < .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:     true,
+		description: "One side is null: >= is false",
+		document:    `a: 5`,
+		expression:  ".a >= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: 5`,
+		expression: ".a <= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: 5`,
+		expression: ".b <= .a",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: 5`,
+		expression: ".b < .a",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+}
+
+func TestCompareOperatorScenarios(t *testing.T) {
+	for _, tt := range compareOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentOperatorScenarios(t, "compare", compareOperatorScenarios)
+}

pkg/yqlib/decoder_base64.go

@@ -9,9 +9,10 @@ import (
 )
 
 type base64Decoder struct {
-	reader   io.Reader
-	finished bool
-	encoding base64.Encoding
+	reader       io.Reader
+	finished     bool
+	readAnything bool
+	encoding     base64.Encoding
 }
 
 func NewBase64Decoder() Decoder {
@@ -20,6 +21,7 @@ func NewBase64Decoder() Decoder {
 
 func (dec *base64Decoder) Init(reader io.Reader) {
 	dec.reader = reader
+	dec.readAnything = false
 	dec.finished = false
 }
 
@@ -35,8 +37,15 @@ func (dec *base64Decoder) Decode(rootYamlNode *yaml.Node) error {
 	}
 	if buf.Len() == 0 {
 		dec.finished = true
-		return io.EOF
+
+		// if we've read _only_ an empty string, lets return that
+		// otherwise if we've already read some bytes, and now we get
+		// an empty string, then we are done.
+		if dec.readAnything {
+			return io.EOF
+		}
 	}
+	dec.readAnything = true
 	rootYamlNode.Kind = yaml.ScalarNode
 	rootYamlNode.Tag = "!!str"
 	rootYamlNode.Value = buf.String()

pkg/yqlib/decoder_xml.go

@@ -2,6 +2,7 @@ package yqlib
 
 import (
 	"encoding/xml"
+	"errors"
 	"io"
 	"strings"
 	"unicode"
@@ -12,20 +13,32 @@ import (
 
 type xmlDecoder struct {
 	reader          io.Reader
+	readAnything    bool
 	attributePrefix string
 	contentName     string
+	strictMode      bool
+	keepNamespace   bool
+	useRawToken     bool
 	finished        bool
 }
 
-func NewXMLDecoder(attributePrefix string, contentName string) Decoder {
+func NewXMLDecoder(attributePrefix string, contentName string, strictMode bool, keepNamespace bool, useRawToken bool) Decoder {
 	if contentName == "" {
 		contentName = "content"
 	}
-	return &xmlDecoder{attributePrefix: attributePrefix, contentName: contentName, finished: false}
+	return &xmlDecoder{
+		attributePrefix: attributePrefix,
+		contentName:     contentName,
+		finished:        false,
+		strictMode:      strictMode,
+		keepNamespace:   keepNamespace,
+		useRawToken:     useRawToken,
+	}
 }
 
 func (dec *xmlDecoder) Init(reader io.Reader) {
 	dec.reader = reader
+	dec.readAnything = false
 	dec.finished = false
 }
 
@@ -135,8 +148,11 @@ func (dec *xmlDecoder) Decode(rootYamlNode *yaml.Node) error {
 		return err
 	} else if firstNode.Tag == "!!null" {
 		dec.finished = true
-		return io.EOF
+		if dec.readAnything {
+			return io.EOF
+		}
 	}
+	dec.readAnything = true
 	rootYamlNode.Kind = yaml.DocumentNode
 	rootYamlNode.Content = []*yaml.Node{firstNode}
 	dec.finished = true
@@ -189,7 +205,7 @@ type element struct {
 // of the map keys.
 func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
 	xmlDec := xml.NewDecoder(dec.reader)
-
+	xmlDec.Strict = dec.strictMode
 	// That will convert the charset if the provided XML is non-UTF-8
 	xmlDec.CharsetReader = charset.NewReaderLabel
 
@@ -199,8 +215,18 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
 		n:      root,
 	}
 
+	getToken := func() (xml.Token, error) {
+		if dec.useRawToken {
+			return xmlDec.RawToken()
+		}
+		return xmlDec.Token()
+	}
+
 	for {
-		t, _ := xmlDec.Token()
+		t, e := getToken()
+		if e != nil && !errors.Is(e, io.EOF) {
+			return e
+		}
 		if t == nil {
 			break
 		}
@@ -218,6 +244,11 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
 
 			// Extract attributes as children
 			for _, a := range se.Attr {
+				if dec.keepNamespace {
+					if a.Name.Space != "" {
+						a.Name.Local = a.Name.Space + ":" + a.Name.Local
+					}
+				}
 				elem.n.AddChild(dec.attributePrefix+a.Name.Local, &xmlNode{Data: a.Value})
 			}
 		case xml.CharData:

pkg/yqlib/doc/operators/add.md

@@ -135,7 +135,9 @@ a:
     b:
       - dog
       - mouse
-  a3: {b: [mouse]}
+  a3:
+    b:
+      - mouse
 ```
 
 ## String concatenation

pkg/yqlib/doc/operators/assign-update.md

@@ -107,13 +107,13 @@ c: fieldC
 ```
 then
 ```bash
-yq '(.a, .c) = "potatoe"' sample.yml
+yq '(.a, .c) = "potato"' sample.yml
 ```
 will output
 ```yaml
-a: potatoe
+a: potato
 b: fieldB
-c: potatoe
+c: potato
 ```
 
 ## Update string value
@@ -151,7 +151,7 @@ a:
 ```
 
 ## Update deeply selected results
-Note that the LHS is wrapped in brackets! This is to ensure we dont first filter out the yaml and then update the snippet.
+Note that the LHS is wrapped in brackets! This is to ensure we don't first filter out the yaml and then update the snippet.
 
 Given a sample.yml file of:
 ```yaml
@@ -199,7 +199,8 @@ yq '.a.b |= "bogs"' sample.yml
 ```
 will output
 ```yaml
-{a: {b: bogs}}
+a:
+  b: bogs
 ```
 
 ## Update node value that has an anchor
@@ -229,6 +230,8 @@ yq '.a.b.[0] |= "bogs"' sample.yml
 ```
 will output
 ```yaml
-{a: {b: [bogs]}}
+a:
+  b:
+    - bogs
 ```
 

pkg/yqlib/doc/operators/boolean-operators.md

@@ -10,6 +10,12 @@ The `or` and `and` operators take two parameters and return a boolean result.
 
 These are most commonly used with the `select` operator to filter particular nodes.
 
+## Related Operators
+
+- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals)
+- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare)
+- select operator [here](https://mikefarah.gitbook.io/yq/operators/select)
+
 {% hint style="warning" %}
 Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
 

pkg/yqlib/doc/operators/comment-operators.md

@@ -74,7 +74,6 @@ yq '. foot_comment=.a' sample.yml
 will output
 ```yaml
 a: cat
-
 # cat
 ```
 
@@ -117,6 +116,7 @@ b:
 Given a sample.yml file of:
 ```yaml
 a: cat # meow
+# have a great day
 ```
 then
 ```bash

pkg/yqlib/doc/operators/compare.md

@@ -0,0 +1,106 @@
+# Compare Operators
+
+Comparison operators (`>`, `>=`, `<`, `<=`) can be used for comparing scalar values of the same time.
+
+The following types are currently supported:
+
+- numbers
+- strings
+- datetimes
+
+## Related Operators
+
+- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals)
+- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators)
+- select operator [here](https://mikefarah.gitbook.io/yq/operators/select)
+
+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
+## Compare numbers (>)
+Given a sample.yml file of:
+```yaml
+a: 5
+b: 4
+```
+then
+```bash
+yq '.a > .b' sample.yml
+```
+will output
+```yaml
+true
+```
+
+## Compare equal numbers (>=)
+Given a sample.yml file of:
+```yaml
+a: 5
+b: 5
+```
+then
+```bash
+yq '.a >= .b' sample.yml
+```
+will output
+```yaml
+true
+```
+
+## Compare strings
+Compares strings by their bytecode.
+
+Given a sample.yml file of:
+```yaml
+a: zoo
+b: apple
+```
+then
+```bash
+yq '.a > .b' sample.yml
+```
+will output
+```yaml
+true
+```
+
+## Compare date times
+You can compare date times. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.
+
+Given a sample.yml file of:
+```yaml
+a: 2021-01-01T03:10:00Z
+b: 2020-01-01T03:10:00Z
+```
+then
+```bash
+yq '.a > .b' sample.yml
+```
+will output
+```yaml
+true
+```
+
+## Both sides are null: > is false
+Running
+```bash
+yq --null-input '.a > .b'
+```
+will output
+```yaml
+false
+```
+
+## Both sides are null: >= is true
+Running
+```bash
+yq --null-input '.a >= .b'
+```
+will output
+```yaml
+true
+```
+

pkg/yqlib/doc/operators/create-collect-into-object.md

@@ -86,3 +86,16 @@ will output
 wrap: frog
 ```
 
+## Creating yaml from scratch with multiple objects
+Running
+```bash
+yq --null-input '(.a.b = "foo") | (.d.e = "bar")'
+```
+will output
+```yaml
+a:
+  b: foo
+d:
+  e: bar
+```
+

pkg/yqlib/doc/operators/encode-decode.md

@@ -4,14 +4,14 @@ Encode operators will take the piped in object structure and encode it as a stri
 
 Note that you can optionally pass an indent value to the encode functions (see below).
 
-These operators are useful to process yaml documents that have stringified embeded yaml/json/props in them.
+These operators are useful to process yaml documents that have stringified embedded yaml/json/props in them.
 
 
 | Format | Decode (from string) | Encode (to string) |
 | --- | -- | --|
 | Yaml | from_yaml | to_yaml(i)/@yaml |
 | JSON | from_json | to_json(i)/@json |
-| Properties |  | to_props/@props |
+| Properties | from_props  | to_props/@props |
 | CSV |  | to_csv/@csv |
 | TSV |  | to_tsv/@tsv |
 | XML | from_xml | to_xml(i)/@xml |
@@ -123,6 +123,24 @@ b: |
   cool = thing
 ```
 
+## Decode props encoded string
+Given a sample.yml file of:
+```yaml
+a: |-
+  cats=great
+  dogs=cool as well
+```
+then
+```bash
+yq '.a |= from_props' sample.yml
+```
+will output
+```yaml
+a:
+  cats: great
+  dogs: cool as well
+```
+
 ## Encode value as yaml string
 Indent defaults to 2
 

pkg/yqlib/doc/operators/equals.md

@@ -14,6 +14,13 @@ select(.a == .b)
 
 The not equals `!=` operator returns `false` if the LHS is equal to the RHS.
 
+## Related Operators
+
+- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare)
+- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators)
+- select operator [here](https://mikefarah.gitbook.io/yq/operators/select)
+
+
 {% hint style="warning" %}
 Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
 
@@ -74,7 +81,7 @@ true
 false
 ```
 
-## Dont match number
+## Don't match number
 Given a sample.yml file of:
 ```yaml
 - 3
@@ -102,7 +109,7 @@ will output
 true
 ```
 
-## Non exisitant key doesn't equal a value
+## Non existent key doesn't equal a value
 Given a sample.yml file of:
 ```yaml
 a: frog

pkg/yqlib/doc/operators/error.md

@@ -0,0 +1,56 @@
+# Error
+
+Use this operation to short-circuit expressions. Useful for validation.
+
+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
+
+`yq e <exp> <file>`
+{% endhint %}
+
+## Validate a particular value
+Given a sample.yml file of:
+```yaml
+a: hello
+```
+then
+```bash
+yq 'select(.a == "howdy") or error(".a [" + .a + "] is not howdy!")' sample.yml
+```
+will output
+```bash
+Error: .a [hello] is not howdy!
+```
+
+## Validate the environment variable is a number - invalid
+Running
+```bash
+numberOfCats="please" yq --null-input 'env(numberOfCats) | select(tag == "!!int") or error("numberOfCats is not a number :(")'
+```
+will output
+```bash
+Error: numberOfCats is not a number :(
+```
+
+## Validate the environment variable is a number - valid
+`with` can be a convenient way of encapsulating validation.
+
+Given a sample.yml file of:
+```yaml
+name: Bob
+favouriteAnimal: cat
+```
+then
+```bash
+numberOfCats="3" yq '
+	with(env(numberOfCats); select(tag == "!!int") or error("numberOfCats is not a number :(")) | 
+	.numPets = env(numberOfCats)
+' sample.yml
+```
+will output
+```yaml
+name: Bob
+favouriteAnimal: cat
+numPets: 3
+```
+

pkg/yqlib/doc/operators/headers/boolean-operators.md

@@ -9,3 +9,9 @@ The `or` and `and` operators take two parameters and return a boolean result.
 `any_c(condition)` and `all_c(condition)` are like `any` and `all` but they take a condition expression that is used against each element to determine if it's `true`. Note: in `jq` you can simply pass a condition to `any` or `all` and it simply works - `yq` isn't that clever..yet
 
 These are most commonly used with the `select` operator to filter particular nodes.
+
+## Related Operators
+
+- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals)
+- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare)
+- select operator [here](https://mikefarah.gitbook.io/yq/operators/select)

pkg/yqlib/doc/operators/headers/compare.md

@@ -0,0 +1,15 @@
+# Compare Operators
+
+Comparison operators (`>`, `>=`, `<`, `<=`) can be used for comparing scalar values of the same time.
+
+The following types are currently supported:
+
+- numbers
+- strings
+- datetimes
+
+## Related Operators
+
+- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals)
+- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators)
+- select operator [here](https://mikefarah.gitbook.io/yq/operators/select)

pkg/yqlib/doc/operators/headers/encode-decode.md

@@ -4,14 +4,14 @@ Encode operators will take the piped in object structure and encode it as a stri
 
 Note that you can optionally pass an indent value to the encode functions (see below).
 
-These operators are useful to process yaml documents that have stringified embeded yaml/json/props in them.
+These operators are useful to process yaml documents that have stringified embedded yaml/json/props in them.
 
 
 | Format | Decode (from string) | Encode (to string) |
 | --- | -- | --|
 | Yaml | from_yaml | to_yaml(i)/@yaml |
 | JSON | from_json | to_json(i)/@json |
-| Properties |  | to_props/@props |
+| Properties | from_props  | to_props/@props |
 | CSV |  | to_csv/@csv |
 | TSV |  | to_tsv/@tsv |
 | XML | from_xml | to_xml(i)/@xml |

pkg/yqlib/doc/operators/headers/equals.md

@@ -13,3 +13,10 @@ select(.a == .b)
 ```
 
 The not equals `!=` operator returns `false` if the LHS is equal to the RHS.
+
+## Related Operators
+
+- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare)
+- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators)
+- select operator [here](https://mikefarah.gitbook.io/yq/operators/select)
+

pkg/yqlib/doc/operators/headers/error.md

@@ -0,0 +1,3 @@
+# Error
+
+Use this operation to short-circuit expressions. Useful for validation.

pkg/yqlib/doc/operators/headers/multiply-merge.md

@@ -28,3 +28,9 @@ Note the use of `eval-all` to ensure all documents are loaded into memory.
 ```bash
 yq eval-all '. as $item ireduce ({}; . * $item )' *.yml
 ```
+
+# Merging complex arrays together by a key field
+By default - `yq` merge is naive. It merges maps when they match the key name, and arrays are merged either by appending them together, or merging the entries by their position in the array.
+
+For more complex array merging (e.g. merging items that match on a certain key) please see the example [here](https://mikefarah.gitbook.io/yq/operators/multiply-merge#merge-arrays-of-objects-together-matching-on-a-key)
+

pkg/yqlib/doc/operators/headers/reduce.md

@@ -19,4 +19,4 @@ On the RHS there is `<init>`, the starting value of the accumulator and `<block>
 ## yq vs jq syntax
 Reduce syntax in `yq` is a little different from `jq` - as `yq` (currently) isn't as sophisticated as `jq` and its only supports infix notation (e.g. a + b, where the operator is in the middle of the two parameters) - where as `jq` uses a mix of infix notation with _prefix_ notation (e.g. `reduce a b` is like writing `+ a b`).
 
-To that end, the reduce operator is called `ireduce` for backwards compatability if a `jq` like prefix version of `reduce` is ever added.
+To that end, the reduce operator is called `ireduce` for backwards compatibility if a `jq` like prefix version of `reduce` is ever added.

pkg/yqlib/doc/operators/headers/select.md

@@ -1,3 +1,9 @@
 # Select
 
 Select is used to filter arrays and maps by a boolean expression.
+
+## Related Operators
+
+- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals)
+- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare)
+- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators)

pkg/yqlib/doc/operators/headers/string-operators.md

@@ -1,7 +1,9 @@
 # String Operators
 
 ## RegEx
-This uses golangs native regex functions under the hood - See https://github.com/google/re2/wiki/Syntax for the supported syntax.
+This uses golangs native regex functions under the hood - See their [docs](https://github.com/google/re2/wiki/Syntax) for the supported syntax.
+
+Case insensitive tip: prefix the regex with `(?i)` - e.g. `test("(?i)cats)"`.
 
 ### match(regEx)
 This operator returns the substring match details of the given regEx.
@@ -9,7 +11,7 @@ This operator returns the substring match details of the given regEx.
 ### capture(regEx)
 Capture returns named RegEx capture groups in a map. Can be more convenient than `match` depending on what you are doing.
 
-# test(regEx)
+## test(regEx)
 Returns true if the string matches the RegEx, false otherwise.
 
 ## sub(regEx, replacement)

pkg/yqlib/doc/operators/headers/unique.md

@@ -1,3 +1,4 @@
 # Unique
 
-This is used to filter out duplicated items in an array.
+This is used to filter out duplicated items in an array. Note that the original order of the array is maintained.
+

pkg/yqlib/doc/operators/multiply-merge.md

@@ -29,6 +29,12 @@ Note the use of `eval-all` to ensure all documents are loaded into memory.
 yq eval-all '. as $item ireduce ({}; . * $item )' *.yml
 ```
 
+# Merging complex arrays together by a key field
+By default - `yq` merge is naive. It merges maps when they match the key name, and arrays are merged either by appending them together, or merging the entries by their position in the array.
+
+For more complex array merging (e.g. merging items that match on a certain key) please see the example [here](https://mikefarah.gitbook.io/yq/operators/multiply-merge#merge-arrays-of-objects-together-matching-on-a-key)
+
+
 {% hint style="warning" %}
 Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
 

pkg/yqlib/doc/operators/reduce.md

@@ -19,7 +19,7 @@ On the RHS there is `<init>`, the starting value of the accumulator and `<block>
 ## yq vs jq syntax
 Reduce syntax in `yq` is a little different from `jq` - as `yq` (currently) isn't as sophisticated as `jq` and its only supports infix notation (e.g. a + b, where the operator is in the middle of the two parameters) - where as `jq` uses a mix of infix notation with _prefix_ notation (e.g. `reduce a b` is like writing `+ a b`).
 
-To that end, the reduce operator is called `ireduce` for backwards compatability if a `jq` like prefix version of `reduce` is ever added.
+To that end, the reduce operator is called `ireduce` for backwards compatibility if a `jq` like prefix version of `reduce` is ever added.
 
 {% hint style="warning" %}
 Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;

pkg/yqlib/doc/operators/select.md

@@ -2,6 +2,12 @@
 
 Select is used to filter arrays and maps by a boolean expression.
 
+## Related Operators
+
+- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals)
+- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare)
+- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators)
+
 {% hint style="warning" %}
 Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
 

pkg/yqlib/doc/operators/string-operators.md

@@ -1,7 +1,9 @@
 # String Operators
 
 ## RegEx
-This uses golangs native regex functions under the hood - See https://github.com/google/re2/wiki/Syntax for the supported syntax.
+This uses golangs native regex functions under the hood - See their [docs](https://github.com/google/re2/wiki/Syntax) for the supported syntax.
+
+Case insensitive tip: prefix the regex with `(?i)` - e.g. `test("(?i)cats)"`.
 
 ### match(regEx)
 This operator returns the substring match details of the given regEx.
@@ -9,7 +11,7 @@ This operator returns the substring match details of the given regEx.
 ### capture(regEx)
 Capture returns named RegEx capture groups in a map. Can be more convenient than `match` depending on what you are doing.
 
-# test(regEx)
+## test(regEx)
 Returns true if the string matches the RegEx, false otherwise.
 
 ## sub(regEx, replacement)
@@ -264,7 +266,7 @@ will output
 ```
 
 ## Test using regex
-Like jq'q equivalent, this works like match but only returns true/false instead of full match details
+Like jq's equivalent, this works like match but only returns true/false instead of full match details
 
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/operators/tag.md

@@ -31,6 +31,29 @@ will output
 !!seq
 ```
 
+## type is an alias for tag
+Given a sample.yml file of:
+```yaml
+a: cat
+b: 5
+c: 3.2
+e: true
+f: []
+```
+then
+```bash
+yq '.. | type' sample.yml
+```
+will output
+```yaml
+!!map
+!!str
+!!int
+!!float
+!!bool
+!!seq
+```
+
 ## Set custom tag
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/operators/unique.md

@@ -1,6 +1,7 @@
 # Unique
 
-This is used to filter out duplicated items in an array.
+This is used to filter out duplicated items in an array. Note that the original order of the array is maintained.
+
 
 {% hint style="warning" %}
 Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
@@ -9,10 +10,12 @@ Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#
 {% endhint %}
 
 ## Unique array of scalars (string/numbers)
+Note that unique maintains the original order of the array.
+
 Given a sample.yml file of:
 ```yaml
-- 1
 - 2
+- 1
 - 3
 - 2
 ```
@@ -22,8 +25,8 @@ yq 'unique' sample.yml
 ```
 will output
 ```yaml
-- 1
 - 2
+- 1
 - 3
 ```
 

pkg/yqlib/doc/operators/variable-operators.md

@@ -46,7 +46,7 @@ Example taken from [jq](https://stedolan.github.io/jq/manual/#Variable/SymbolicB
 Given a sample.yml file of:
 ```yaml
 "posts":
-  - "title": Frist psot
+  - "title": First post
     "author": anon
   - "title": A well-written article
     "author": person1
@@ -60,7 +60,7 @@ yq '.realnames as $names | .posts[] | {"title":.title, "author": $names[.author]
 ```
 will output
 ```yaml
-title: Frist psot
+title: First post
 author: Anonymous Coward
 title: A well-written article
 author: Person McPherson

pkg/yqlib/doc/usage/properties.md

@@ -17,7 +17,7 @@ Given a sample.yml file of:
 ```yaml
 # block comments don't come through
 person: # neither do comments on maps
-    name: Mike # comments on values appear
+    name: Mike Wazowski # comments on values appear
     pets: 
     - cat # comments on array values appear
     food: [pizza] # comments on arrays do not
@@ -32,7 +32,36 @@ yq -o=props sample.yml
 will output
 ```properties
 # comments on values appear
-person.name = Mike
+person.name = Mike Wazowski
+
+# comments on array values appear
+person.pets.0 = cat
+person.food.0 = pizza
+```
+
+## Encode properties: scalar encapsulation
+Note that string values with blank characters in them are encapsulated with double quotes
+
+Given a sample.yml file of:
+```yaml
+# block comments don't come through
+person: # neither do comments on maps
+    name: Mike Wazowski # comments on values appear
+    pets: 
+    - cat # comments on array values appear
+    food: [pizza] # comments on arrays do not
+emptyArray: []
+emptyMap: []
+
+```
+then
+```bash
+yq -o=props --unwrapScalar=false sample.yml
+```
+will output
+```properties
+# comments on values appear
+person.name = "Mike Wazowski"
 
 # comments on array values appear
 person.pets.0 = cat
@@ -44,7 +73,7 @@ Given a sample.yml file of:
 ```yaml
 # block comments don't come through
 person: # neither do comments on maps
-    name: Mike # comments on values appear
+    name: Mike Wazowski # comments on values appear
     pets: 
     - cat # comments on array values appear
     food: [pizza] # comments on arrays do not
@@ -58,7 +87,7 @@ yq -o=props '... comments = ""' sample.yml
 ```
 will output
 ```properties
-person.name = Mike
+person.name = Mike Wazowski
 person.pets.0 = cat
 person.food.0 = pizza
 ```
@@ -70,7 +99,7 @@ Given a sample.yml file of:
 ```yaml
 # block comments don't come through
 person: # neither do comments on maps
-    name: Mike # comments on values appear
+    name: Mike Wazowski # comments on values appear
     pets: 
     - cat # comments on array values appear
     food: [pizza] # comments on arrays do not
@@ -85,7 +114,7 @@ yq -o=props '(.. | select( (tag == "!!map" or tag =="!!seq") and length == 0)) =
 will output
 ```properties
 # comments on values appear
-person.name = Mike
+person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = cat
@@ -98,7 +127,7 @@ emptyMap =
 Given a sample.properties file of:
 ```properties
 # comments on values appear
-person.name = Mike
+person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = cat
@@ -112,7 +141,7 @@ yq -p=props sample.properties
 will output
 ```yaml
 person:
-    name: Mike # comments on values appear
+    name: Mike Wazowski # comments on values appear
     pets:
         - cat # comments on array values appear
     food:
@@ -123,7 +152,7 @@ person:
 Given a sample.properties file of:
 ```properties
 # comments on values appear
-person.name = Mike
+person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = cat
@@ -137,7 +166,7 @@ yq -p=props -o=props '.person.pets.0 = "dog"' sample.properties
 will output
 ```properties
 # comments on values appear
-person.name = Mike
+person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = dog

pkg/yqlib/doc/usage/xml.md

@@ -120,6 +120,31 @@ cat:
   +legs: "4"
 ```
 
+## Parse xml: custom dtd
+DTD entities are ignored.
+
+Given a sample.xml file of:
+```xml
+
+<?xml version="1.0"?>
+<!DOCTYPE root [
+<!ENTITY writer "Blah.">
+<!ENTITY copyright "Blah">
+]>
+<root>
+    <item>&writer;&copyright;</item>
+</root>
+```
+then
+```bash
+yq -p=xml '.' sample.xml
+```
+will output
+```yaml
+root:
+  item: '&writer;&copyright;'
+```
+
 ## Parse xml: with comments
 A best attempt is made to preserve comments.
 
@@ -167,6 +192,52 @@ cat:
 # after cat
 ```
 
+## Parse xml: keep attribute namespace
+Given a sample.xml file of:
+```xml
+
+<?xml version="1.0"?>
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
+</map>
+
+```
+then
+```bash
+yq -p=xml -o=xml --xml-keep-namespace '.' sample.xml
+```
+will output
+```xml
+<map xmlns="some-namespace" xmlns:xsi="some-instance" some-instance:schemaLocation="some-url"></map>
+```
+
+instead of
+```xml
+<map xmlns="some-namespace" xsi="some-instance" schemaLocation="some-url"></map>
+```
+
+## Parse xml: keep raw attribute namespace
+Given a sample.xml file of:
+```xml
+
+<?xml version="1.0"?>
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
+</map>
+
+```
+then
+```bash
+yq -p=xml -o=xml --xml-keep-namespace --xml-raw-token '.' sample.xml
+```
+will output
+```xml
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url"></map>
+```
+
+instead of
+```xml
+<map xmlns="some-namespace" xsi="some-instance" schemaLocation="some-url"></map>
+```
+
 ## Encode xml: simple
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/encoder_json.go

@@ -1,6 +1,7 @@
 package yqlib
 
 import (
+	"bytes"
 	"encoding/json"
 	"io"
 
@@ -9,6 +10,7 @@ import (
 
 type jsonEncoder struct {
 	indentString string
+	colorise     bool
 }
 
 func mapKeysToStrings(node *yaml.Node) {
@@ -26,14 +28,14 @@ func mapKeysToStrings(node *yaml.Node) {
 	}
 }
 
-func NewJONEncoder(indent int) Encoder {
+func NewJONEncoder(indent int, colorise bool) Encoder {
 	var indentString = ""
 
 	for index := 0; index < indent; index++ {
 		indentString = indentString + " "
 	}
 
-	return &jsonEncoder{indentString}
+	return &jsonEncoder{indentString, colorise}
 }
 
 func (je *jsonEncoder) CanHandleAliases() bool {
@@ -49,7 +51,14 @@ func (je *jsonEncoder) PrintLeadingContent(writer io.Writer, content string) err
 }
 
 func (je *jsonEncoder) Encode(writer io.Writer, node *yaml.Node) error {
-	var encoder = json.NewEncoder(writer)
+
+	destination := writer
+	tempBuffer := bytes.NewBuffer(nil)
+	if je.colorise {
+		destination = tempBuffer
+	}
+
+	var encoder = json.NewEncoder(destination)
 	encoder.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
 	encoder.SetIndent("", je.indentString)
 
@@ -60,5 +69,12 @@ func (je *jsonEncoder) Encode(writer io.Writer, node *yaml.Node) error {
 	if errorDecoding != nil {
 		return errorDecoding
 	}
-	return encoder.Encode(dataBucket)
+	err := encoder.Encode(dataBucket)
+	if err != nil {
+		return err
+	}
+	if je.colorise {
+		return colorizeAndPrint(tempBuffer.Bytes(), writer)
+	}
+	return nil
 }

pkg/yqlib/encoder_properties.go

@@ -12,10 +12,13 @@ import (
 )
 
 type propertiesEncoder struct {
+	unwrapScalar bool
 }
 
-func NewPropertiesEncoder() Encoder {
-	return &propertiesEncoder{}
+func NewPropertiesEncoder(unwrapScalar bool) Encoder {
+	return &propertiesEncoder{
+		unwrapScalar: unwrapScalar,
+	}
 }
 
 func (pe *propertiesEncoder) CanHandleAliases() bool {
@@ -75,7 +78,13 @@ func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yaml.Node,
 	p.SetComment(path, headAndLineComment(node))
 	switch node.Kind {
 	case yaml.ScalarNode:
-		_, _, err := p.Set(path, node.Value)
+		var nodeValue string
+		if pe.unwrapScalar || !strings.Contains(node.Value, " ") {
+			nodeValue = node.Value
+		} else {
+			nodeValue = fmt.Sprintf("%q", node.Value)
+		}
+		_, _, err := p.Set(path, nodeValue)
 		return err
 	case yaml.DocumentNode:
 		return pe.doEncode(p, node.Content[0], path)

pkg/yqlib/encoder_properties_test.go

@@ -9,11 +9,11 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
-func yamlToProps(sampleYaml string) string {
+func yamlToProps(sampleYaml string, unwrapScalar bool) string {
 	var output bytes.Buffer
 	writer := bufio.NewWriter(&output)
 
-	var propsEncoder = NewPropertiesEncoder()
+	var propsEncoder = NewPropertiesEncoder(unwrapScalar)
 	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
 		panic(err)
@@ -28,51 +28,100 @@ func yamlToProps(sampleYaml string) string {
 	return strings.TrimSuffix(output.String(), "\n")
 }
 
-func TestPropertiesEncoderSimple(t *testing.T) {
+func TestPropertiesEncoderSimple_Unwrapped(t *testing.T) {
 	var sampleYaml = `a: 'bob cool'`
 
 	var expectedProps = `a = bob cool`
-	var actualProps = yamlToProps(sampleYaml)
+	var actualProps = yamlToProps(sampleYaml, true)
 	test.AssertResult(t, expectedProps, actualProps)
 }
 
-func TestPropertiesEncoderSimpleWithComments(t *testing.T) {
+func TestPropertiesEncoderSimple_Wrapped(t *testing.T) {
+	var sampleYaml = `a: 'bob cool'`
+
+	var expectedProps = `a = "bob cool"`
+	var actualProps = yamlToProps(sampleYaml, false)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderSimpleWithComments_Unwrapped(t *testing.T) {
 	var sampleYaml = `a: 'bob cool' # line`
 
 	var expectedProps = `# line
 a = bob cool`
-	var actualProps = yamlToProps(sampleYaml)
+	var actualProps = yamlToProps(sampleYaml, true)
 	test.AssertResult(t, expectedProps, actualProps)
 }
 
-func TestPropertiesEncoderDeep(t *testing.T) {
+func TestPropertiesEncoderSimpleWithComments_Wrapped(t *testing.T) {
+	var sampleYaml = `a: 'bob cool' # line`
+
+	var expectedProps = `# line
+a = "bob cool"`
+	var actualProps = yamlToProps(sampleYaml, false)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderDeep_Unwrapped(t *testing.T) {
 	var sampleYaml = `a: 
   b: "bob cool"
 `
 
 	var expectedProps = `a.b = bob cool`
-	var actualProps = yamlToProps(sampleYaml)
+	var actualProps = yamlToProps(sampleYaml, true)
 	test.AssertResult(t, expectedProps, actualProps)
 }
 
-func TestPropertiesEncoderDeepWithComments(t *testing.T) {
+func TestPropertiesEncoderDeep_Wrapped(t *testing.T) {
+	var sampleYaml = `a: 
+  b: "bob cool"
+`
+
+	var expectedProps = `a.b = "bob cool"`
+	var actualProps = yamlToProps(sampleYaml, false)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderDeepWithComments_Unwrapped(t *testing.T) {
 	var sampleYaml = `a:  # a thing
   b: "bob cool" # b thing
 `
 
 	var expectedProps = `# b thing
 a.b = bob cool`
-	var actualProps = yamlToProps(sampleYaml)
+	var actualProps = yamlToProps(sampleYaml, true)
 	test.AssertResult(t, expectedProps, actualProps)
 }
 
-func TestPropertiesEncoderArray(t *testing.T) {
+func TestPropertiesEncoderDeepWithComments_Wrapped(t *testing.T) {
+	var sampleYaml = `a:  # a thing
+  b: "bob cool" # b thing
+`
+
+	var expectedProps = `# b thing
+a.b = "bob cool"`
+	var actualProps = yamlToProps(sampleYaml, false)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderArray_Unwrapped(t *testing.T) {
 	var sampleYaml = `a: 
   b: [{c: dog}, {c: cat}]
 `
 
 	var expectedProps = `a.b.0.c = dog
 a.b.1.c = cat`
-	var actualProps = yamlToProps(sampleYaml)
+	var actualProps = yamlToProps(sampleYaml, true)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderArray_Wrapped(t *testing.T) {
+	var sampleYaml = `a: 
+  b: [{c: dog named jim}, {c: cat named jam}]
+`
+
+	var expectedProps = `a.b.0.c = "dog named jim"
+a.b.1.c = "cat named jam"`
+	var actualProps = yamlToProps(sampleYaml, false)
 	test.AssertResult(t, expectedProps, actualProps)
 }

pkg/yqlib/encoder_test.go

@@ -13,7 +13,7 @@ func yamlToJSON(sampleYaml string, indent int) string {
 	var output bytes.Buffer
 	writer := bufio.NewWriter(&output)
 
-	var jsonEncoder = NewJONEncoder(indent)
+	var jsonEncoder = NewJONEncoder(indent, false)
 	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
 		panic(err)

pkg/yqlib/encoder_xml.go

@@ -9,7 +9,7 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-var XMLPreferences = xmlPreferences{AttributePrefix: "+", ContentName: "+content"}
+var XMLPreferences = xmlPreferences{AttributePrefix: "+", ContentName: "+content", StrictMode: false, UseRawToken: false}
 
 type xmlEncoder struct {
 	attributePrefix string

pkg/yqlib/expression_parser.go

@@ -21,7 +21,7 @@ type expressionParserImpl struct {
 }
 
 func newExpressionParser() ExpressionParserInterface {
-	return &expressionParserImpl{newExpressionTokeniser(), newExpressionPostFixer()}
+	return &expressionParserImpl{newParticipleLexer(), newExpressionPostFixer()}
 }
 
 func (p *expressionParserImpl) ParseExpression(expression string) (*ExpressionNode, error) {

pkg/yqlib/expression_postfix.go

@@ -50,7 +50,7 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
 			log.Debugf("put %v onto the opstack", currentToken.toString(true))
 		case closeCollect, closeCollectObject:
 			var opener tokenType = openCollect
-			var collectOperator *operationType = collectOpType
+			var collectOperator = collectOpType
 			if currentToken.TokenType == closeCollectObject {
 				opener = openCollectObject
 				collectOperator = collectObjectOpType
@@ -76,7 +76,7 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
 			// on the close op - move it to the traverse array op
 			// allows for .["cat"]?
 			prefs := traversePreferences{}
-			closeTokenMatch := string(currentToken.Match.Bytes)
+			closeTokenMatch := currentToken.Match
 			if closeTokenMatch[len(closeTokenMatch)-1:] == "?" {
 				prefs.OptionalTraverse = true
 			}

pkg/yqlib/expression_processing_test.go

@@ -312,7 +312,7 @@ var pathTests = []struct {
 	},
 }
 
-var tokeniser = newExpressionTokeniser()
+var tokeniser = newParticipleLexer()
 var postFixer = newExpressionPostFixer()
 
 func TestPathParsing(t *testing.T) {

pkg/yqlib/expression_tokeniser.go

@@ -1,674 +0,0 @@
-package yqlib
-
-import (
-	"fmt"
-	"regexp"
-	"strconv"
-	"strings"
-
-	lex "github.com/timtadh/lexmachine"
-	"github.com/timtadh/lexmachine/machines"
-)
-
-func skip(*lex.Scanner, *machines.Match) (interface{}, error) {
-	return nil, nil
-}
-
-type tokenType uint32
-
-const (
-	operationToken = 1 << iota
-	openBracket
-	closeBracket
-	openCollect
-	closeCollect
-	openCollectObject
-	closeCollectObject
-	traverseArrayCollect
-)
-
-type token struct {
-	TokenType            tokenType
-	Operation            *Operation
-	AssignOperation      *Operation      // e.g. tag (GetTag) op becomes AssignTag if '=' follows it
-	CheckForPostTraverse bool            // e.g. [1]cat should really be [1].cat
-	Match                *machines.Match // match that created this token
-
-}
-
-func (t *token) toString(detail bool) string {
-	if t.TokenType == operationToken {
-		if detail {
-			return fmt.Sprintf("%v (%v)", t.Operation.toString(), t.Operation.OperationType.Precedence)
-		}
-		return t.Operation.toString()
-	} else if t.TokenType == openBracket {
-		return "("
-	} else if t.TokenType == closeBracket {
-		return ")"
-	} else if t.TokenType == openCollect {
-		return "["
-	} else if t.TokenType == closeCollect {
-		return "]"
-	} else if t.TokenType == openCollectObject {
-		return "{"
-	} else if t.TokenType == closeCollectObject {
-		return "}"
-	} else if t.TokenType == traverseArrayCollect {
-		return ".["
-
-	} else {
-		return "NFI"
-	}
-}
-
-func pathToken(wrapped bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-		prefs := traversePreferences{}
-
-		if value[len(value)-1:] == "?" {
-			prefs.OptionalTraverse = true
-			value = value[:len(value)-1]
-		}
-
-		value = value[1:]
-		if wrapped {
-			value = unwrap(value)
-		}
-		log.Debug("PathToken %v", value)
-		op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
-	}
-}
-
-func opToken(op *operationType) lex.Action {
-	return opTokenWithPrefs(op, nil, nil)
-}
-
-func opAssignableToken(opType *operationType, assignOpType *operationType) lex.Action {
-	return opTokenWithPrefs(opType, assignOpType, nil)
-}
-
-func assignOpToken(updateAssign bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		log.Debug("assignOpToken %v", string(m.Bytes))
-		value := string(m.Bytes)
-		prefs := assignPreferences{DontOverWriteAnchor: true}
-		op := &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, UpdateAssign: updateAssign, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func multiplyWithPrefs(op *operationType) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		prefs := multiplyPreferences{}
-		options := string(m.Bytes)
-		if strings.Contains(options, "+") {
-			prefs.AppendArrays = true
-		}
-		if strings.Contains(options, "?") {
-			prefs.TraversePrefs = traversePreferences{DontAutoCreate: true}
-		}
-		if strings.Contains(options, "n") {
-			prefs.AssignPrefs = assignPreferences{OnlyWriteNull: true}
-		}
-		if strings.Contains(options, "d") {
-			prefs.DeepMergeArrays = true
-		}
-		prefs.TraversePrefs.DontFollowAlias = true
-		op := &Operation{OperationType: op, Value: multiplyOpType.Type, StringValue: options, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func opTokenWithPrefs(op *operationType, assignOpType *operationType, preferences interface{}) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		log.Debug("opTokenWithPrefs %v", string(m.Bytes))
-		value := string(m.Bytes)
-		op := &Operation{OperationType: op, Value: op.Type, StringValue: value, Preferences: preferences}
-		var assign *Operation
-		if assignOpType != nil {
-			assign = &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, Preferences: preferences}
-		}
-		return &token{TokenType: operationToken, Operation: op, AssignOperation: assign}, nil
-	}
-}
-
-func hasOptionParameter(value string, option string) bool {
-	parameterParser := regexp.MustCompile(`.*\([^\)]*\)`)
-	matches := parameterParser.FindStringSubmatch(value)
-	if len(matches) == 0 {
-		return false
-	}
-	parameterString := matches[0]
-	optionParser := regexp.MustCompile(fmt.Sprintf("\\b%v\\b", option))
-	return len(optionParser.FindStringSubmatch(parameterString)) > 0
-}
-
-func extractNumberParameter(value string) (int, error) {
-	parameterParser := regexp.MustCompile(`.*\(([0-9]+)\)`)
-	matches := parameterParser.FindStringSubmatch(value)
-	var indent, errParsingInt = strconv.ParseInt(matches[1], 10, 32)
-	if errParsingInt != nil {
-		return 0, errParsingInt
-	}
-	return int(indent), nil
-}
-
-func envSubstWithOptions() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-		noEmpty := hasOptionParameter(value, "ne")
-		noUnset := hasOptionParameter(value, "nu")
-		failFast := hasOptionParameter(value, "ff")
-		envsubstOpType.Type = "ENVSUBST"
-		prefs := envOpPreferences{NoUnset: noUnset, NoEmpty: noEmpty, FailFast: failFast}
-		if noEmpty {
-			envsubstOpType.Type = envsubstOpType.Type + "_NO_EMPTY"
-		}
-		if noUnset {
-			envsubstOpType.Type = envsubstOpType.Type + "_NO_UNSET"
-		}
-
-		op := &Operation{OperationType: envsubstOpType, Value: envsubstOpType.Type, StringValue: value, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func flattenWithDepth() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-		var depth, errParsingInt = extractNumberParameter(value)
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-
-		prefs := flattenPreferences{depth: depth}
-		op := &Operation{OperationType: flattenOpType, Value: flattenOpType.Type, StringValue: value, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func encodeWithIndent(outputFormat PrinterOutputFormat) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-		var indent, errParsingInt = extractNumberParameter(value)
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-
-		prefs := encoderPreferences{format: outputFormat, indent: indent}
-		op := &Operation{OperationType: encodeOpType, Value: encodeOpType.Type, StringValue: value, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func assignAllCommentsOp(updateAssign bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		log.Debug("assignAllCommentsOp %v", string(m.Bytes))
-		value := string(m.Bytes)
-		op := &Operation{
-			OperationType: assignCommentOpType,
-			Value:         assignCommentOpType.Type,
-			StringValue:   value,
-			UpdateAssign:  updateAssign,
-			Preferences:   commentOpPreferences{LineComment: true, HeadComment: true, FootComment: true},
-		}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func literalToken(pType tokenType, checkForPost bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &token{TokenType: pType, CheckForPostTraverse: checkForPost, Match: m}, nil
-	}
-}
-
-func unwrap(value string) string {
-	return value[1 : len(value)-1]
-}
-
-func numberValue() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		var numberString = string(m.Bytes)
-		var number, errParsingInt = strconv.ParseInt(numberString, 10, 64)
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-
-		return &token{TokenType: operationToken, Operation: createValueOperation(number, numberString)}, nil
-	}
-}
-
-func hexValue() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		var originalString = string(m.Bytes)
-		var numberString = originalString[2:]
-		log.Debugf("numberString: %v", numberString)
-		var number, errParsingInt = strconv.ParseInt(numberString, 16, 64)
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-
-		return &token{TokenType: operationToken, Operation: createValueOperation(number, originalString)}, nil
-	}
-}
-
-func floatValue() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		var numberString = string(m.Bytes)
-		var number, errParsingInt = strconv.ParseFloat(numberString, 64)
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-		return &token{TokenType: operationToken, Operation: createValueOperation(number, numberString)}, nil
-	}
-}
-
-func booleanValue(val bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &token{TokenType: operationToken, Operation: createValueOperation(val, string(m.Bytes))}, nil
-	}
-}
-
-func stringValue(wrapped bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-		if wrapped {
-			value = unwrap(value)
-		}
-		value = strings.ReplaceAll(value, "\\\"", "\"")
-		return &token{TokenType: operationToken, Operation: createValueOperation(value, value)}, nil
-	}
-}
-
-func getVariableOpToken() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-
-		value = value[1:]
-
-		getVarOperation := createValueOperation(value, value)
-		getVarOperation.OperationType = getVariableOpType
-
-		return &token{TokenType: operationToken, Operation: getVarOperation, CheckForPostTraverse: true}, nil
-	}
-}
-
-func envOp(strenv bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-		preferences := envOpPreferences{}
-
-		if strenv {
-			// strenv( )
-			value = value[7 : len(value)-1]
-			preferences.StringValue = true
-		} else {
-			//env( )
-			value = value[4 : len(value)-1]
-		}
-
-		envOperation := createValueOperation(value, value)
-		envOperation.OperationType = envOpType
-		envOperation.Preferences = preferences
-
-		return &token{TokenType: operationToken, Operation: envOperation}, nil
-	}
-}
-
-func nullValue() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &token{TokenType: operationToken, Operation: createValueOperation(nil, string(m.Bytes))}, nil
-	}
-}
-
-func selfToken() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		op := &Operation{OperationType: selfReferenceOpType}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func initLexer() (*lex.Lexer, error) {
-	lexer := lex.NewLexer()
-	lexer.Add([]byte(`\(`), literalToken(openBracket, false))
-	lexer.Add([]byte(`\)`), literalToken(closeBracket, true))
-
-	lexer.Add([]byte(`\.\[`), literalToken(traverseArrayCollect, false))
-	lexer.Add([]byte(`\.\.`), opTokenWithPrefs(recursiveDescentOpType, nil, recursiveDescentPreferences{RecurseArray: true,
-		TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: false}}))
-
-	lexer.Add([]byte(`\.\.\.`), opTokenWithPrefs(recursiveDescentOpType, nil, recursiveDescentPreferences{RecurseArray: true,
-		TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: true}}))
-
-	lexer.Add([]byte(`,`), opToken(unionOpType))
-	lexer.Add([]byte(`:\s*`), opToken(createMapOpType))
-	lexer.Add([]byte(`length`), opToken(lengthOpType))
-	lexer.Add([]byte(`line`), opToken(lineOpType))
-	lexer.Add([]byte(`column`), opToken(columnOpType))
-
-	lexer.Add([]byte(`eval`), opToken(evalOpType))
-
-	lexer.Add([]byte(`map`), opToken(mapOpType))
-	lexer.Add([]byte(`map_values`), opToken(mapValuesOpType))
-	lexer.Add([]byte(`pick`), opToken(pickOpType))
-
-	lexer.Add([]byte(`flatten\([0-9]+\)`), flattenWithDepth())
-	lexer.Add([]byte(`flatten`), opTokenWithPrefs(flattenOpType, nil, flattenPreferences{depth: -1}))
-
-	lexer.Add([]byte(`format_datetime`), opToken(formatDateTimeOpType))
-	lexer.Add([]byte(`now`), opToken(nowOpType))
-	lexer.Add([]byte(`tz`), opToken(tzOpType))
-	lexer.Add([]byte(`with_dtf`), opToken(withDtFormatOpType))
-
-	lexer.Add([]byte(`toyaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat))
-	lexer.Add([]byte(`to_yaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat))
-
-	lexer.Add([]byte(`toxml\([0-9]+\)`), encodeWithIndent(XMLOutputFormat))
-	lexer.Add([]byte(`to_xml\([0-9]+\)`), encodeWithIndent(XMLOutputFormat))
-
-	lexer.Add([]byte(`tojson\([0-9]+\)`), encodeWithIndent(JSONOutputFormat))
-	lexer.Add([]byte(`to_json\([0-9]+\)`), encodeWithIndent(JSONOutputFormat))
-
-	lexer.Add([]byte(`toyaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
-	lexer.Add([]byte(`to_yaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
-	// 0 indent doesn't work with yaml.
-	lexer.Add([]byte(`@yaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
-
-	lexer.Add([]byte(`tojson`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JSONOutputFormat, indent: 2}))
-	lexer.Add([]byte(`to_json`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JSONOutputFormat, indent: 2}))
-	lexer.Add([]byte(`@json`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JSONOutputFormat, indent: 0}))
-
-	lexer.Add([]byte(`toprops`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2}))
-	lexer.Add([]byte(`to_props`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2}))
-	lexer.Add([]byte(`@props`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2}))
-
-	lexer.Add([]byte(`tocsv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: CSVOutputFormat}))
-	lexer.Add([]byte(`to_csv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: CSVOutputFormat}))
-	lexer.Add([]byte(`@csv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: CSVOutputFormat}))
-
-	lexer.Add([]byte(`totsv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: TSVOutputFormat}))
-	lexer.Add([]byte(`to_tsv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: TSVOutputFormat}))
-	lexer.Add([]byte(`@tsv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: TSVOutputFormat}))
-
-	lexer.Add([]byte(`toxml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: XMLOutputFormat}))
-	lexer.Add([]byte(`to_xml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: XMLOutputFormat, indent: 2}))
-	lexer.Add([]byte(`@xml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: XMLOutputFormat, indent: 0}))
-
-	lexer.Add([]byte(`@base64`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: Base64OutputFormat}))
-	lexer.Add([]byte(`@base64d`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: Base64InputFormat}))
-
-	lexer.Add([]byte(`fromyaml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
-	lexer.Add([]byte(`fromjson`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
-	lexer.Add([]byte(`fromxml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: XMLInputFormat}))
-
-	lexer.Add([]byte(`from_yaml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
-	lexer.Add([]byte(`from_json`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
-	lexer.Add([]byte(`from_xml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: XMLInputFormat}))
-
-	lexer.Add([]byte(`sortKeys`), opToken(sortKeysOpType))
-	lexer.Add([]byte(`sort_keys`), opToken(sortKeysOpType))
-
-	lexer.Add([]byte(`load`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewYamlDecoder()}))
-
-	lexer.Add([]byte(`xmlload`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName)}))
-	lexer.Add([]byte(`load_xml`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName)}))
-	lexer.Add([]byte(`loadxml`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName)}))
-
-	lexer.Add([]byte(`load_base64`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewBase64Decoder()}))
-
-	lexer.Add([]byte(`load_props`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewPropertiesDecoder()}))
-	lexer.Add([]byte(`loadprops`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewPropertiesDecoder()}))
-
-	lexer.Add([]byte(`strload`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: true}))
-	lexer.Add([]byte(`load_str`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: true}))
-	lexer.Add([]byte(`loadstr`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: true}))
-
-	lexer.Add([]byte(`select`), opToken(selectOpType))
-	lexer.Add([]byte(`has`), opToken(hasOpType))
-	lexer.Add([]byte(`unique`), opToken(uniqueOpType))
-	lexer.Add([]byte(`unique_by`), opToken(uniqueByOpType))
-	lexer.Add([]byte(`group_by`), opToken(groupByOpType))
-	lexer.Add([]byte(`explode`), opToken(explodeOpType))
-	lexer.Add([]byte(`or`), opToken(orOpType))
-	lexer.Add([]byte(`and`), opToken(andOpType))
-	lexer.Add([]byte(`not`), opToken(notOpType))
-	lexer.Add([]byte(`ireduce`), opToken(reduceOpType))
-	lexer.Add([]byte(`;`), opToken(blockOpType))
-	lexer.Add([]byte(`\/\/`), opToken(alternativeOpType))
-
-	lexer.Add([]byte(`documentIndex`), opToken(getDocumentIndexOpType))
-	lexer.Add([]byte(`document_index`), opToken(getDocumentIndexOpType))
-
-	lexer.Add([]byte(`di`), opToken(getDocumentIndexOpType))
-
-	lexer.Add([]byte(`splitDoc`), opToken(splitDocumentOpType))
-	lexer.Add([]byte(`split_doc`), opToken(splitDocumentOpType))
-
-	lexer.Add([]byte(`join`), opToken(joinStringOpType))
-	lexer.Add([]byte(`sub`), opToken(subStringOpType))
-	lexer.Add([]byte(`match`), opToken(matchOpType))
-	lexer.Add([]byte(`capture`), opToken(captureOpType))
-	lexer.Add([]byte(`test`), opToken(testOpType))
-
-	lexer.Add([]byte(`upcase`), opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: true}))
-	lexer.Add([]byte(`ascii_upcase`), opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: true}))
-
-	lexer.Add([]byte(`downcase`), opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: false}))
-	lexer.Add([]byte(`ascii_downcase`), opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: false}))
-
-	lexer.Add([]byte(`sort`), opToken(sortOpType))
-	lexer.Add([]byte(`sort_by`), opToken(sortByOpType))
-	lexer.Add([]byte(`reverse`), opToken(reverseOpType))
-
-	lexer.Add([]byte(`any`), opToken(anyOpType))
-	lexer.Add([]byte(`any_c`), opToken(anyConditionOpType))
-	lexer.Add([]byte(`all`), opToken(allOpType))
-	lexer.Add([]byte(`all_c`), opToken(allConditionOpType))
-	lexer.Add([]byte(`contains`), opToken(containsOpType))
-
-	lexer.Add([]byte(`split`), opToken(splitStringOpType))
-
-	lexer.Add([]byte(`parent`), opToken(getParentOpType))
-	lexer.Add([]byte(`key`), opToken(getKeyOpType))
-	lexer.Add([]byte(`keys`), opToken(keysOpType))
-
-	lexer.Add([]byte(`style`), opAssignableToken(getStyleOpType, assignStyleOpType))
-
-	lexer.Add([]byte(`tag`), opAssignableToken(getTagOpType, assignTagOpType))
-	lexer.Add([]byte(`anchor`), opAssignableToken(getAnchorOpType, assignAnchorOpType))
-	lexer.Add([]byte(`alias`), opAssignableToken(getAliasOptype, assignAliasOpType))
-	lexer.Add([]byte(`filename`), opToken(getFilenameOpType))
-
-	lexer.Add([]byte(`fileIndex`), opToken(getFileIndexOpType))
-	lexer.Add([]byte(`file_index`), opToken(getFileIndexOpType))
-
-	lexer.Add([]byte(`fi`), opToken(getFileIndexOpType))
-	lexer.Add([]byte(`path`), opToken(getPathOpType))
-	lexer.Add([]byte(`to_entries`), opToken(toEntriesOpType))
-	lexer.Add([]byte(`from_entries`), opToken(fromEntriesOpType))
-	lexer.Add([]byte(`with_entries`), opToken(withEntriesOpType))
-
-	lexer.Add([]byte(`with`), opToken(withOpType))
-
-	lexer.Add([]byte(`lineComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}))
-	lexer.Add([]byte(`line_comment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}))
-
-	lexer.Add([]byte(`headComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{HeadComment: true}))
-	lexer.Add([]byte(`head_comment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{HeadComment: true}))
-
-	lexer.Add([]byte(`footComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{FootComment: true}))
-	lexer.Add([]byte(`foot_comment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{FootComment: true}))
-
-	lexer.Add([]byte(`comments\s*=`), assignAllCommentsOp(false))
-	lexer.Add([]byte(`comments\s*\|=`), assignAllCommentsOp(true))
-
-	lexer.Add([]byte(`collect`), opToken(collectOpType))
-
-	lexer.Add([]byte(`\s*==\s*`), opToken(equalsOpType))
-	lexer.Add([]byte(`\s*!=\s*`), opToken(notEqualsOpType))
-	lexer.Add([]byte(`\s*=\s*`), assignOpToken(false))
-
-	lexer.Add([]byte(`del`), opToken(deleteChildOpType))
-
-	lexer.Add([]byte(`\s*\|=\s*`), assignOpToken(true))
-
-	lexer.Add([]byte("( |\t|\n|\r)+"), skip)
-
-	lexer.Add([]byte(`\."[^ "]+"\??`), pathToken(true))
-	lexer.Add([]byte(`\.[^ ;\}\{\:\[\],\|\.\[\(\)=\n]+\??`), pathToken(false))
-	lexer.Add([]byte(`\.`), selfToken())
-
-	lexer.Add([]byte(`\|`), opToken(pipeOpType))
-
-	lexer.Add([]byte(`0[xX][0-9A-Fa-f]+`), hexValue())
-	lexer.Add([]byte(`-?\d+(\.\d+)`), floatValue())
-	lexer.Add([]byte(`-?[1-9](\.\d+)?[Ee][-+]?\d+`), floatValue())
-	lexer.Add([]byte(`-?\d+`), numberValue())
-
-	lexer.Add([]byte(`[Tt][Rr][Uu][Ee]`), booleanValue(true))
-	lexer.Add([]byte(`[Ff][Aa][Ll][Ss][Ee]`), booleanValue(false))
-
-	lexer.Add([]byte(`[Nn][Uu][Ll][Ll]`), nullValue())
-	lexer.Add([]byte(`~`), nullValue())
-
-	lexer.Add([]byte(`"([^"\\]*(\\.[^"\\]*)*)"`), stringValue(true))
-	lexer.Add([]byte(`strenv\([^\)]+\)`), envOp(true))
-	lexer.Add([]byte(`env\([^\)]+\)`), envOp(false))
-
-	lexer.Add([]byte(`envsubst\((ne|nu|ff| |,)+\)`), envSubstWithOptions())
-	lexer.Add([]byte(`envsubst`), opToken(envsubstOpType))
-
-	lexer.Add([]byte(`\[`), literalToken(openCollect, false))
-	lexer.Add([]byte(`\]\??`), literalToken(closeCollect, true))
-	lexer.Add([]byte(`\{`), literalToken(openCollectObject, false))
-	lexer.Add([]byte(`\}`), literalToken(closeCollectObject, true))
-	lexer.Add([]byte(`\*=[\+|\?dn]*`), multiplyWithPrefs(multiplyAssignOpType))
-	lexer.Add([]byte(`\*[\+|\?dn]*`), multiplyWithPrefs(multiplyOpType))
-
-	lexer.Add([]byte(`\+`), opToken(addOpType))
-	lexer.Add([]byte(`\+=`), opToken(addAssignOpType))
-
-	lexer.Add([]byte(`\-`), opToken(subtractOpType))
-	lexer.Add([]byte(`\-=`), opToken(subtractAssignOpType))
-	lexer.Add([]byte(`\$[a-zA-Z_-0-9]+`), getVariableOpToken())
-	lexer.Add([]byte(`as`), opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{}))
-	lexer.Add([]byte(`ref`), opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{IsReference: true}))
-
-	err := lexer.CompileNFA()
-	if err != nil {
-		return nil, err
-	}
-	return lexer, nil
-}
-
-type expressionTokeniser interface {
-	Tokenise(expression string) ([]*token, error)
-}
-
-type expressionTokeniserImpl struct {
-	lexer *lex.Lexer
-}
-
-func newExpressionTokeniser() expressionTokeniser {
-	var lexer, err = initLexer()
-	if err != nil {
-		panic(err)
-	}
-	return &expressionTokeniserImpl{lexer}
-}
-
-func (p *expressionTokeniserImpl) Tokenise(expression string) ([]*token, error) {
-	scanner, err := p.lexer.Scanner([]byte(expression))
-
-	if err != nil {
-		return nil, fmt.Errorf("parsing expression: %w", err)
-	}
-	var tokens []*token
-	for tok, err, eof := scanner.Next(); !eof; tok, err, eof = scanner.Next() {
-
-		if tok != nil {
-			currentToken := tok.(*token)
-			log.Debugf("Tokenising %v", currentToken.toString(true))
-			tokens = append(tokens, currentToken)
-		}
-		if err != nil {
-			return nil, fmt.Errorf("parsing expression: %w", err)
-		}
-	}
-	var postProcessedTokens = make([]*token, 0)
-
-	skipNextToken := false
-
-	for index := range tokens {
-		if skipNextToken {
-			skipNextToken = false
-		} else {
-			postProcessedTokens, skipNextToken = p.handleToken(tokens, index, postProcessedTokens)
-		}
-	}
-
-	return postProcessedTokens, nil
-}
-
-func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postProcessedTokens []*token) (tokensAccum []*token, skipNextToken bool) {
-	skipNextToken = false
-	currentToken := tokens[index]
-
-	log.Debug("processing %v", currentToken.toString(true))
-
-	if currentToken.TokenType == traverseArrayCollect {
-		// `.[exp]`` works by creating a traversal array of [self, exp] and piping that into the traverse array operator
-		//need to put a traverse array then a collect currentToken
-		// do this by adding traverse then converting currentToken to collect
-
-		log.Debug("  adding self")
-		op := &Operation{OperationType: selfReferenceOpType, StringValue: "SELF"}
-		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
-
-		log.Debug("  adding traverse array")
-		op = &Operation{OperationType: traverseArrayOpType, StringValue: "TRAVERSE_ARRAY"}
-		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
-
-		currentToken = &token{TokenType: openCollect}
-
-	}
-
-	if index != len(tokens)-1 && currentToken.AssignOperation != nil &&
-		tokens[index+1].TokenType == operationToken &&
-		tokens[index+1].Operation.OperationType == assignOpType {
-		log.Debug("  its an update assign")
-		currentToken.Operation = currentToken.AssignOperation
-		currentToken.Operation.UpdateAssign = tokens[index+1].Operation.UpdateAssign
-		skipNextToken = true
-	}
-
-	log.Debug("  adding token to the fixed list")
-	postProcessedTokens = append(postProcessedTokens, currentToken)
-
-	if index != len(tokens)-1 &&
-		((currentToken.TokenType == openCollect && tokens[index+1].TokenType == closeCollect) ||
-			(currentToken.TokenType == openCollectObject && tokens[index+1].TokenType == closeCollectObject)) {
-		log.Debug("  adding empty")
-		op := &Operation{OperationType: emptyOpType, StringValue: "EMPTY"}
-		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
-	}
-
-	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
-		((tokens[index+1].TokenType == operationToken && (tokens[index+1].Operation.OperationType == traversePathOpType)) ||
-			(tokens[index+1].TokenType == traverseArrayCollect)) {
-		log.Debug("  adding pipe because the next thing is traverse")
-		op := &Operation{OperationType: shortPipeOpType, Value: "PIPE", StringValue: "."}
-		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
-	}
-	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
-		tokens[index+1].TokenType == openCollect {
-
-		log.Debug("  adding traverArray because next is opencollect")
-		op := &Operation{OperationType: traverseArrayOpType}
-		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
-	}
-	return postProcessedTokens, skipNextToken
-}

pkg/yqlib/file_utils.go

@@ -39,7 +39,7 @@ func copyFileContents(src, dst string) (err error) {
 		return err
 	}
 	defer safelyCloseFile(in)
-	out, err := os.Create(dst)
+	out, err := os.Create(dst) // #nosec
 	if err != nil {
 		return err
 	}

pkg/yqlib/json_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
-var complexExpectYaml = `D0, P[], (!!map)::a: Easy! as one two three
+const complexExpectYaml = `D0, P[], (!!map)::a: Easy! as one two three
 b:
     c: 2
     d:
@@ -96,11 +96,15 @@ func decodeJSON(t *testing.T, jsonString string) *CandidateNode {
 }
 
 func testJSONScenario(t *testing.T, s formatScenario) {
-	if s.scenarioType == "encode" || s.scenarioType == "roundtrip" {
-		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewJONEncoder(s.indent)), s.description)
-	} else {
+	switch s.scenarioType {
+	case "encode", "decode":
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewJONEncoder(s.indent, false)), s.description)
+	case "":
 		var actual = resultToString(t, decodeJSON(t, s.input))
 		test.AssertResultWithContext(t, s.expected, actual, s.description)
+
+	default:
+		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
 	}
 }
 
@@ -135,14 +139,17 @@ func documentJSONDecodeScenario(t *testing.T, w *bufio.Writer, s formatScenario)
 
 func documentJSONScenario(t *testing.T, w *bufio.Writer, i interface{}) {
 	s := i.(formatScenario)
-
 	if s.skipDoc {
 		return
 	}
-	if s.scenarioType == "encode" {
-		documentJSONEncodeScenario(w, s)
-	} else {
+	switch s.scenarioType {
+	case "":
 		documentJSONDecodeScenario(t, w, s)
+	case "encode":
+		documentJSONEncodeScenario(w, s)
+
+	default:
+		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
 	}
 }
 
@@ -171,7 +178,7 @@ func documentJSONEncodeScenario(w *bufio.Writer, s formatScenario) {
 	}
 	writeOrPanic(w, "will output\n")
 
-	writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewJONEncoder(s.indent))))
+	writeOrPanic(w, fmt.Sprintf("```json\n%v```\n\n", processFormatScenario(s, NewYamlDecoder(), NewJONEncoder(s.indent, false))))
 }
 
 func TestJSONScenarios(t *testing.T) {

pkg/yqlib/lexer.go

@@ -0,0 +1,159 @@
+package yqlib
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+)
+
+type expressionTokeniser interface {
+	Tokenise(expression string) ([]*token, error)
+}
+
+type tokenType uint32
+
+const (
+	operationToken = 1 << iota
+	openBracket
+	closeBracket
+	openCollect
+	closeCollect
+	openCollectObject
+	closeCollectObject
+	traverseArrayCollect
+)
+
+type token struct {
+	TokenType            tokenType
+	Operation            *Operation
+	AssignOperation      *Operation // e.g. tag (GetTag) op becomes AssignTag if '=' follows it
+	CheckForPostTraverse bool       // e.g. [1]cat should really be [1].cat
+	Match                string
+}
+
+func (t *token) toString(detail bool) string {
+	if t.TokenType == operationToken {
+		if detail {
+			return fmt.Sprintf("%v (%v)", t.Operation.toString(), t.Operation.OperationType.Precedence)
+		}
+		return t.Operation.toString()
+	} else if t.TokenType == openBracket {
+		return "("
+	} else if t.TokenType == closeBracket {
+		return ")"
+	} else if t.TokenType == openCollect {
+		return "["
+	} else if t.TokenType == closeCollect {
+		return "]"
+	} else if t.TokenType == openCollectObject {
+		return "{"
+	} else if t.TokenType == closeCollectObject {
+		return "}"
+	} else if t.TokenType == traverseArrayCollect {
+		return ".["
+
+	} else {
+		return "NFI"
+	}
+}
+
+func unwrap(value string) string {
+	return value[1 : len(value)-1]
+}
+
+func extractNumberParameter(value string) (int, error) {
+	parameterParser := regexp.MustCompile(`.*\(([0-9]+)\)`)
+	matches := parameterParser.FindStringSubmatch(value)
+	var indent, errParsingInt = strconv.ParseInt(matches[1], 10, 32)
+	if errParsingInt != nil {
+		return 0, errParsingInt
+	}
+	return int(indent), nil
+}
+
+func hasOptionParameter(value string, option string) bool {
+	parameterParser := regexp.MustCompile(`.*\([^\)]*\)`)
+	matches := parameterParser.FindStringSubmatch(value)
+	if len(matches) == 0 {
+		return false
+	}
+	parameterString := matches[0]
+	optionParser := regexp.MustCompile(fmt.Sprintf("\\b%v\\b", option))
+	return len(optionParser.FindStringSubmatch(parameterString)) > 0
+}
+
+func postProcessTokens(tokens []*token) []*token {
+	var postProcessedTokens = make([]*token, 0)
+
+	skipNextToken := false
+
+	for index := range tokens {
+		if skipNextToken {
+			skipNextToken = false
+		} else {
+			postProcessedTokens, skipNextToken = handleToken(tokens, index, postProcessedTokens)
+		}
+	}
+
+	return postProcessedTokens
+}
+
+func handleToken(tokens []*token, index int, postProcessedTokens []*token) (tokensAccum []*token, skipNextToken bool) {
+	skipNextToken = false
+	currentToken := tokens[index]
+
+	log.Debug("processing %v", currentToken.toString(true))
+
+	if currentToken.TokenType == traverseArrayCollect {
+		// `.[exp]`` works by creating a traversal array of [self, exp] and piping that into the traverse array operator
+		//need to put a traverse array then a collect currentToken
+		// do this by adding traverse then converting currentToken to collect
+
+		log.Debug("  adding self")
+		op := &Operation{OperationType: selfReferenceOpType, StringValue: "SELF"}
+		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
+
+		log.Debug("  adding traverse array")
+		op = &Operation{OperationType: traverseArrayOpType, StringValue: "TRAVERSE_ARRAY"}
+		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
+
+		currentToken = &token{TokenType: openCollect}
+
+	}
+
+	if index != len(tokens)-1 && currentToken.AssignOperation != nil &&
+		tokens[index+1].TokenType == operationToken &&
+		tokens[index+1].Operation.OperationType == assignOpType {
+		log.Debug("  its an update assign")
+		currentToken.Operation = currentToken.AssignOperation
+		currentToken.Operation.UpdateAssign = tokens[index+1].Operation.UpdateAssign
+		skipNextToken = true
+	}
+
+	log.Debug("  adding token to the fixed list")
+	postProcessedTokens = append(postProcessedTokens, currentToken)
+
+	if index != len(tokens)-1 &&
+		((currentToken.TokenType == openCollect && tokens[index+1].TokenType == closeCollect) ||
+			(currentToken.TokenType == openCollectObject && tokens[index+1].TokenType == closeCollectObject)) {
+		log.Debug("  adding empty")
+		op := &Operation{OperationType: emptyOpType, StringValue: "EMPTY"}
+		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
+	}
+
+	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
+		((tokens[index+1].TokenType == operationToken && (tokens[index+1].Operation.OperationType == traversePathOpType)) ||
+			(tokens[index+1].TokenType == traverseArrayCollect)) {
+		log.Debug("  adding pipe because the next thing is traverse")
+		op := &Operation{OperationType: shortPipeOpType, Value: "PIPE", StringValue: "."}
+		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
+	}
+	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
+		tokens[index+1].TokenType == openCollect {
+
+		log.Debug("  adding traverArray because next is opencollect")
+		op := &Operation{OperationType: traverseArrayOpType}
+		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
+	}
+	return postProcessedTokens, skipNextToken
+}

pkg/yqlib/lexer_participle.go

@@ -0,0 +1,530 @@
+package yqlib
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/alecthomas/participle/v2/lexer"
+)
+
+var participleYqRules = []*participleYqRule{
+	{"LINE_COMMENT", `line_?comment|lineComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}), 0},
+	{"HEAD_COMMENT", `head_?comment|headComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{HeadComment: true}), 0},
+	{"FOOT_COMMENT", `foot_?comment|footComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{FootComment: true}), 0},
+
+	{"OpenBracket", `\(`, literalToken(openBracket, false), 0},
+	{"CloseBracket", `\)`, literalToken(closeBracket, true), 0},
+	{"OpenTraverseArrayCollect", `\.\[`, literalToken(traverseArrayCollect, false), 0},
+
+	{"OpenCollect", `\[`, literalToken(openCollect, false), 0},
+	{"CloseCollect", `\]\??`, literalToken(closeCollect, true), 0},
+
+	{"OpenCollectObject", `\{`, literalToken(openCollectObject, false), 0},
+	{"CloseCollectObject", `\}`, literalToken(closeCollectObject, true), 0},
+
+	{"RecursiveDecentIncludingKeys", `\.\.\.`, recursiveDecentOpToken(true), 0},
+	{"RecursiveDecent", `\.\.`, recursiveDecentOpToken(false), 0},
+
+	{"GetVariable", `\$[a-zA-Z_\-0-9]+`, getVariableOpToken(), 0},
+	{"AsignAsVariable", `as`, opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{}), 0},
+	{"AsignRefVariable", `ref`, opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{IsReference: true}), 0},
+
+	{"CreateMap", `:\s*`, opToken(createMapOpType), 0},
+	simpleOp("length", lengthOpType),
+	simpleOp("line", lineOpType),
+	simpleOp("column", columnOpType),
+	simpleOp("eval", evalOpType),
+
+	{"MapValues", `map_?values`, opToken(mapValuesOpType), 0},
+	simpleOp("map", mapOpType),
+	simpleOp("pick", pickOpType),
+
+	{"FlattenWithDepth", `flatten\([0-9]+\)`, flattenWithDepth(), 0},
+	{"Flatten", `flatten`, opTokenWithPrefs(flattenOpType, nil, flattenPreferences{depth: -1}), 0},
+
+	simpleOp("format_datetime", formatDateTimeOpType),
+	simpleOp("now", nowOpType),
+	simpleOp("tz", tzOpType),
+	simpleOp("with_dtf", withDtFormatOpType),
+	simpleOp("error", errorOpType),
+	simpleOp("sortKeys", sortKeysOpType),
+	simpleOp("sort_?keys", sortKeysOpType),
+
+	{"YamlEncodeWithIndent", `to_?yaml\([0-9]+\)`, encodeParseIndent(YamlOutputFormat), 0},
+	{"XMLEncodeWithIndent", `to_?xml\([0-9]+\)`, encodeParseIndent(XMLOutputFormat), 0},
+	{"JSONEncodeWithIndent", `to_?json\([0-9]+\)`, encodeParseIndent(JSONOutputFormat), 0},
+
+	{"YamlDecode", `from_?yaml|@yamld|from_?json|@jsond`, decodeOp(YamlInputFormat), 0},
+	{"YamlEncode", `to_?yaml|@yaml`, encodeWithIndent(YamlOutputFormat, 2), 0},
+
+	{"JSONEncode", `to_?json`, encodeWithIndent(JSONOutputFormat, 2), 0},
+	{"JSONEncodeNoIndent", `@json`, encodeWithIndent(JSONOutputFormat, 0), 0},
+
+	{"PropertiesDecode", `from_?props|@propsd`, decodeOp(PropertiesInputFormat), 0},
+	{"PropsEncode", `to_?props|@props`, encodeWithIndent(PropsOutputFormat, 2), 0},
+
+	{"XmlDecode", `from_?xml|@xmld`, decodeOp(XMLInputFormat), 0},
+	{"XMLEncode", `to_?xml`, encodeWithIndent(XMLOutputFormat, 2), 0},
+	{"XMLEncodeNoIndent", `@xml`, encodeWithIndent(XMLOutputFormat, 0), 0},
+
+	{"CSVEncode", `to_?csv|@csv`, encodeWithIndent(CSVOutputFormat, 0), 0},
+	{"TSVEncode", `to_?tsv|@tsv`, encodeWithIndent(TSVOutputFormat, 0), 0},
+
+	{"Base64d", `@base64d`, decodeOp(Base64InputFormat), 0},
+	{"Base64", `@base64`, encodeWithIndent(Base64OutputFormat, 0), 0},
+
+	{"LoadXML", `load_?xml|xml_?load`, loadOp(NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName, XMLPreferences.StrictMode, XMLPreferences.KeepNamespace, XMLPreferences.UseRawToken), false), 0},
+
+	{"LoadBase64", `load_?base64`, loadOp(NewBase64Decoder(), false), 0},
+
+	{"LoadProperties", `load_?props`, loadOp(NewPropertiesDecoder(), false), 0},
+
+	{"LoadString", `load_?str|str_?load`, loadOp(nil, true), 0},
+
+	{"LoadYaml", `load`, loadOp(NewYamlDecoder(), false), 0},
+
+	{"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0},
+
+	simpleOp("select", selectOpType),
+	simpleOp("has", hasOpType),
+	simpleOp("unique_?by", uniqueByOpType),
+	simpleOp("unique", uniqueOpType),
+
+	simpleOp("group_?by", groupByOpType),
+	simpleOp("explode", explodeOpType),
+	simpleOp("or", orOpType),
+	simpleOp("and", andOpType),
+	simpleOp("not", notOpType),
+	simpleOp("ireduce", reduceOpType),
+
+	simpleOp("join", joinStringOpType),
+	simpleOp("sub", subStringOpType),
+	simpleOp("match", matchOpType),
+	simpleOp("capture", captureOpType),
+	simpleOp("test", testOpType),
+
+	simpleOp("sort_?by", sortByOpType),
+	simpleOp("sort", sortOpType),
+
+	simpleOp("reverse", reverseOpType),
+
+	simpleOp("any_c", anyConditionOpType),
+	simpleOp("any", anyOpType),
+
+	simpleOp("all_c", allConditionOpType),
+	simpleOp("all", allOpType),
+
+	simpleOp("contains", containsOpType),
+	simpleOp("split", splitStringOpType),
+	simpleOp("parent", getParentOpType),
+
+	simpleOp("keys", keysOpType),
+	simpleOp("key", getKeyOpType),
+
+	simpleOp("file_?name|fileName", getFilenameOpType),
+	simpleOp("file_?index|fileIndex|fi", getFileIndexOpType),
+	simpleOp("path", getPathOpType),
+
+	simpleOp("to_?entries|toEntries", toEntriesOpType),
+	simpleOp("from_?entries|fromEntries", fromEntriesOpType),
+	simpleOp("with_?entries|withEntries", withEntriesOpType),
+
+	simpleOp("with", withOpType),
+
+	simpleOp("collect", collectOpType),
+	simpleOp("del", deleteChildOpType),
+
+	assignableOp("style", getStyleOpType, assignStyleOpType),
+	assignableOp("tag|type", getTagOpType, assignTagOpType),
+	assignableOp("anchor", getAnchorOpType, assignAnchorOpType),
+	assignableOp("alias", getAliasOpType, assignAliasOpType),
+
+	{"ALL_COMMENTS", `comments\s*=`, assignAllCommentsOp(false), 0},
+	{"ALL_COMMENTS_ASSIGN_RELATIVE", `comments\s*\|=`, assignAllCommentsOp(true), 0},
+
+	{"Block", `;`, opToken(blockOpType), 0},
+	{"Alternative", `\/\/`, opToken(alternativeOpType), 0},
+
+	{"DocumentIndex", `documentIndex|document_?index|di`, opToken(getDocumentIndexOpType), 0},
+
+	{"Uppercase", `upcase|ascii_?upcase`, opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: true}), 0},
+	{"Downcase", `downcase|ascii_?downcase`, opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: false}), 0},
+
+	{"HexValue", `0[xX][0-9A-Fa-f]+`, hexValue(), 0},
+	{"FloatValueScientific", `-?[1-9](\.\d+)?[Ee][-+]?\d+`, floatValue(), 0},
+	{"FloatValue", `-?\d+(\.\d+)`, floatValue(), 0},
+
+	{"NumberValue", `-?\d+`, numberValue(), 0},
+
+	{"TrueBooleanValue", `[Tt][Rr][Uu][Ee]`, booleanValue(true), 0},
+	{"FalseBooleanValue", `[Ff][Aa][Ll][Ss][Ee]`, booleanValue(false), 0},
+
+	{"NullValue", `[Nn][Uu][Ll][Ll]|~`, nullValue(), 0},
+
+	{"QuotedStringValue", `"([^"\\]*(\\.[^"\\]*)*)"`, stringValue(), 0},
+
+	{"StrEnvOp", `strenv\([^\)]+\)`, envOp(true), 0},
+	{"EnvOp", `env\([^\)]+\)`, envOp(false), 0},
+
+	{"EnvSubstWithOptions", `envsubst\((ne|nu|ff| |,)+\)`, envSubstWithOptions(), 0},
+	simpleOp("envsubst", envsubstOpType),
+
+	{"Equals", `\s*==\s*`, opToken(equalsOpType), 0},
+	{"NotEquals", `\s*!=\s*`, opToken(notEqualsOpType), 0},
+
+	{"GreaterThanEquals", `\s*>=\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: true, Greater: true}), 0},
+	{"LessThanEquals", `\s*<=\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: true, Greater: false}), 0},
+
+	{"GreaterThan", `\s*>\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: false, Greater: true}), 0},
+	{"LessThan", `\s*<\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: false, Greater: false}), 0},
+
+	{"AssignRelative", `\|=`, assignOpToken(true), 0},
+	{"Assign", `=`, assignOpToken(false), 0},
+
+	{`whitespace`, `[ \t\n]+`, nil, 0},
+
+	{"WrappedPathElement", `\."[^ "]+"\??`, pathToken(true), 0},
+	{"PathElement", `\.[^ ;\}\{\:\[\],\|\.\[\(\)=\n]+\??`, pathToken(false), 0},
+	{"Pipe", `\|`, opToken(pipeOpType), 0},
+	{"Self", `\.`, opToken(selfReferenceOpType), 0},
+
+	{"Union", `,`, opToken(unionOpType), 0},
+
+	{"MultiplyAssign", `\*=[\+|\?dn]*`, multiplyWithPrefs(multiplyAssignOpType), 0},
+	{"Multiply", `\*[\+|\?dn]*`, multiplyWithPrefs(multiplyOpType), 0},
+
+	{"AddAssign", `\+=`, opToken(addAssignOpType), 0},
+	{"Add", `\+`, opToken(addOpType), 0},
+
+	{"SubtractAssign", `\-=`, opToken(subtractAssignOpType), 0},
+	{"Subtract", `\-`, opToken(subtractOpType), 0},
+}
+
+type yqAction func(lexer.Token) (*token, error)
+
+type participleYqRule struct {
+	Name                string
+	Pattern             string
+	CreateYqToken       yqAction
+	ParticipleTokenType lexer.TokenType
+}
+
+type participleLexer struct {
+	lexerDefinition lexer.StringDefinition
+}
+
+func simpleOp(name string, opType *operationType) *participleYqRule {
+	return &participleYqRule{strings.ToUpper(string(name[1])) + name[1:], name, opToken(opType), 0}
+}
+
+func assignableOp(name string, opType *operationType, assignOpType *operationType) *participleYqRule {
+	return &participleYqRule{strings.ToUpper(string(name[1])) + name[1:], name, opTokenWithPrefs(opType, assignOpType, nil), 0}
+}
+
+func newParticipleLexer() expressionTokeniser {
+	simpleRules := make([]lexer.SimpleRule, len(participleYqRules))
+	for i, yqRule := range participleYqRules {
+		simpleRules[i] = lexer.SimpleRule{Name: yqRule.Name, Pattern: yqRule.Pattern}
+	}
+	lexerDefinition := lexer.MustSimple(simpleRules)
+	symbols := lexerDefinition.Symbols()
+
+	for _, yqRule := range participleYqRules {
+		yqRule.ParticipleTokenType = symbols[yqRule.Name]
+	}
+
+	return &participleLexer{lexerDefinition}
+}
+
+func pathToken(wrapped bool) yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		value := rawToken.Value
+		prefs := traversePreferences{}
+
+		if value[len(value)-1:] == "?" {
+			prefs.OptionalTraverse = true
+			value = value[:len(value)-1]
+		}
+
+		value = value[1:]
+		if wrapped {
+			value = unwrap(value)
+		}
+		log.Debug("PathToken %v", value)
+		op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: prefs}
+		return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
+	}
+}
+
+func recursiveDecentOpToken(includeMapKeys bool) yqAction {
+	prefs := recursiveDescentPreferences{
+		RecurseArray: true,
+		TraversePreferences: traversePreferences{
+			DontFollowAlias: true,
+			IncludeMapKeys:  includeMapKeys,
+		},
+	}
+	return opTokenWithPrefs(recursiveDescentOpType, nil, prefs)
+}
+
+func opTokenWithPrefs(opType *operationType, assignOpType *operationType, preferences interface{}) yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		value := rawToken.Value
+		op := &Operation{OperationType: opType, Value: opType.Type, StringValue: value, Preferences: preferences}
+		var assign *Operation
+		if assignOpType != nil {
+			assign = &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, Preferences: preferences}
+		}
+		return &token{TokenType: operationToken, Operation: op, AssignOperation: assign}, nil
+	}
+}
+
+func flattenWithDepth() yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		value := rawToken.Value
+		var depth, errParsingInt = extractNumberParameter(value)
+		if errParsingInt != nil {
+			return nil, errParsingInt
+		}
+
+		prefs := flattenPreferences{depth: depth}
+		op := &Operation{OperationType: flattenOpType, Value: flattenOpType.Type, StringValue: value, Preferences: prefs}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
+func assignAllCommentsOp(updateAssign bool) yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		log.Debug("assignAllCommentsOp %v", rawToken.Value)
+		value := rawToken.Value
+		op := &Operation{
+			OperationType: assignCommentOpType,
+			Value:         assignCommentOpType.Type,
+			StringValue:   value,
+			UpdateAssign:  updateAssign,
+			Preferences:   commentOpPreferences{LineComment: true, HeadComment: true, FootComment: true},
+		}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
+func assignOpToken(updateAssign bool) yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		log.Debug("assignOpToken %v", rawToken.Value)
+		value := rawToken.Value
+		prefs := assignPreferences{DontOverWriteAnchor: true}
+		op := &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, UpdateAssign: updateAssign, Preferences: prefs}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
+func booleanValue(val bool) yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		return &token{TokenType: operationToken, Operation: createValueOperation(val, rawToken.Value)}, nil
+	}
+}
+
+func nullValue() yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		return &token{TokenType: operationToken, Operation: createValueOperation(nil, rawToken.Value)}, nil
+	}
+}
+
+func stringValue() yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		value := unwrap(rawToken.Value)
+		value = strings.ReplaceAll(value, "\\\"", "\"")
+		return &token{TokenType: operationToken, Operation: createValueOperation(value, value)}, nil
+	}
+}
+
+func envOp(strenv bool) yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		value := rawToken.Value
+		preferences := envOpPreferences{}
+
+		if strenv {
+			// strenv( )
+			value = value[7 : len(value)-1]
+			preferences.StringValue = true
+		} else {
+			//env( )
+			value = value[4 : len(value)-1]
+		}
+
+		envOperation := createValueOperation(value, value)
+		envOperation.OperationType = envOpType
+		envOperation.Preferences = preferences
+
+		return &token{TokenType: operationToken, Operation: envOperation}, nil
+	}
+}
+
+func envSubstWithOptions() yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		value := rawToken.Value
+		noEmpty := hasOptionParameter(value, "ne")
+		noUnset := hasOptionParameter(value, "nu")
+		failFast := hasOptionParameter(value, "ff")
+		envsubstOpType.Type = "ENVSUBST"
+		prefs := envOpPreferences{NoUnset: noUnset, NoEmpty: noEmpty, FailFast: failFast}
+		if noEmpty {
+			envsubstOpType.Type = envsubstOpType.Type + "_NO_EMPTY"
+		}
+		if noUnset {
+			envsubstOpType.Type = envsubstOpType.Type + "_NO_UNSET"
+		}
+
+		op := &Operation{OperationType: envsubstOpType, Value: envsubstOpType.Type, StringValue: value, Preferences: prefs}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
+func multiplyWithPrefs(op *operationType) yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		prefs := multiplyPreferences{}
+		options := rawToken.Value
+		if strings.Contains(options, "+") {
+			prefs.AppendArrays = true
+		}
+		if strings.Contains(options, "?") {
+			prefs.TraversePrefs = traversePreferences{DontAutoCreate: true}
+		}
+		if strings.Contains(options, "n") {
+			prefs.AssignPrefs = assignPreferences{OnlyWriteNull: true}
+		}
+		if strings.Contains(options, "d") {
+			prefs.DeepMergeArrays = true
+		}
+		prefs.TraversePrefs.DontFollowAlias = true
+		op := &Operation{OperationType: op, Value: multiplyOpType.Type, StringValue: options, Preferences: prefs}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+
+}
+
+func getVariableOpToken() yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		value := rawToken.Value
+
+		value = value[1:]
+
+		getVarOperation := createValueOperation(value, value)
+		getVarOperation.OperationType = getVariableOpType
+
+		return &token{TokenType: operationToken, Operation: getVarOperation, CheckForPostTraverse: true}, nil
+	}
+}
+
+func hexValue() yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		var originalString = rawToken.Value
+		var numberString = originalString[2:]
+		log.Debugf("numberString: %v", numberString)
+		var number, errParsingInt = strconv.ParseInt(numberString, 16, 64)
+		if errParsingInt != nil {
+			return nil, errParsingInt
+		}
+
+		return &token{TokenType: operationToken, Operation: createValueOperation(number, originalString)}, nil
+	}
+}
+
+func floatValue() yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		var numberString = rawToken.Value
+		var number, errParsingInt = strconv.ParseFloat(numberString, 64)
+		if errParsingInt != nil {
+			return nil, errParsingInt
+		}
+		return &token{TokenType: operationToken, Operation: createValueOperation(number, numberString)}, nil
+	}
+}
+
+func numberValue() yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		var numberString = rawToken.Value
+		var number, errParsingInt = strconv.ParseInt(numberString, 10, 64)
+		if errParsingInt != nil {
+			return nil, errParsingInt
+		}
+
+		return &token{TokenType: operationToken, Operation: createValueOperation(number, numberString)}, nil
+	}
+}
+
+func encodeParseIndent(outputFormat PrinterOutputFormat) yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		value := rawToken.Value
+		var indent, errParsingInt = extractNumberParameter(value)
+		if errParsingInt != nil {
+			return nil, errParsingInt
+		}
+
+		prefs := encoderPreferences{format: outputFormat, indent: indent}
+		op := &Operation{OperationType: encodeOpType, Value: encodeOpType.Type, StringValue: value, Preferences: prefs}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
+func encodeWithIndent(outputFormat PrinterOutputFormat, indent int) yqAction {
+	prefs := encoderPreferences{format: outputFormat, indent: indent}
+	return opTokenWithPrefs(encodeOpType, nil, prefs)
+}
+
+func decodeOp(inputFormat InputFormat) yqAction {
+	prefs := decoderPreferences{format: inputFormat}
+	return opTokenWithPrefs(decodeOpType, nil, prefs)
+}
+
+func loadOp(decoder Decoder, loadAsString bool) yqAction {
+	prefs := loadPrefs{decoder: decoder, loadAsString: loadAsString}
+	return opTokenWithPrefs(loadOpType, nil, prefs)
+}
+
+func opToken(op *operationType) yqAction {
+	return opTokenWithPrefs(op, nil, nil)
+}
+
+func literalToken(tt tokenType, checkForPost bool) yqAction {
+	return func(rawToken lexer.Token) (*token, error) {
+		return &token{TokenType: tt, CheckForPostTraverse: checkForPost, Match: rawToken.Value}, nil
+	}
+}
+
+func (p *participleLexer) getYqDefinition(rawToken lexer.Token) *participleYqRule {
+	for _, yqRule := range participleYqRules {
+		if yqRule.ParticipleTokenType == rawToken.Type {
+			return yqRule
+		}
+	}
+	return &participleYqRule{}
+}
+
+func (p *participleLexer) Tokenise(expression string) ([]*token, error) {
+	myLexer, err := p.lexerDefinition.LexString("", expression)
+	if err != nil {
+		return nil, err
+	}
+	tokens := make([]*token, 0)
+
+	for {
+		rawToken, e := myLexer.Next()
+		if e != nil {
+			return nil, e
+		} else if rawToken.Type == lexer.EOF {
+			return postProcessTokens(tokens), nil
+		}
+
+		definition := p.getYqDefinition(rawToken)
+		if definition.CreateYqToken != nil {
+			token, e := definition.CreateYqToken(rawToken)
+			if e != nil {
+				return nil, e
+			}
+			tokens = append(tokens, token)
+		}
+
+	}
+
+}

pkg/yqlib/lexer_participle_test.go

@@ -0,0 +1,538 @@
+package yqlib
+
+import (
+	"testing"
+
+	"github.com/alecthomas/repr"
+	"github.com/mikefarah/yq/v4/test"
+	"gopkg.in/yaml.v3"
+)
+
+type participleLexerScenario struct {
+	expression string
+	tokens     []*token
+}
+
+var participleLexerScenarios = []participleLexerScenario{
+	{
+		expression: ".a",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "a",
+					StringValue:   "a",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+		},
+	},
+	{
+		expression: ".a.b",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "a",
+					StringValue:   "a",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: shortPipeOpType,
+					Value:         "PIPE",
+					StringValue:   ".",
+					Preferences:   nil,
+				},
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "b",
+					StringValue:   "b",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+		},
+	},
+	{
+		expression: ".a.b?",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "a",
+					StringValue:   "a",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: shortPipeOpType,
+					Value:         "PIPE",
+					StringValue:   ".",
+					Preferences:   nil,
+				},
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "b",
+					StringValue:   "b",
+					Preferences: traversePreferences{
+						OptionalTraverse: true,
+					},
+				},
+				CheckForPostTraverse: true,
+			},
+		},
+	},
+	{
+		expression: `.a."b?"`,
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "a",
+					StringValue:   "a",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: shortPipeOpType,
+					Value:         "PIPE",
+					StringValue:   ".",
+					Preferences:   nil,
+				},
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "b?",
+					StringValue:   "b?",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+		},
+	},
+	{
+		expression: `   .a  ."b?"`,
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "a",
+					StringValue:   "a",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: shortPipeOpType,
+					Value:         "PIPE",
+					StringValue:   ".",
+					Preferences:   nil,
+				},
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "b?",
+					StringValue:   "b?",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+		},
+	},
+	{
+		expression: `.a | .b`,
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "a",
+					StringValue:   "a",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: pipeOpType,
+					Value:         "PIPE",
+					StringValue:   "|",
+					Preferences:   nil,
+				},
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "b",
+					StringValue:   "b",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+		},
+	},
+	{
+		expression: "(.a)",
+		tokens: []*token{
+			{
+				TokenType: openBracket,
+				Match:     "(",
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "a",
+					StringValue:   "a",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+			{
+				TokenType:            closeBracket,
+				Match:                ")",
+				CheckForPostTraverse: true,
+			},
+		},
+	},
+	{
+		expression: "..",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: recursiveDescentOpType,
+					Value:         "RECURSIVE_DESCENT",
+					StringValue:   "..",
+					Preferences: recursiveDescentPreferences{
+						RecurseArray: true,
+						TraversePreferences: traversePreferences{
+							DontFollowAlias: true,
+							IncludeMapKeys:  false,
+						},
+					},
+				},
+			},
+		},
+	},
+	{
+		expression: "...",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: recursiveDescentOpType,
+					Value:         "RECURSIVE_DESCENT",
+					StringValue:   "...",
+					Preferences: recursiveDescentPreferences{
+						RecurseArray: true,
+						TraversePreferences: traversePreferences{
+							DontFollowAlias: true,
+							IncludeMapKeys:  true,
+						},
+					},
+				},
+			},
+		},
+	},
+	{
+		expression: ".a,.b",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "a",
+					StringValue:   "a",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: unionOpType,
+					Value:         "UNION",
+					StringValue:   ",",
+					Preferences:   nil,
+				},
+			},
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: traversePathOpType,
+					Value:         "b",
+					StringValue:   "b",
+					Preferences:   traversePreferences{},
+				},
+				CheckForPostTraverse: true,
+			},
+		},
+	},
+	{
+		expression: "map_values",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: mapValuesOpType,
+					Value:         "MAP_VALUES",
+					StringValue:   "map_values",
+					Preferences:   nil,
+				},
+			},
+		},
+	},
+	{
+		expression: "mapvalues",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: mapValuesOpType,
+					Value:         "MAP_VALUES",
+					StringValue:   "mapvalues",
+					Preferences:   nil,
+				},
+			},
+		},
+	},
+	{
+		expression: "flatten(3)",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: flattenOpType,
+					Value:         "FLATTEN_BY",
+					StringValue:   "flatten(3)",
+					Preferences:   flattenPreferences{depth: 3},
+				},
+			},
+		},
+	},
+	{
+		expression: "flatten",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: flattenOpType,
+					Value:         "FLATTEN_BY",
+					StringValue:   "flatten",
+					Preferences:   flattenPreferences{depth: -1},
+				},
+			},
+		},
+	},
+	{
+		expression: "length",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: lengthOpType,
+					Value:         "LENGTH",
+					StringValue:   "length",
+					Preferences:   nil,
+				},
+			},
+		},
+	},
+	{
+		expression: "format_datetime",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: formatDateTimeOpType,
+					Value:         "FORMAT_DATE_TIME",
+					StringValue:   "format_datetime",
+					Preferences:   nil,
+				},
+			},
+		},
+	},
+	{
+		expression: "to_yaml(3)",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: encodeOpType,
+					Value:         "ENCODE",
+					StringValue:   "to_yaml(3)",
+					Preferences: encoderPreferences{
+						format: YamlOutputFormat,
+						indent: 3,
+					},
+				},
+			},
+		},
+	},
+	{
+		expression: "tojson(2)",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: encodeOpType,
+					Value:         "ENCODE",
+					StringValue:   "tojson(2)",
+					Preferences: encoderPreferences{
+						format: JSONOutputFormat,
+						indent: 2,
+					},
+				},
+			},
+		},
+	},
+	{
+		expression: "@yaml",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: encodeOpType,
+					Value:         "ENCODE",
+					StringValue:   "@yaml",
+					Preferences: encoderPreferences{
+						format: YamlOutputFormat,
+						indent: 2,
+					},
+				},
+			},
+		},
+	},
+	{
+		expression: "to_props",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: encodeOpType,
+					Value:         "ENCODE",
+					StringValue:   "to_props",
+					Preferences: encoderPreferences{
+						format: PropsOutputFormat,
+						indent: 2,
+					},
+				},
+			},
+		},
+	},
+	{
+		expression: "@base64d",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: decodeOpType,
+					Value:         "DECODE",
+					StringValue:   "@base64d",
+					Preferences: decoderPreferences{
+						format: Base64InputFormat,
+					},
+				},
+			},
+		},
+	},
+	{
+		expression: "@base64",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: encodeOpType,
+					Value:         "ENCODE",
+					StringValue:   "@base64",
+					Preferences: encoderPreferences{
+						format: Base64OutputFormat,
+					},
+				},
+			},
+		},
+	},
+	{
+		expression: "@yamld",
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: decodeOpType,
+					Value:         "DECODE",
+					StringValue:   "@yamld",
+					Preferences: decoderPreferences{
+						format: YamlInputFormat,
+					},
+				},
+			},
+		},
+	},
+	{
+		expression: `"string with a \""`,
+		tokens: []*token{
+			{
+				TokenType: operationToken,
+				Operation: &Operation{
+					OperationType: valueOpType,
+					Value:         `string with a "`,
+					StringValue:   `string with a "`,
+					Preferences:   nil,
+					CandidateNode: &CandidateNode{
+						Node: &yaml.Node{
+							Kind:  yaml.ScalarNode,
+							Tag:   "!!str",
+							Value: `string with a "`,
+						},
+					},
+				},
+			},
+		},
+	},
+}
+
+func TestParticipleLexer(t *testing.T) {
+	log.Errorf("TestParticiple")
+	lexer := newParticipleLexer()
+
+	for _, scenario := range participleLexerScenarios {
+		actual, err := lexer.Tokenise(scenario.expression)
+		if err != nil {
+			t.Error(err)
+		} else {
+			test.AssertResultWithContext(t, repr.String(scenario.tokens, repr.Indent(" ")), repr.String(actual, repr.Indent(" ")), scenario.expression)
+		}
+
+	}
+}

pkg/yqlib/lib.go

@@ -6,6 +6,7 @@ import (
 	"bytes"
 	"container/list"
 	"fmt"
+	"math"
 	"strconv"
 	"strings"
 
@@ -24,6 +25,9 @@ func InitExpressionParser() {
 type xmlPreferences struct {
 	AttributePrefix string
 	ContentName     string
+	StrictMode      bool
+	KeepNamespace   bool
+	UseRawToken     bool
 }
 
 var log = logging.MustGetLogger("yq-lib")
@@ -72,7 +76,9 @@ var subtractOpType = &operationType{Type: "SUBTRACT", NumArgs: 2, Precedence: 42
 var alternativeOpType = &operationType{Type: "ALTERNATIVE", NumArgs: 2, Precedence: 42, Handler: alternativeOperator}
 
 var equalsOpType = &operationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: equalsOperator}
-var notEqualsOpType = &operationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: notEqualsOperator}
+var notEqualsOpType = &operationType{Type: "NOT_EQUALS", NumArgs: 2, Precedence: 40, Handler: notEqualsOperator}
+
+var compareOpType = &operationType{Type: "COMPARE", NumArgs: 2, Precedence: 40, Handler: compareOperator}
 
 //createmap needs to be above union, as we use union to build the components of the objects
 var createMapOpType = &operationType{Type: "CREATE_MAP", NumArgs: 2, Precedence: 15, Handler: createMapOperator}
@@ -85,6 +91,7 @@ var columnOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Hand
 
 var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50, Handler: collectOperator}
 var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler: mapOperator}
+var errorOpType = &operationType{Type: "ERROR", NumArgs: 1, Precedence: 50, Handler: errorOperator}
 var pickOpType = &operationType{Type: "PICK", NumArgs: 1, Precedence: 50, Handler: pickOperator}
 var evalOpType = &operationType{Type: "EVAL", NumArgs: 1, Precedence: 50, Handler: evalOperator}
 var mapValuesOpType = &operationType{Type: "MAP_VALUES", NumArgs: 1, Precedence: 50, Handler: mapValuesOperator}
@@ -119,7 +126,7 @@ var getParentOpType = &operationType{Type: "GET_PARENT", NumArgs: 0, Precedence:
 
 var getCommentOpType = &operationType{Type: "GET_COMMENT", NumArgs: 0, Precedence: 50, Handler: getCommentsOperator}
 var getAnchorOpType = &operationType{Type: "GET_ANCHOR", NumArgs: 0, Precedence: 50, Handler: getAnchorOperator}
-var getAliasOptype = &operationType{Type: "GET_ALIAS", NumArgs: 0, Precedence: 50, Handler: getAliasOperator}
+var getAliasOpType = &operationType{Type: "GET_ALIAS", NumArgs: 0, Precedence: 50, Handler: getAliasOperator}
 var getDocumentIndexOpType = &operationType{Type: "GET_DOCUMENT_INDEX", NumArgs: 0, Precedence: 50, Handler: getDocumentIndexOperator}
 var getFilenameOpType = &operationType{Type: "GET_FILENAME", NumArgs: 0, Precedence: 50, Handler: getFilenameOperator}
 var getFileIndexOpType = &operationType{Type: "GET_FILE_INDEX", NumArgs: 0, Precedence: 50, Handler: getFileIndexOperator}
@@ -317,7 +324,7 @@ func deepCloneWithOptions(node *yaml.Node, cloneContent bool) *yaml.Node {
 }
 
 // yaml numbers can be hex encoded...
-func parseInt(numberString string) (string, int64, error) {
+func parseInt64(numberString string) (string, int64, error) {
 	if strings.HasPrefix(numberString, "0x") ||
 		strings.HasPrefix(numberString, "0X") {
 		num, err := strconv.ParseInt(numberString[2:], 16, 64)
@@ -327,6 +334,32 @@ func parseInt(numberString string) (string, int64, error) {
 	return "%v", num, err
 }
 
+func parseInt(numberString string) (int, error) {
+	var err error
+	var parsed int64
+	if strings.HasPrefix(numberString, "0x") ||
+		strings.HasPrefix(numberString, "0X") {
+		parsed, err = strconv.ParseInt(numberString[2:], 16, 64)
+	} else {
+		parsed, err = strconv.ParseInt(numberString, 10, 64)
+	}
+
+	if err != nil {
+		return 0, err
+	} else if parsed > math.MaxInt {
+		return 0, fmt.Errorf("%v is too big (larger than %v)", parsed, math.MaxInt)
+	}
+
+	return int(parsed), err
+}
+
+func createStringScalarNode(stringValue string) *yaml.Node {
+	var node = &yaml.Node{Kind: yaml.ScalarNode}
+	node.Value = stringValue
+	node.Tag = "!!str"
+	return node
+}
+
 func createScalarNode(value interface{}, stringValue string) *yaml.Node {
 	var node = &yaml.Node{Kind: yaml.ScalarNode}
 	node.Value = stringValue

pkg/yqlib/operator_add.go

@@ -56,7 +56,9 @@ func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Candida
 		return lhs.CreateReplacement(rhs.Node), nil
 	}
 
-	target := lhs.CreateReplacement(&yaml.Node{})
+	target := lhs.CreateReplacement(&yaml.Node{
+		Anchor: lhs.Node.Anchor,
+	})
 
 	switch lhsNode.Kind {
 	case yaml.MappingNode:
@@ -103,12 +105,15 @@ func addScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yam
 	} else if lhsTag == "!!str" {
 		target.Node.Tag = lhs.Tag
 		target.Node.Value = lhs.Value + rhs.Value
+	} else if rhsTag == "!!str" {
+		target.Node.Tag = rhs.Tag
+		target.Node.Value = lhs.Value + rhs.Value
 	} else if lhsTag == "!!int" && rhsTag == "!!int" {
-		format, lhsNum, err := parseInt(lhs.Value)
+		format, lhsNum, err := parseInt64(lhs.Value)
 		if err != nil {
 			return err
 		}
-		_, rhsNum, err := parseInt(rhs.Value)
+		_, rhsNum, err := parseInt64(rhs.Value)
 		if err != nil {
 			return err
 		}
@@ -183,7 +188,7 @@ func addMaps(target *CandidateNode, lhsC *CandidateNode, rhsC *CandidateNode) {
 		key := rhs.Content[index]
 		value := rhs.Content[index+1]
 		log.Debug("finding %v", key.Value)
-		indexInLHS := findInArray(target.Node, key)
+		indexInLHS := findKeyInMap(target.Node, key)
 		log.Debug("indexInLhs %v", indexInLHS)
 		if indexInLHS < 0 {
 			// not in there, append it

pkg/yqlib/operator_add_test.go

@@ -14,6 +14,14 @@ var addOperatorScenarios = []expressionScenario{
 			"D0, P[1 a], (!!int)::3\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `a: key`,
+		expression: `. += {"key": "b"}`,
+		expected: []string{
+			"D0, P[], (!!map)::a: key\nkey: b\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `[[c], [b]]`,
@@ -191,6 +199,24 @@ var addOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: catmeow, b: meow}\n",
 		},
 	},
+	{
+		description: "String concatenation - str + int",
+		skipDoc:     true,
+		document:    `{a: !cool cat, b: meow}`,
+		expression:  `.a + 3`,
+		expected: []string{
+			"D0, P[a], (!cool)::cat3\n",
+		},
+	},
+	{
+		description: "String concatenation - int + str",
+		skipDoc:     true,
+		document:    `{a: !cool cat, b: meow}`,
+		expression:  `3 + .a`,
+		expected: []string{
+			"D0, P[], (!cool)::3cat\n",
+		},
+	},
 	{
 		description:    "Number addition - float",
 		subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.",
@@ -315,6 +341,15 @@ var addOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: !horse [a, b]\nb: !goat [b]\n",
 		},
 	},
+	{
+		skipDoc:     true,
+		description: "Keep anchors",
+		document:    "a: &horse [1]",
+		expression:  `.a += 2`,
+		expected: []string{
+			"D0, P[], (doc)::a: &horse [1, 2]\n",
+		},
+	},
 }
 
 func TestAddOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_assign.go

@@ -22,8 +22,10 @@ func assignUpdateOperator(d *dataTreeNavigator, context Context, expressionNode
 	}
 
 	prefs := assignPreferences{}
-	if expressionNode.Operation.Preferences != nil {
-		prefs = expressionNode.Operation.Preferences.(assignPreferences)
+	// they way *= (multipleAssign) is handled, we set the multiplePrefs
+	// on the node, not assignPrefs. Long story.
+	if p, ok := expressionNode.Operation.Preferences.(assignPreferences); ok {
+		prefs = p
 	}
 
 	if !expressionNode.Operation.UpdateAssign {

pkg/yqlib/operator_assign_test.go

@@ -17,7 +17,7 @@ var assignOperatorScenarios = []expressionScenario{
 		document:   "{}",
 		expression: `.a |= .b`,
 		expected: []string{
-			"D0, P[], (doc)::{a: null}\n",
+			"D0, P[], (doc)::a: null\n",
 		},
 	},
 	{
@@ -25,7 +25,7 @@ var assignOperatorScenarios = []expressionScenario{
 		document:   "{}",
 		expression: `.a = .b`,
 		expected: []string{
-			"D0, P[], (doc)::{a: null}\n",
+			"D0, P[], (doc)::a: null\n",
 		},
 	},
 	{
@@ -101,9 +101,9 @@ var assignOperatorScenarios = []expressionScenario{
 	{
 		description: "Updated multiple paths",
 		document:    `{a: fieldA, b: fieldB, c: fieldC}`,
-		expression:  `(.a, .c) = "potatoe"`,
+		expression:  `(.a, .c) = "potato"`,
 		expected: []string{
-			"D0, P[], (doc)::{a: potatoe, b: fieldB, c: potatoe}\n",
+			"D0, P[], (doc)::{a: potato, b: fieldB, c: potato}\n",
 		},
 	},
 	{
@@ -149,7 +149,7 @@ var assignOperatorScenarios = []expressionScenario{
 	},
 	{
 		description:    "Update deeply selected results",
-		subdescription: "Note that the LHS is wrapped in brackets! This is to ensure we dont first filter out the yaml and then update the snippet.",
+		subdescription: "Note that the LHS is wrapped in brackets! This is to ensure we don't first filter out the yaml and then update the snippet.",
 		document:       `{a: {b: apple, c: cactus}}`,
 		expression:     `(.a[] | select(. == "apple")) = "frog"`,
 		expected: []string{
@@ -178,7 +178,7 @@ var assignOperatorScenarios = []expressionScenario{
 		document:              `{}`,
 		expression:            `.a.b |= "bogs"`,
 		expected: []string{
-			"D0, P[], (doc)::{a: {b: bogs}}\n",
+			"D0, P[], (doc)::a:\n    b: bogs\n",
 		},
 	},
 	{
@@ -197,7 +197,7 @@ var assignOperatorScenarios = []expressionScenario{
 		document:              `{}`,
 		expression:            `.a.b.[0] |= "bogs"`,
 		expected: []string{
-			"D0, P[], (doc)::{a: {b: [bogs]}}\n",
+			"D0, P[], (doc)::a:\n    b:\n        - bogs\n",
 		},
 	},
 	{
@@ -205,7 +205,7 @@ var assignOperatorScenarios = []expressionScenario{
 		document:   `{}`,
 		expression: `.a.b.[1].c |= "bogs"`,
 		expected: []string{
-			"D0, P[], (doc)::{a: {b: [null, {c: bogs}]}}\n",
+			"D0, P[], (doc)::a:\n    b:\n        - null\n        - c: bogs\n",
 		},
 	},
 }

pkg/yqlib/operator_collect_object_test.go

@@ -115,12 +115,12 @@ var collectObjectOperatorScenarios = []expressionScenario{
 	},
 	{
 		skipDoc:    true,
-		document:   `{name: Mike, pets: {cows: [apl, bba]}}`,
-		document2:  `{name: Rosey, pets: {sheep: [frog, meow]}}`,
+		document:   "name: Mike\npets:\n  cows:\n    - apl\n    - bba",
+		document2:  "name: Rosey\npets:\n  sheep:\n    - frog\n    - meow",
 		expression: `{"a":.name, "b":.pets}`,
 		expected: []string{
-			"D0, P[], (!!map)::a: Mike\nb: {cows: [apl, bba]}\n",
-			"D0, P[], (!!map)::a: Rosey\nb: {sheep: [frog, meow]}\n",
+			"D0, P[], (!!map)::a: Mike\nb:\n    cows:\n        - apl\n        - bba\n",
+			"D0, P[], (!!map)::a: Rosey\nb:\n    sheep:\n        - frog\n        - meow\n",
 		},
 	},
 	{
@@ -154,6 +154,13 @@ var collectObjectOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!map)::{wrap: {further: {name: Mike}}}\n",
 		},
 	},
+	{
+		description: "Creating yaml from scratch with multiple objects",
+		expression:  `(.a.b = "foo") | (.d.e = "bar")`,
+		expected: []string{
+			"D0, P[], ()::a:\n    b: foo\nd:\n    e: bar\n",
+		},
+	},
 }
 
 func TestCollectObjectOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_collect_test.go

@@ -29,14 +29,23 @@ var collectOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		skipDoc:  true,
-		document: "{a: apple}\n---\n{b: frog}",
-
+		skipDoc:    true,
+		document:   "{a: apple}\n---\n{b: frog}",
 		expression: `[.]`,
 		expected: []string{
 			"D0, P[], (!!seq)::- {a: apple}\n- {b: frog}\n",
 		},
 	},
+	{
+		description: "with comments",
+		skipDoc:     true,
+		document:    "# abc\n[{a: apple}]\n\n# xyz\n",
+
+		expression: `[.[]]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- {a: apple}\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   ``,

pkg/yqlib/operator_comments.go

@@ -61,7 +61,9 @@ func assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNod
 			candidate.Node.HeadComment = comment
 			candidate.LeadingContent = "" // clobber the leading content, if there was any.
 		}
-		if preferences.FootComment {
+		if preferences.FootComment && candidate.Node.Kind == yaml.DocumentNode && comment != "" {
+			candidate.TrailingContent = "# " + comment
+		} else if preferences.FootComment && candidate.Node.Kind != yaml.DocumentNode {
 			candidate.Node.FootComment = comment
 		}
 
@@ -97,6 +99,8 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *
 			comment = chompRegexp.ReplaceAllString(comment, "")
 		} else if preferences.HeadComment {
 			comment = candidate.Node.HeadComment
+		} else if preferences.FootComment && candidate.Node.Kind == yaml.DocumentNode && candidate.TrailingContent != "" {
+			comment = candidate.TrailingContent
 		} else if preferences.FootComment {
 			comment = candidate.Node.FootComment
 		}
@@ -105,6 +109,7 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *
 
 		node := &yaml.Node{Kind: yaml.ScalarNode, Value: comment, Tag: "!!str"}
 		result := candidate.CreateReplacement(node)
+		result.LeadingContent = "" // don't include the leading yaml content when retrieving a comment
 		results.PushBack(result)
 	}
 	return context.ChildContext(results), nil

pkg/yqlib/operator_comments_test.go

@@ -59,7 +59,7 @@ var commentOperatorScenarios = []expressionScenario{
 		document:    `a: cat`,
 		expression:  `. foot_comment=.a`,
 		expected: []string{
-			"D0, P[], (doc)::a: cat\n\n# cat\n",
+			"D0, P[], (doc)::a: cat\n# cat\n",
 		},
 	},
 	{

pkg/yqlib/operator_encoder_decoder.go

@@ -13,9 +13,9 @@ import (
 func configureEncoder(format PrinterOutputFormat, indent int) Encoder {
 	switch format {
 	case JSONOutputFormat:
-		return NewJONEncoder(indent)
+		return NewJONEncoder(indent, false)
 	case PropsOutputFormat:
-		return NewPropertiesEncoder()
+		return NewPropertiesEncoder(true)
 	case CSVOutputFormat:
 		return NewCsvEncoder(',')
 	case TSVOutputFormat:
@@ -104,9 +104,16 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 	case YamlInputFormat:
 		decoder = NewYamlDecoder()
 	case XMLInputFormat:
-		decoder = NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName)
+		decoder = NewXMLDecoder(
+			XMLPreferences.AttributePrefix,
+			XMLPreferences.ContentName,
+			XMLPreferences.StrictMode,
+			XMLPreferences.KeepNamespace,
+			XMLPreferences.UseRawToken)
 	case Base64InputFormat:
 		decoder = NewBase64Decoder()
+	case PropertiesInputFormat:
+		decoder = NewPropertiesDecoder()
 	}
 
 	var results = list.New()

pkg/yqlib/operator_encoder_decoder_test.go

@@ -63,6 +63,14 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 `,
 		},
 	},
+	{
+		description: "Decode props encoded string",
+		document:    `a: "cats=great\ndogs=cool as well"`,
+		expression:  `.a |= from_props`,
+		expected: []string{
+			"D0, P[], (doc)::a:\n    cats: great\n    dogs: cool as well\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   "a:\n  cool:\n    bob: dylan",
@@ -235,6 +243,22 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::coolData:\n    a: apple\n",
 		},
 	},
+	{
+		description: "empty base64 decode",
+		skipDoc:     true,
+		expression:  `"" | @base64d`,
+		expected: []string{
+			"D0, P[], (!!str)::\n",
+		},
+	},
+	{
+		description: "empty xml decode",
+		skipDoc:     true,
+		expression:  `"" | @xmld`,
+		expected: []string{
+			"D0, P[], (!!null)::\n",
+		},
+	},
 }
 
 func TestEncoderDecoderOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_entries.go

@@ -20,7 +20,7 @@ func entrySeqFor(key *yaml.Node, value *yaml.Node) *yaml.Node {
 
 func toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode {
 	var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
-	var entriesNode = candidateNode.CreateReplacement(sequence)
+	var entriesNode = candidateNode.CreateReplacementWithDocWrappers(sequence)
 
 	var contents = unwrapDoc(candidateNode.Node).Content
 	for index := 0; index < len(contents); index = index + 2 {
@@ -34,7 +34,7 @@ func toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode {
 
 func toEntriesfromSeq(candidateNode *CandidateNode) *CandidateNode {
 	var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
-	var entriesNode = candidateNode.CreateReplacement(sequence)
+	var entriesNode = candidateNode.CreateReplacementWithDocWrappers(sequence)
 
 	var contents = unwrapDoc(candidateNode.Node).Content
 	for index := 0; index < len(contents); index = index + 1 {
@@ -72,20 +72,20 @@ func parseEntry(entry *yaml.Node, position int) (*yaml.Node, *yaml.Node, error)
 	prefs := traversePreferences{DontAutoCreate: true}
 	candidateNode := &CandidateNode{Node: entry}
 
-	keyResults, err := traverseMap(Context{}, candidateNode, "key", prefs, false)
+	keyResults, err := traverseMap(Context{}, candidateNode, createStringScalarNode("key"), prefs, false)
 
 	if err != nil {
 		return nil, nil, err
 	} else if keyResults.Len() != 1 {
-		return nil, nil, fmt.Errorf("Expected to find one 'key' entry but found %v in position %v", keyResults.Len(), position)
+		return nil, nil, fmt.Errorf("expected to find one 'key' entry but found %v in position %v", keyResults.Len(), position)
 	}
 
-	valueResults, err := traverseMap(Context{}, candidateNode, "value", prefs, false)
+	valueResults, err := traverseMap(Context{}, candidateNode, createStringScalarNode("value"), prefs, false)
 
 	if err != nil {
 		return nil, nil, err
 	} else if valueResults.Len() != 1 {
-		return nil, nil, fmt.Errorf("Expected to find one 'value' entry but found %v in position %v", valueResults.Len(), position)
+		return nil, nil, fmt.Errorf("expected to find one 'value' entry but found %v in position %v", valueResults.Len(), position)
 	}
 
 	return keyResults.Front().Value.(*CandidateNode).Node, valueResults.Front().Value.(*CandidateNode).Node, nil
@@ -94,7 +94,7 @@ func parseEntry(entry *yaml.Node, position int) (*yaml.Node, *yaml.Node, error)
 
 func fromEntries(candidateNode *CandidateNode) (*CandidateNode, error) {
 	var node = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
-	var mapCandidateNode = candidateNode.CreateReplacement(node)
+	var mapCandidateNode = candidateNode.CreateReplacementWithDocWrappers(node)
 
 	var contents = unwrapDoc(candidateNode.Node).Content
 
@@ -141,9 +141,11 @@ func withEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *
 	var results = list.New()
 
 	for el := toEntries.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+
 		//run expression against entries
 		// splat toEntries and pipe it into Rhs
-		splatted, err := splat(context.SingleChildContext(el.Value.(*CandidateNode)), traversePreferences{})
+		splatted, err := splat(context.SingleChildContext(candidate), traversePreferences{})
 		if err != nil {
 			return Context{}, err
 		}
@@ -160,6 +162,10 @@ func withEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *
 		if err != nil {
 			return Context{}, err
 		}
+		collected.LeadingContent = candidate.LeadingContent
+		collected.TrailingContent = candidate.TrailingContent
+
+		log.Debugf("**** collected %v", collected.LeadingContent)
 
 		fromEntries, err := fromEntriesOperator(d, context.SingleChildContext(collected), expressionNode)
 		if err != nil {

pkg/yqlib/operator_entries_test.go

@@ -62,6 +62,15 @@ var entriesOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!map)::KEY_c: 1\nKEY_d: 2\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `[{a: 1, b: 2}, {c: 1, d: 2}]`,
+		expression: `.[] | with_entries(.key |= "KEY_" + .)`,
+		expected: []string{
+			"D0, P[], (!!map)::KEY_a: 1\nKEY_b: 2\n",
+			"D0, P[], (!!map)::KEY_c: 1\nKEY_d: 2\n",
+		},
+	},
 	{
 		description: "Use with_entries to filter the map",
 		document:    `{a: { b: bird }, c: { d: dog }}`,
@@ -70,6 +79,15 @@ var entriesOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!map)::a: {b: bird}\n",
 		},
 	},
+	{
+		description: "Use with_entries to filter the map; head comment",
+		skipDoc:     true,
+		document:    "# abc\n{a: { b: bird }, c: { d: dog }}\n# xyz",
+		expression:  `with_entries(select(.value | has("b")))`,
+		expected: []string{
+			"D0, P[], (!!map)::# abc\na: {b: bird}\n# xyz\n",
+		},
+	},
 }
 
 func TestEntriesOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_env_test.go

@@ -150,6 +150,16 @@ var envOperatorScenarios = []expressionScenario{
 		expression:    `"the ${notThere} ${alsoNotThere}" | envsubst(nu,ff)`,
 		expectedError: "variable ${notThere} not set",
 	},
+	{
+		description:          "with header/footer",
+		skipDoc:              true,
+		environmentVariables: map[string]string{"myenv": "cat meow"},
+		document:             "# abc\n{v: \"${myenv}\"}\n# xyz\n",
+		expression:           `(.. | select(tag == "!!str")) |= envsubst`,
+		expected: []string{
+			"D0, P[], (!!map)::# abc\n{v: \"cat meow\"}\n# xyz\n",
+		},
+	},
 }
 
 func TestEnvOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_equals_test.go

@@ -130,7 +130,7 @@ var equalsOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description: "Dont match number",
+		description: "Don't match number",
 		document:    `[3, 4, 5]`,
 		expression:  `.[] | (. != 4)`,
 		expected: []string{
@@ -165,7 +165,7 @@ var equalsOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description: "Non exisitant key doesn't equal a value",
+		description: "Non existent key doesn't equal a value",
 		document:    "a: frog",
 		expression:  `select(.b != "thing")`,
 		expected: []string{

pkg/yqlib/operator_error.go

@@ -0,0 +1,20 @@
+package yqlib
+
+import (
+	"fmt"
+)
+
+func errorOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	log.Debugf("-- errorOperation")
+
+	rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS)
+	if err != nil {
+		return Context{}, err
+	}
+	errorMessage := "aborted"
+	if rhs.MatchingNodes.Len() > 0 {
+		errorMessage = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node.Value
+	}
+	return Context{}, fmt.Errorf(errorMessage)
+}

pkg/yqlib/operator_error_test.go

@@ -0,0 +1,40 @@
+package yqlib
+
+import "testing"
+
+const validationExpression = `
+	with(env(numberOfCats); select(tag == "!!int") or error("numberOfCats is not a number :(")) | 
+	.numPets = env(numberOfCats)
+`
+
+var errorOperatorScenarios = []expressionScenario{
+	{
+		description:   "Validate a particular value",
+		document:      `a: hello`,
+		expression:    `select(.a == "howdy") or error(".a [" + .a + "] is not howdy!")`,
+		expectedError: ".a [hello] is not howdy!",
+	},
+	{
+		description:          "Validate the environment variable is a number - invalid",
+		environmentVariables: map[string]string{"numberOfCats": "please"},
+		expression:           `env(numberOfCats) | select(tag == "!!int") or error("numberOfCats is not a number :(")`,
+		expectedError:        "numberOfCats is not a number :(",
+	},
+	{
+		description:          "Validate the environment variable is a number - valid",
+		subdescription:       "`with` can be a convenient way of encapsulating validation.",
+		environmentVariables: map[string]string{"numberOfCats": "3"},
+		document:             "name: Bob\nfavouriteAnimal: cat\n",
+		expression:           validationExpression,
+		expected: []string{
+			"D0, P[], (doc)::name: Bob\nfavouriteAnimal: cat\nnumPets: 3\n",
+		},
+	},
+}
+
+func TestErrorOperatorScenarios(t *testing.T) {
+	for _, tt := range errorOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentOperatorScenarios(t, "error", errorOperatorScenarios)
+}

pkg/yqlib/operator_length_test.go

@@ -32,7 +32,7 @@ var lengthOperatorScenarios = []expressionScenario{
 	},
 	{
 		skipDoc:    true,
-		document:   `{a: key no exist}`,
+		document:   "# abc\n{a: key no exist}",
 		expression: `.b | length`,
 		expected: []string{
 			"D0, P[b], (!!int)::0\n",

pkg/yqlib/operator_multiply.go

@@ -27,7 +27,6 @@ func createMultiplyOp(prefs interface{}) func(lhs *ExpressionNode, rhs *Expressi
 
 func multiplyAssignOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	var multiplyPrefs = expressionNode.Operation.Preferences
-	expressionNode.Operation.Preferences = nil
 
 	return compoundAssignFunction(d, context, expressionNode, createMultiplyOp(multiplyPrefs))
 }
@@ -124,11 +123,11 @@ func multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, e
 	target.Node.Style = lhs.Node.Style
 	target.Node.Tag = lhs.Node.Tag
 
-	format, lhsNum, err := parseInt(lhs.Node.Value)
+	format, lhsNum, err := parseInt64(lhs.Node.Value)
 	if err != nil {
 		return nil, err
 	}
-	_, rhsNum, err := parseInt(rhs.Node.Value)
+	_, rhsNum, err := parseInt64(rhs.Node.Value)
 	if err != nil {
 		return nil, err
 	}

pkg/yqlib/operator_multiply_test.go

@@ -144,7 +144,7 @@ var multiplyOperatorScenarios = []expressionScenario{
 		document2:  docNoComments,
 		expression: `select(fi == 0) * select(fi == 1)`,
 		expected: []string{
-			"D0, P[], (!!map)::a: apple\nb: banana\n\n# footer\n",
+			"D0, P[], (!!map)::a: apple\nb: banana\n# footer\n",
 		},
 	},
 	{
@@ -157,12 +157,13 @@ var multiplyOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		skipDoc:    true,
-		document:   docNoComments,
-		document2:  docWithFooter,
-		expression: `select(fi == 0) * select(fi == 1)`,
+		description: "doc 2 has footer",
+		skipDoc:     true,
+		document:    docNoComments,
+		document2:   docWithFooter,
+		expression:  `select(fi == 0) * select(fi == 1)`,
 		expected: []string{
-			"D0, P[], (!!map)::b: banana\na: apple\n\n# footer\n",
+			"D0, P[], (!!map)::b: banana\na: apple\n# footer\n",
 		},
 	},
 	{
@@ -239,7 +240,7 @@ var multiplyOperatorScenarios = []expressionScenario{
 		document:   `{a: &a { b: &b { c: &c cat } } }`,
 		expression: `{} * .`,
 		expected: []string{
-			"D0, P[], (!!map)::{a: &a {b: &b {c: &c cat}}}\n",
+			"D0, P[], (!!map)::a: &a\n    b: &b\n        c: &c cat\n",
 		},
 	},
 	{
@@ -388,7 +389,7 @@ var multiplyOperatorScenarios = []expressionScenario{
 		document:   `{a: {array: [1]}, b: {}}`,
 		expression: `.b *+ .a`,
 		expected: []string{
-			"D0, P[b], (!!map)::{array: [1]}\n",
+			"D0, P[b], (!!map)::array: [1]\n",
 		},
 	},
 	{

pkg/yqlib/operator_pick.go

@@ -31,12 +31,12 @@ func pickSequence(original *yaml.Node, indices *yaml.Node) (*yaml.Node, error) {
 
 	filteredContent := make([]*yaml.Node, 0)
 	for index := 0; index < len(indices.Content); index = index + 1 {
-		_, indexInArray, err := parseInt(indices.Content[index].Value)
+		indexInArray, err := parseInt(indices.Content[index].Value)
 		if err != nil {
 			return nil, fmt.Errorf("cannot index array with %v", indices.Content[index].Value)
 		}
 
-		if int(indexInArray) > -1 && int(indexInArray) < len(original.Content) {
+		if indexInArray > -1 && indexInArray < len(original.Content) {
 			filteredContent = append(filteredContent, deepClone(original.Content[indexInArray]))
 		}
 	}
@@ -79,7 +79,7 @@ func pickOperator(d *dataTreeNavigator, context Context, expressionNode *Express
 			return Context{}, fmt.Errorf("cannot pick indicies from type %v (%v)", node.Tag, candidate.GetNicePath())
 		}
 
-		results.PushBack(candidate.CreateReplacement(replacement))
+		results.PushBack(candidate.CreateReplacementWithDocWrappers(replacement))
 	}
 
 	return context.ChildContext(results), nil

pkg/yqlib/operator_pick_test.go

@@ -14,6 +14,15 @@ var pickOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::myMap: {hamster: squeek, cat: meow}\n",
 		},
 	},
+	{
+		description: "Pick keys from map with comments",
+		skipDoc:     true,
+		document:    "# abc\nmyMap: {cat: meow, dog: bark, thing: hamster, hamster: squeek}\n# xyz\n",
+		expression:  `.myMap |= pick(["hamster", "cat", "goat"])`,
+		expected: []string{
+			"D0, P[], (doc)::# abc\nmyMap: {hamster: squeek, cat: meow}\n# xyz\n",
+		},
+	},
 	{
 		description:    "Pick indices from array",
 		subdescription: "Note that the order of the indexes matches the pick order and non existent indexes are skipped.",
@@ -23,6 +32,15 @@ var pickOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::[lion, cat]\n",
 		},
 	},
+	{
+		description: "Pick indices from array with comments",
+		skipDoc:     true,
+		document:    "# abc\n[cat, leopard, lion]\n# xyz",
+		expression:  `pick([2, 0, 734, -5])`,
+		expected: []string{
+			"D0, P[], (!!seq)::# abc\n[lion, cat]\n# xyz\n",
+		},
+	},
 }
 
 func TestPickOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_reverse.go

@@ -25,7 +25,7 @@ func reverseOperator(d *dataTreeNavigator, context Context, expressionNode *Expr
 		for i, originalNode := range candidateNode.Content {
 			reverseList.Content[len(candidateNode.Content)-i-1] = originalNode
 		}
-		results.PushBack(candidate.CreateReplacement(reverseList))
+		results.PushBack(candidate.CreateReplacementWithDocWrappers(reverseList))
 
 	}
 

pkg/yqlib/operator_reverse_test.go

@@ -44,6 +44,15 @@ var reverseOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::[{a: cat}, {a: banana}, {a: apple}]\n",
 		},
 	},
+	{
+		description: "Sort descending by string field, with comments",
+		skipDoc:     true,
+		document:    "# abc\n[{a: banana},{a: cat},{a: apple}]\n# xyz",
+		expression:  `sort_by(.a) | reverse`,
+		expected: []string{
+			"D0, P[], (!!seq)::# abc\n[{a: cat}, {a: banana}, {a: apple}]\n# xyz\n",
+		},
+	},
 }
 
 func TestReverseOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_sort.go

@@ -64,7 +64,7 @@ func sortByOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 		for i, sortedNode := range sortableArray {
 			sortedList.Content[i] = sortedNode.Node
 		}
-		results.PushBack(candidate.CreateReplacement(sortedList))
+		results.PushBack(candidate.CreateReplacementWithDocWrappers(sortedList))
 	}
 	return context.ChildContext(results), nil
 }
@@ -106,11 +106,11 @@ func (a sortableNodeArray) Less(i, j int) bool {
 	} else if lhs.Tag != rhs.Tag || lhs.Tag == "!!str" {
 		return strings.Compare(lhs.Value, rhs.Value) < 0
 	} else if lhs.Tag == "!!int" && rhs.Tag == "!!int" {
-		_, lhsNum, err := parseInt(lhs.Value)
+		_, lhsNum, err := parseInt64(lhs.Value)
 		if err != nil {
 			panic(err)
 		}
-		_, rhsNum, err := parseInt(rhs.Value)
+		_, rhsNum, err := parseInt64(rhs.Value)
 		if err != nil {
 			panic(err)
 		}

pkg/yqlib/operator_sort_test.go

@@ -79,6 +79,15 @@ var sortByOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::[{a: false, b: 1}, {a: false, b: 3}, {a: true, b: 2}]\n",
 		},
 	},
+	{
+		skipDoc:     true,
+		description: "head comment",
+		document:    "# abc\n- def\n# ghi",
+		expression:  `sort`,
+		expected: []string{
+			"D0, P[], (!!seq)::# abc\n- def\n# ghi\n",
+		},
+	},
 }
 
 func TestSortByOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_strings_test.go

@@ -165,7 +165,7 @@ var stringsOperatorScenarios = []expressionScenario{
 	},
 	{
 		description:    "Test using regex",
-		subdescription: "Like jq'q equivalent, this works like match but only returns true/false instead of full match details",
+		subdescription: "Like jq's equivalent, this works like match but only returns true/false instead of full match details",
 		document:       `["cat", "dog"]`,
 		expression:     `.[] | test("at")`,
 		expected: []string{

pkg/yqlib/operator_subtract.go

@@ -104,11 +104,11 @@ func subtractScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs
 	} else if lhsTag == "!!str" {
 		return fmt.Errorf("strings cannot be subtracted")
 	} else if lhsTag == "!!int" && rhsTag == "!!int" {
-		format, lhsNum, err := parseInt(lhs.Value)
+		format, lhsNum, err := parseInt64(lhs.Value)
 		if err != nil {
 			return err
 		}
-		_, rhsNum, err := parseInt(rhs.Value)
+		_, rhsNum, err := parseInt64(rhs.Value)
 		if err != nil {
 			return err
 		}

pkg/yqlib/operator_tag_test.go

@@ -18,6 +18,19 @@ var tagOperatorScenarios = []expressionScenario{
 			"D0, P[f], (!!str)::!!seq\n",
 		},
 	},
+	{
+		description: "type is an alias for tag",
+		document:    `{a: cat, b: 5, c: 3.2, e: true, f: []}`,
+		expression:  `.. | type`,
+		expected: []string{
+			"D0, P[], (!!str)::!!map\n",
+			"D0, P[a], (!!str)::!!str\n",
+			"D0, P[b], (!!str)::!!int\n",
+			"D0, P[c], (!!str)::!!float\n",
+			"D0, P[e], (!!str)::!!bool\n",
+			"D0, P[f], (!!str)::!!seq\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `{a: cat, b: 5, c: 3.2, e: true, f: []}`,
@@ -42,6 +55,15 @@ var tagOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: !!mikefarah str}\n",
 		},
 	},
+	{
+		skipDoc:     true,
+		description: "Set custom type",
+		document:    `{a: str}`,
+		expression:  `.a type = "!!mikefarah"`,
+		expected: []string{
+			"D0, P[], (doc)::{a: !!mikefarah str}\n",
+		},
+	},
 	{
 		description: "Find numbers and convert them to strings",
 		document:    `{a: cat, b: 5, c: 3.2, e: true}`,