action uses built docker image

1. Escaping behavior changed as a result of set-output deprecation.
-  Otherwise you'll encounter breaking changes for example "appVersion: "1.0.0"%0Aapplica..."
   where %0A is escaped newline \n but shouldn't be escaped.

2. Delimeter is required for multiline strings.
-  See here: https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings
-  Delimeter must be random. UUID is a solid choice.
   I verified that /proc/sys/kernel/random/uuid does indeed exist on the yq image.

.github/workflows/codeql.yml

@@ -0,0 +1,70 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '24 3 * * 1'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2

.github/workflows/docker-release-github-action.yml

@@ -0,0 +1,63 @@
+name: Release Github action Docker
+
+on:
+  release:
+    types: [released]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  publishDocker:
+    environment: dockerhub
+    env:
+      IMAGE_NAME: mikefarah/yq
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: all
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          version: latest
+
+      - name: Available platforms
+        run: echo ${{ steps.buildx.outputs.platforms }} && docker version
+
+      - name: Build and push image
+        run: |
+          echo "GithubRef: ${GITHUB_REF}"
+          VERSION=${GITHUB_REF##*/}
+          echo "VERSION: ${VERSION}"
+          IMAGE_VERSION=${VERSION:1}
+          echo "IMAGE_VERSION: ${IMAGE_VERSION}"
+
+          PLATFORMS="linux/amd64,linux/ppc64le,linux/arm64"
+
+          echo "Building and pushing version ${IMAGE_VERSION} of image ${IMAGE_NAME}"
+          echo '${{ secrets.DOCKER_PASSWORD }}' | docker login -u '${{ secrets.DOCKER_USERNAME }}' --password-stdin
+          cd github-action
+          docker buildx build \
+            --label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
+            --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
+            --label "org.opencontainers.image.description=yq is a portable command-line YAML processor" \
+            --label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
+            --label "org.opencontainers.image.licenses=MIT" \
+            --label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \
+            --label "org.opencontainers.image.source=https://github.com/mikefarah/yq" \
+            --label "org.opencontainers.image.title=yq" \
+            --label "org.opencontainers.image.url=https://mikefarah.gitbook.io/yq/" \
+            --label "org.opencontainers.image.version=${IMAGE_VERSION}" \
+            --platform "${PLATFORMS}" \
+            --pull \
+            --push \
+            -t "${IMAGE_NAME}:${IMAGE_VERSION}-githubaction" \
+            -t "${IMAGE_NAME}:4-githubaction" \
+            -t "${IMAGE_NAME}:latest-githubaction" \
+            .

.github/workflows/docker-release.yml

@@ -0,0 +1,62 @@
+name: Release Docker
+
+on:
+  release:
+    types: [released]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  publishDocker:
+    environment: dockerhub
+    env:
+      IMAGE_NAME: mikefarah/yq
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+        with:
+          platforms: all
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          version: latest
+
+      - name: Available platforms
+        run: echo ${{ steps.buildx.outputs.platforms }} && docker version
+
+      - name: Build and push image
+        run: |
+          echo "GithubRef: ${GITHUB_REF}"
+          VERSION=${GITHUB_REF##*/}
+          echo "VERSION: ${VERSION}"
+          IMAGE_VERSION=${VERSION:1}
+          echo "IMAGE_VERSION: ${IMAGE_VERSION}"
+
+          PLATFORMS="linux/amd64,linux/ppc64le,linux/arm64"
+
+          echo "Building and pushing version ${IMAGE_VERSION} of image ${IMAGE_NAME}"
+          echo '${{ secrets.DOCKER_PASSWORD }}' | docker login -u '${{ secrets.DOCKER_USERNAME }}' --password-stdin
+          docker buildx build \
+            --label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
+            --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
+            --label "org.opencontainers.image.description=yq is a portable command-line YAML processor" \
+            --label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
+            --label "org.opencontainers.image.licenses=MIT" \
+            --label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \
+            --label "org.opencontainers.image.source=https://github.com/mikefarah/yq" \
+            --label "org.opencontainers.image.title=yq" \
+            --label "org.opencontainers.image.url=https://mikefarah.gitbook.io/yq/" \
+            --label "org.opencontainers.image.version=${IMAGE_VERSION}" \
+            --platform "${PLATFORMS}" \
+            --pull \
+            --push \
+            -t "${IMAGE_NAME}:${IMAGE_VERSION}" \
+            -t "${IMAGE_NAME}:4" \
+            -t "${IMAGE_NAME}:latest" \
+            .

.github/workflows/go.yml

@@ -1,5 +1,8 @@
 name: Build
 on: [push, pull_request]
+permissions:
+  contents: read
+
 jobs:
 
   build:
@@ -7,10 +10,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.17
-      uses: actions/setup-go@v2
+    - name: Set up Go 1.19
+      uses: actions/setup-go@v3
       with:
-        go-version: 1.17
+        go-version: 1.19
       id: go
 
     - name: Check out code into the Go module directory

.github/workflows/release.yml

@@ -3,15 +3,16 @@ on:
   push:
     tags:
       - 'v*'
+      - 'draft-*'
 
 jobs:
   publishGitRelease:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
-          go-version: '^1.17.5'
+          go-version: '^1.19'
       - name: Compile man page markup
         id: gen-man-page-md
         run: |
@@ -39,72 +40,14 @@ jobs:
       - name: Cross compile
         run: |
           sudo apt-get install rhash -y
-          go get github.com/mitchellh/gox
+          go install github.com/mitchellh/gox@v1.0.1
           mkdir -p build
           cp yq.1 build/yq.1
           ./scripts/xcompile.sh
 
-      - name: Create Release
-        id: create_release
-        uses: actions/create-release@v1.1.4
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Release
+        uses: softprops/action-gh-release@v1
         with:
-          tag_name: ${{ github.ref }}
-          release_name: ${{ github.ref }}
+          files: build/*
           draft: true
-          prerelease: false
-
-      - uses: shogo82148/actions-upload-release-asset@v1
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: build/*
-
-  publishDocker:
-    environment: dockerhub
-    env:
-      IMAGE_NAME: mikefarah/yq
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-        with:
-          platforms: all
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          version: latest
-
-      - name: Available platforms
-        run: echo ${{ steps.buildx.outputs.platforms }} && docker version
-
-      - name: Build and push image
-        run: |
-          IMAGE_V_VERSION="$(git describe --tags --abbrev=0)"
-          IMAGE_VERSION=${IMAGE_V_VERSION:1}
-          PLATFORMS="linux/amd64,linux/ppc64le,linux/arm64"
-
-          echo "Building and pushing version ${IMAGE_VERSION} of image ${IMAGE_NAME}"
-          echo '${{ secrets.DOCKER_PASSWORD }}' | docker login -u '${{ secrets.DOCKER_USERNAME }}' --password-stdin
-          docker buildx build \
-            --label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
-            --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
-            --label "org.opencontainers.image.description=yq is a portable command-line YAML processor" \
-            --label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
-            --label "org.opencontainers.image.licenses=MIT" \
-            --label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \
-            --label "org.opencontainers.image.source=https://github.com/mikefarah/yq" \
-            --label "org.opencontainers.image.title=yq" \
-            --label "org.opencontainers.image.url=https://mikefarah.gitbook.io/yq/" \
-            --label "org.opencontainers.image.version=${IMAGE_VERSION}" \
-            --platform "${PLATFORMS}" \
-            --pull \
-            --push \
-            -t "${IMAGE_NAME}:${IMAGE_VERSION}" \
-            -t "${IMAGE_NAME}:4" \
-            -t "${IMAGE_NAME}:latest" \
-            .
+          fail_on_unmatched_files: true

.github/workflows/test-yq.yml

@@ -1,5 +1,8 @@
 name: Test Yq Action
 on: [push]
+permissions:
+  contents: read
+
 jobs:
 
   build:
@@ -9,11 +12,14 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Get test
+        id: get_value
         uses: mikefarah/yq@master
         with:
-          cmd: yq eval '.a' examples/sample.yaml
+          cmd: yq '.test' examples/multiline-text.yaml
+      - name: Multiline test
+        run: echo "### It was [${{ steps.get_value.outputs.result }}]" >> $GITHUB_STEP_SUMMARY
       - name: Write inplace test
         id: lookupSdkVersion
         uses: mikefarah/yq@master
         with:
-          cmd: yq eval -i '.b.c = 5' examples/sample.yaml
+          cmd: yq -i '.b.c = 5' examples/sample.yaml

.gitignore

@@ -41,6 +41,7 @@ yq*.snap
 
 test.yml
 test*.yml
+test*.yaml
 0.yml
 1.yml
 2.yml
@@ -53,3 +54,11 @@ yq.1
 # debian pkg
 _build
 debian/files
+
+# intellij
+/.idea
+
+# vscode
+.vscode
+
+yq3

CONTRIBUTING.md

@@ -1,3 +1,5 @@
+# Development
+
 1. Install (golang)[https://golang.org/]
 1. Run  `scripts/devtools.sh` to install the required devtools
 2. Run `make [local] vendor` to install the vendor dependencies
@@ -6,3 +8,33 @@
 4. Make the code changes.
 5. `make [local] test` to lint code and run tests
 6. Profit! ok no profit, but raise a PR and get kudos :)
+
+
+# Documentation
+
+The documentation is a bit of a mixed bag (sorry in advanced, I do plan on simplifying it...) - with some parts automatically generated and stiched together and some statically defined.
+
+Documentation is written in markdown, and is published in the 'gitbook' branch.
+
+The various operator documentation (e.g. 'strings') are generated from the 'master' branch, and have a statically defined header (e.g. `pkg/yqlib/doc/operators/headers/add.md`) and the bulk of the docs are generated from the unit tests e.g. `pkg/yqlib/operator_add_test.go`.
+
+The pipeline will run the tests and automatically concatenate the files together, and put them under 
+`pkg/qylib/doc/add.md`. These files are checked in the master branch (and are copied to the gitbook branch as part of the release process).
+
+## How to contribute
+
+The first step is to find if what you want is automatically generated or not - start by looking in the master branch. 
+
+### Updating dynamic documentation from master
+- Search for the documentation you want to update. If you find matches in a `*_test.go` file - update that, as that will automatically update the matching `*.md` file 
+- Assuming you are updating a `*_test.go` file, once updated, run the test to regenerated the docs. E.g. for the 'Add' test generated docs, from the pkg/yqlib folder run:
+`go test -run TestAddOperatorScenarios` which will run that test defined in the `operator_add_test.go` file.
+- Ensure the tests still pass, and check the generated documentation have your update.
+- Note: If the documentation is only in a `headers/*.md` file, then just update that directly
+- Raise a PR to merge the changes into master!
+
+### Updating static documentation from the gitbook branch
+If you haven't found what you want to update in the master branch, then check the gitbook branch directly as there are a few pages in there that are not in master.
+
+- Update the `*.md` files
+- Raise a PR to merge the changes into gitbook.

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.18.0 as builder
+FROM golang:1.19.2 as builder
 
 WORKDIR /go/src/mikefarah/yq
 

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM golang:1.18.0
+FROM golang:1.19.2
 
 COPY scripts/devtools.sh /opt/devtools.sh
 

README.md

@@ -1,9 +1,9 @@
 # yq
 
-![Build](https://github.com/mikefarah/yq/workflows/Build/badge.svg)  ![Docker Pulls](https://img.shields.io/docker/pulls/mikefarah/yq.svg) ![Github Releases (by Release)](https://img.shields.io/github/downloads/mikefarah/yq/total.svg) ![Go Report](https://goreportcard.com/badge/github.com/mikefarah/yq)
+![Build](https://github.com/mikefarah/yq/workflows/Build/badge.svg)  ![Docker Pulls](https://img.shields.io/docker/pulls/mikefarah/yq.svg) ![Github Releases (by Release)](https://img.shields.io/github/downloads/mikefarah/yq/total.svg) ![Go Report](https://goreportcard.com/badge/github.com/mikefarah/yq) ![CodeQL](https://github.com/mikefarah/yq/workflows/CodeQL/badge.svg)
 
 
-a lightweight and portable command-line YAML, JSON and XML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json and xml. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.
+a lightweight and portable command-line YAML, JSON and XML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json, xml, properties, csv and tsv. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.
 
 yq is written in go - so you can download a dependency free binary for your platform and you are good to go! If you prefer there are a variety of package managers that can be used as well as Docker and Podman, all listed below.
 
@@ -18,12 +18,12 @@ TLDR:
 
 Prior to 4.18.1 
 ```bash
-cat file.yaml | yq e '.cool' -
+yq e '.cool' - < file.yaml
 ```
 
 4.18+ 
 ```bash
-cat file.yaml | yq '.cool'
+yq '.cool' < file.yaml
 ```
 
 When merging multiple files together, `eval-all/ea` is still required to tell `yq` to run the expression against all the document at once.
@@ -37,7 +37,7 @@ yq '.a.b[0].c' file.yaml
 
 Pipe from STDIN:
 ```bash
-cat file.yaml | yq '.a.b[0].c'
+yq '.a.b[0].c' < file.yaml
 ```
 
 Update a yaml file, inplace
@@ -73,6 +73,8 @@ yq -P sample.json
 
 See the [documentation](https://mikefarah.gitbook.io/yq/) for more examples.
 
+Take a look at the discussions for [common questions](https://github.com/mikefarah/yq/discussions/categories/q-a), and [cool ideas](https://github.com/mikefarah/yq/discussions/categories/show-and-tell)
+
 ## Install
 
 ### [Download the latest binary](https://github.com/mikefarah/yq/releases/latest)
@@ -143,11 +145,11 @@ podman run --rm -v "${PWD}":/workdir mikefarah/yq [command] [flags] [expression
 You'll need to pass the `-i\--interactive` flag to docker:
 
 ```bash
-cat myfile.yml | docker run -i --rm mikefarah/yq '.this.thing'
+docker run -i --rm mikefarah/yq '.this.thing' < myfile.yml
 ```
 
 ```bash
-cat myfile.yml | podman run -i --rm mikefarah/yq '.this.thing'
+podman run -i --rm mikefarah/yq '.this.thing' < myfile.yml
 ```
 
 #### Run commands interactively:
@@ -207,19 +209,34 @@ RUN apk add --no-cache tzdata
 USER yq
 ```
 
+#### Podman with SELinux
+
+If you are using podman with SELinux, you will need to set the shared volume flag `:z` on the volume mount:
+
+```
+-v "${PWD}":/workdir:z
+```
+
 ### GitHub Action
 ```
   - name: Set foobar to cool
     uses: mikefarah/yq@master
     with:
       cmd: yq -i '.foo.bar = "cool"' 'config.yml'
+  - name: Get an entry with a variable that might contain dots or spaces
+    id: get_username
+    uses: mikefarah/yq@master
+    with:
+      cmd: yq '.all.children.["${{ matrix.ip_address }}"].username' ops/inventories/production.yml
+  - name: Reuse a variable obtained in another step
+    run: echo ${{ steps.get_username.outputs.result }}
 ```
 
 See https://mikefarah.gitbook.io/yq/usage/github-action for more.
 
-### Go Get:
+### Go Install:
 ```
-GO111MODULE=on go get github.com/mikefarah/yq/v4
+go install github.com/mikefarah/yq/v4@latest
 ```
 
 ## Community Supported Installation methods
@@ -248,6 +265,13 @@ choco install yq
 ```
 Supported by @chillum (https://chocolatey.org/packages/yq)
 
+and
+
+### Winget
+winget install yq
+
+https://winget.run/pkg/MikeFarah/yq
+
 ### Mac:
 Using [MacPorts](https://www.macports.org/)
 ```
@@ -290,10 +314,10 @@ Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
 - Keeps yaml formatting and comments when updating (though there are issues with whitespace)
 - [Decode/Encode base64 data](https://mikefarah.gitbook.io/yq/operators/encode-decode)
 - [Load content from other files](https://mikefarah.gitbook.io/yq/operators/load)
-- [Convert to/from json](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
+- [Convert to/from json/ndjson](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
 - [Convert to/from xml](https://mikefarah.gitbook.io/yq/v/v4.x/usage/xml)
 - [Convert to/from properties](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties)
-- [Convert to csv/tsv](https://mikefarah.gitbook.io/yq/usage/csv-tsv)
+- [Convert to/from csv/tsv](https://mikefarah.gitbook.io/yq/usage/csv-tsv)
 - [General shell completion scripts (bash/zsh/fish/powershell)](https://mikefarah.gitbook.io/yq/v/v4.x/commands/shell-completion)
 - [Reduce](https://mikefarah.gitbook.io/yq/operators/reduce) to merge multiple files or sum an array or other fancy things.
 - [Github Action](https://mikefarah.gitbook.io/yq/usage/github-action) to use in your automated pipeline (thanks @devorbitus)
@@ -310,7 +334,7 @@ Usage:
 Examples:
 
 # yq defaults to 'eval' command if no command is specified. See "yq eval --help" for more examples.
-cat myfile.yml | yq '.stuff' # outputs the data at the "stuff" node from "myfile.yml"
+yq '.stuff' < myfile.yml # outputs the data at the "stuff" node from "myfile.yml"
 
 yq -i '.stuff = "foo"' myfile.yml # update myfile.yml inplace
 

acceptance_tests/basic.sh

@@ -12,6 +12,62 @@ testBasicEvalRoundTrip() {
   assertEquals 123 "$X"
 }
 
+testBasicTrailingContent() {
+  cat >test-trailing.yml <<EOL
+test:
+# this comment will be removed
+EOL
+
+  read -r -d '' expected << EOM
+test:
+# this comment will be removed
+EOM
+  X=$(./yq test-trailing.yml -P)
+  assertEquals "$expected" "$X"
+}
+
+testBasicTrailingContent() {
+  cat >test-trailing.yml <<EOL
+test:
+# this comment will be removed
+EOL
+
+  read -r -d '' expected << EOM
+test:
+# hi
+EOM
+  X=$(./yq '. footComment = "hi"' test-trailing.yml)
+  assertEquals "$expected" "$X"
+}
+
+testBasicTrailingContentEvalAll() {
+  cat >test-trailing.yml <<EOL
+test:
+# this comment will be removed
+EOL
+
+  read -r -d '' expected << EOM
+test:
+# this comment will be removed
+EOM
+  X=$(./yq ea test-trailing.yml -P)
+  assertEquals "$expected" "$X"
+}
+
+testBasicTrailingContentEvalAll() {
+  cat >test-trailing.yml <<EOL
+test:
+# this comment will be removed
+EOL
+
+  read -r -d '' expected << EOM
+test:
+# hi
+EOM
+  X=$(./yq ea '. footComment = "hi"' test-trailing.yml)
+  assertEquals "$expected" "$X"
+}
+
 testBasicPipeWithDot() {
   ./yq -n ".a = 123" > test.yml
   X=$(cat test.yml | ./yq '.')
@@ -86,6 +142,12 @@ testBasicCatWithFilesNoDash() {
   assertEquals "$Y" "$X"
 }
 
+# when the nullinput flag is used
+# dont automatically read STDIN (this breaks github actions)
+testBasicCreateFileGithubAction() {
+  cat /dev/null | ./yq -n ".a = 123" > test.yml
+}
+
 testBasicEvalAllCatWithFilesNoDash() {
   ./yq -n ".a = 123" > test.yml
   ./yq -n ".a = 124" > test2.yml
@@ -188,6 +250,40 @@ EOL
   assertEquals "10" "$X"
 }
 
+testBasicUpdateInPlaceMultipleFilesNoExpressionEval() {
+  cat >test.yml <<EOL
+a: 0
+EOL
+  cat >test2.yml <<EOL
+a: 1
+EOL
+read -r -d '' expected << EOM
+0
+---
+1
+EOM
+  ./yq -i test.yml test2.yml
+  X=$(./yq e '.a' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testBasicUpdateInPlaceMultipleFilesNoExpressionEvalAll() {
+  cat >test.yml <<EOL
+a: 0
+EOL
+  cat >test2.yml <<EOL
+a: 1
+EOL
+read -r -d '' expected << EOM
+0
+---
+1
+EOM
+  ./yq -i ea test.yml test2.yml
+  X=$(./yq e '.a' test.yml)
+  assertEquals "$expected" "$X"
+}
+
 testBasicNoExitStatus() {
   echo "a: cat" > test.yml
   X=$(./yq e '.z' test.yml)
@@ -235,4 +331,26 @@ EOM
   assertEquals "$expected" "$X"
 }
 
+testBasicMultiplyAssignMultiDoc() {
+      cat >test.yml <<EOL
+a: 1
+---
+b: 2
+EOL
+
+read -r -d '' expected << EOM
+a: 1
+c: 3
+---
+b: 2
+c: 3
+EOM
+
+
+  X=$(./yq '. *= {"c":3}' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+
+
 source ./scripts/shunit2

acceptance_tests/header-processing-off.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+setUp() {
+  rm test*.yml || true
+  
+}
+
+testLineCountFirstLineComment() {
+  cat >test.yml <<EOL
+#test123 
+abc: 123 
+test123: 123123 
+#comment 
+lalilu: lalilu
+EOL
+
+  X=$(./yq '.lalilu | line' --header-preprocess=false < test.yml)
+  assertEquals "5" "$X"
+}
+
+testArrayOfDocs() {
+  cat >test.yml <<EOL
+---
+# leading comment doc 1
+a: 1
+---
+# leading comment doc 2
+a: 2
+EOL
+
+read -r -d '' expected << EOM
+- # leading comment doc 1
+  a: 1
+- # leading comment doc 2
+  a: 2
+EOM
+
+  X=$(./yq ea '[.]' --header-preprocess=false < test.yml)
+  assertEquals "$expected" "$X"
+
+}
+
+source ./scripts/shunit2

acceptance_tests/inputs-format.sh

@@ -3,6 +3,8 @@
 setUp() {
   rm test*.yml 2>/dev/null || true
   rm test*.properties 2>/dev/null || true
+  rm test*.csv 2>/dev/null || true
+  rm test*.tsv 2>/dev/null || true
   rm test*.xml 2>/dev/null || true
 }
 
@@ -40,6 +42,65 @@ EOM
   assertEquals "$expected" "$X"
 }
 
+testInputCSV() {
+  cat >test.csv <<EOL
+fruit,yumLevel
+apple,5
+banana,4
+EOL
+
+  read -r -d '' expected << EOM
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOM
+
+  X=$(./yq e -p=csv test.csv)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -p=csv test.csv)
+  assertEquals "$expected" "$X"
+}
+
+testInputCSVUTF8() {
+  read -r -d '' expected << EOM
+- id: 1
+  first: john
+  last: smith
+- id: 1
+  first: jane
+  last: smith
+EOM
+
+  X=$(./yq -p=csv utf8.csv)
+  assertEquals "$expected" "$X"
+}
+
+testInputTSV() {
+  cat >test.tsv <<EOL
+fruit	yumLevel
+apple	5
+banana	4
+EOL
+
+  read -r -d '' expected << EOM
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOM
+
+  X=$(./yq e -p=t test.tsv)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -p=t test.tsv)
+  assertEquals "$expected" "$X"
+}
+
+
+
+
 testInputXml() {
   cat >test.yml <<EOL
 <cat legs="4">BiBi</cat>
@@ -58,6 +119,48 @@ EOM
   assertEquals "$expected" "$X"
 }
 
+testInputXmlNamespaces() {
+  cat >test.yml <<EOL
+<?xml version="1.0"?>
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
+</map>
+EOL
+
+  read -r -d '' expected << EOM
+map:
+  +xmlns: some-namespace
+  +xmlns:xsi: some-instance
+  +xsi:schemaLocation: some-url
+EOM
+
+  X=$(./yq e -p=xml test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -p=xml test.yml)
+  assertEquals "$expected" "$X"
+}
+
+
+testInputXmlStrict() {
+  cat >test.yml <<EOL
+<?xml version="1.0"?>
+<!DOCTYPE root [
+<!ENTITY writer "Catherine.">
+<!ENTITY copyright "(r) Great">
+]>
+<root>
+    <item>&writer;&copyright;</item>
+</root>
+EOL
+
+  X=$(./yq -p=xml --xml-strict-mode test.yml 2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: bad file 'test.yml': XML syntax error on line 7: invalid character entity &writer;" "$X"
+
+  X=$(./yq ea -p=xml --xml-strict-mode test.yml 2>&1)
+  assertEquals "Error: bad file 'test.yml': XML syntax error on line 7: invalid character entity &writer;" "$X"
+}
+
 testInputXmlGithubAction() {
   cat >test.yml <<EOL
 <cat legs="4">BiBi</cat>

acceptance_tests/leading-seperator.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+
+# examples where header-preprocess is required
+
 setUp() {
   rm test*.yml || true
   cat >test.yml <<EOL

acceptance_tests/output-format.sh

@@ -102,6 +102,48 @@ EOM
   assertEquals "$expected" "$X"
 }
 
+testOutputCSV() {
+  cat >test.yml <<EOL
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOL
+
+  read -r -d '' expected << EOM
+fruit,yumLevel
+apple,5
+banana,4
+EOM
+
+  X=$(./yq -o=c test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -o=csv test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testOutputTSV() {
+  cat >test.yml <<EOL
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOL
+
+  read -r -d '' expected << EOM
+fruit	yumLevel
+apple	5
+banana	4
+EOM
+
+  X=$(./yq -o=t test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -o=tsv test.yml)
+  assertEquals "$expected" "$X"
+}
+
 testOutputXml() {
   cat >test.yml <<EOL
 a: {b: {c: ["cat"]}}

acceptance_tests/split-printer.sh

@@ -25,6 +25,56 @@ EOM
   assertEquals "$expectedDoc2" "$doc2"
 }
 
+testBasicSplitWithNameCustomExtension() {
+  rm test*.yaml || true
+  cat >test.yml <<EOL
+a: test_doc1
+--- 
+a: test_doc2
+EOL
+
+  ./yq e test.yml -s '.a + ".yaml"'
+
+  doc1=$(cat test_doc1.yaml)
+  
+  assertEquals "a: test_doc1" "$doc1"
+
+  doc2=$(cat test_doc2.yaml)
+  read -r -d '' expectedDoc2 << EOM
+---
+a: test_doc2
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+}
+
+
+
+
+testSplitFromFile() {
+  cat >test.yml <<EOL
+a: test_doc1
+--- 
+a: test_doc2
+EOL
+
+cat >test_splitExp.yml <<EOL
+.a
+EOL
+
+  ./yq test.yml --split-exp-file test_splitExp.yml
+
+  doc1=$(cat test_doc1.yml)
+  
+  assertEquals "a: test_doc1" "$doc1"
+
+  doc2=$(cat test_doc2.yml)
+  read -r -d '' expectedDoc2 << EOM
+---
+a: test_doc2
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+}
+
 testBasicSplitWithNameEvalAll() {
   cat >test.yml <<EOL
 a: test_doc1

action.yml

@@ -12,6 +12,6 @@ outputs:
     description: "The complete result from the yq command being run"
 runs:
   using: 'docker'
-  image: 'github-action/Dockerfile'
+  image: 'docker://mikefarah/yq:4-githubaction'
   args:
     - ${{ inputs.cmd }}

cmd/constant.go

@@ -10,6 +10,9 @@ var inputFormat = "yaml"
 
 var xmlAttributePrefix = "+"
 var xmlContentName = "+content"
+var xmlStrictMode = false
+var xmlKeepNamespace = true
+var xmlUseRawToken = true
 
 var exitStatus = false
 var forceColor = false
@@ -26,6 +29,7 @@ var prettyPrint = false
 var frontMatter = ""
 
 var splitFileExp = ""
+var splitFileExpFile = ""
 
 var completedSuccessfully = false
 

cmd/evaluate_all_command.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"errors"
-	"os"
 
 	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"github.com/spf13/cobra"
@@ -47,36 +46,17 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 
 	var err error
 
-	firstFileIndex, err := initCommand(cmd, args)
+	expression, args, err := initCommand(cmd, args)
 	if err != nil {
 		return err
 	}
 
-	stat, _ := os.Stdin.Stat()
-	pipingStdIn := (stat.Mode() & os.ModeCharDevice) == 0
-	yqlib.GetLogger().Debug("pipingStdIn: %v", pipingStdIn)
-
-	yqlib.GetLogger().Debug("stat.Mode(): %v", stat.Mode())
-	yqlib.GetLogger().Debug("ModeDir: %v", stat.Mode()&os.ModeDir)
-	yqlib.GetLogger().Debug("ModeAppend: %v", stat.Mode()&os.ModeAppend)
-	yqlib.GetLogger().Debug("ModeExclusive: %v", stat.Mode()&os.ModeExclusive)
-	yqlib.GetLogger().Debug("ModeTemporary: %v", stat.Mode()&os.ModeTemporary)
-	yqlib.GetLogger().Debug("ModeSymlink: %v", stat.Mode()&os.ModeSymlink)
-	yqlib.GetLogger().Debug("ModeDevice: %v", stat.Mode()&os.ModeDevice)
-	yqlib.GetLogger().Debug("ModeNamedPipe: %v", stat.Mode()&os.ModeNamedPipe)
-	yqlib.GetLogger().Debug("ModeSocket: %v", stat.Mode()&os.ModeSocket)
-	yqlib.GetLogger().Debug("ModeSetuid: %v", stat.Mode()&os.ModeSetuid)
-	yqlib.GetLogger().Debug("ModeSetgid: %v", stat.Mode()&os.ModeSetgid)
-	yqlib.GetLogger().Debug("ModeCharDevice: %v", stat.Mode()&os.ModeCharDevice)
-	yqlib.GetLogger().Debug("ModeSticky: %v", stat.Mode()&os.ModeSticky)
-	yqlib.GetLogger().Debug("ModeIrregular: %v", stat.Mode()&os.ModeIrregular)
-
 	out := cmd.OutOrStdout()
 
 	if writeInplace {
 		// only use colors if its forced
 		colorsEnabled = forceColor
-		writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[firstFileIndex])
+		writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[0])
 		out, err = writeInPlaceHandler.CreateTempFile()
 		if err != nil {
 			return err
@@ -109,12 +89,12 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 	printer := yqlib.NewPrinter(encoder, printerWriter)
 
 	if frontMatter != "" {
-		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])
+		frontMatterHandler := yqlib.NewFrontMatterHandler(args[0])
 		err = frontMatterHandler.Split()
 		if err != nil {
 			return err
 		}
-		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
+		args[0] = frontMatterHandler.GetYamlFrontMatterFilename()
 
 		if frontMatter == "process" {
 			reader := frontMatterHandler.GetContentReader()
@@ -126,12 +106,6 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 
 	allAtOnceEvaluator := yqlib.NewAllAtOnceEvaluator()
 
-	expression, args, err := processArgs(pipingStdIn, args)
-	if err != nil {
-		return err
-	}
-	yqlib.GetLogger().Debugf("processed args: %v", args)
-
 	switch len(args) {
 	case 0:
 		if nullInput {

cmd/evalute_sequence_command.go

@@ -3,7 +3,6 @@ package cmd
 import (
 	"errors"
 	"fmt"
-	"os"
 
 	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"github.com/spf13/cobra"
@@ -59,42 +58,19 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 	// 1 arg, read file in sequence
 	// 2+ args, [0] = expression, file the rest
 
+	out := cmd.OutOrStdout()
+
 	var err error
-	firstFileIndex, err := initCommand(cmd, args)
+
+	expression, args, err := initCommand(cmd, args)
 	if err != nil {
 		return err
 	}
 
-	stat, _ := os.Stdin.Stat()
-	pipingStdIn := (stat.Mode() & os.ModeCharDevice) == 0
-	yqlib.GetLogger().Debug("pipingStdIn: %v", pipingStdIn)
-
-	yqlib.GetLogger().Debug("stat.Mode(): %v", stat.Mode())
-	yqlib.GetLogger().Debug("ModeDir: %v", stat.Mode()&os.ModeDir)
-	yqlib.GetLogger().Debug("ModeAppend: %v", stat.Mode()&os.ModeAppend)
-	yqlib.GetLogger().Debug("ModeExclusive: %v", stat.Mode()&os.ModeExclusive)
-	yqlib.GetLogger().Debug("ModeTemporary: %v", stat.Mode()&os.ModeTemporary)
-	yqlib.GetLogger().Debug("ModeSymlink: %v", stat.Mode()&os.ModeSymlink)
-	yqlib.GetLogger().Debug("ModeDevice: %v", stat.Mode()&os.ModeDevice)
-	yqlib.GetLogger().Debug("ModeNamedPipe: %v", stat.Mode()&os.ModeNamedPipe)
-	yqlib.GetLogger().Debug("ModeSocket: %v", stat.Mode()&os.ModeSocket)
-	yqlib.GetLogger().Debug("ModeSetuid: %v", stat.Mode()&os.ModeSetuid)
-	yqlib.GetLogger().Debug("ModeSetgid: %v", stat.Mode()&os.ModeSetgid)
-	yqlib.GetLogger().Debug("ModeCharDevice: %v", stat.Mode()&os.ModeCharDevice)
-	yqlib.GetLogger().Debug("ModeSticky: %v", stat.Mode()&os.ModeSticky)
-	yqlib.GetLogger().Debug("ModeIrregular: %v", stat.Mode()&os.ModeIrregular)
-
-	// Mask for the type bits. For regular files, none will be set.
-	yqlib.GetLogger().Debug("ModeType: %v", stat.Mode()&os.ModeType)
-
-	yqlib.GetLogger().Debug("ModePerm: %v", stat.Mode()&os.ModePerm)
-
-	out := cmd.OutOrStdout()
-
 	if writeInplace {
 		// only use colors if its forced
 		colorsEnabled = forceColor
-		writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[firstFileIndex])
+		writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[0])
 		out, err = writeInPlaceHandler.CreateTempFile()
 		if err != nil {
 			return err
@@ -129,12 +105,12 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 
 	if frontMatter != "" {
 		yqlib.GetLogger().Debug("using front matter handler")
-		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])
+		frontMatterHandler := yqlib.NewFrontMatterHandler(args[0])
 		err = frontMatterHandler.Split()
 		if err != nil {
 			return err
 		}
-		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
+		args[0] = frontMatterHandler.GetYamlFrontMatterFilename()
 
 		if frontMatter == "process" {
 			reader := frontMatterHandler.GetContentReader()
@@ -143,10 +119,6 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 		}
 		defer frontMatterHandler.CleanUp()
 	}
-	expression, args, err := processArgs(pipingStdIn, args)
-	if err != nil {
-		return err
-	}
 
 	switch len(args) {
 	case 0:

cmd/root.go

@@ -18,7 +18,7 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 # yq defaults to 'eval' command if no command is specified. See "yq eval --help" for more examples.
 
 # read the "stuff" node from "myfile.yml"
-cat myfile.yml | yq '.stuff' 
+yq '.stuff' < myfile.yml
 
 # update myfile.yml in place
 yq -i '.stuff = "foo"' myfile.yml
@@ -54,6 +54,7 @@ yq -P sample.json
 			yqlib.InitExpressionParser()
 			yqlib.XMLPreferences.AttributePrefix = xmlAttributePrefix
 			yqlib.XMLPreferences.ContentName = xmlContentName
+			yqlib.XMLPreferences.StrictMode = xmlStrictMode
 		},
 	}
 
@@ -70,6 +71,9 @@ yq -P sample.json
 
 	rootCmd.PersistentFlags().StringVar(&xmlAttributePrefix, "xml-attribute-prefix", "+", "prefix for xml attributes")
 	rootCmd.PersistentFlags().StringVar(&xmlContentName, "xml-content-name", "+content", "name for xml content (if no attribute name is present).")
+	rootCmd.PersistentFlags().BoolVar(&xmlStrictMode, "xml-strict-mode", false, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
+	rootCmd.PersistentFlags().BoolVar(&xmlKeepNamespace, "xml-keep-namespace", true, "enables keeping namespace after parsing attributes")
+	rootCmd.PersistentFlags().BoolVar(&xmlUseRawToken, "xml-raw-token", true, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")
 
 	rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.")
 	rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)")
@@ -77,7 +81,7 @@ yq -P sample.json
 	rootCmd.PersistentFlags().IntVarP(&indent, "indent", "I", 2, "sets indent level for output")
 	rootCmd.Flags().BoolVarP(&version, "version", "V", false, "Print version information and quit")
 	rootCmd.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the file inplace of first file given.")
-	rootCmd.PersistentFlags().BoolVarP(&unwrapScalar, "unwrapScalar", "", true, "unwrap scalar, print the value with no quotes, colors or comments")
+	rootCmd.PersistentFlags().BoolVarP(&unwrapScalar, "unwrapScalar", "r", true, "unwrap scalar, print the value with no quotes, colors or comments")
 	rootCmd.PersistentFlags().BoolVarP(&prettyPrint, "prettyPrint", "P", false, "pretty print, shorthand for '... style = \"\"'")
 	rootCmd.PersistentFlags().BoolVarP(&exitStatus, "exit-status", "e", false, "set exit status if there are no matches or null or false is returned")
 
@@ -88,6 +92,7 @@ yq -P sample.json
 	rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.")
 
 	rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter.")
+	rootCmd.PersistentFlags().StringVarP(&splitFileExpFile, "split-exp-file", "", "", "Use a file to specify the split-exp expression.")
 
 	rootCmd.PersistentFlags().StringVarP(&expressionFile, "from-file", "", "", "Load expression from specified file.")
 

cmd/shell-completion.go

@@ -42,7 +42,7 @@ $ yq shell-completion fish > ~/.config/fish/completions/yq.fish
 `,
 	DisableFlagsInUseLine: true,
 	ValidArgs:             []string{"bash", "zsh", "fish", "powershell"},
-	Args:                  cobra.ExactValidArgs(1),
+	Args:                  cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs),
 	RunE: func(cmd *cobra.Command, args []string) error {
 		var err error = nil
 		switch args[0] {

cmd/utils.go

@@ -10,7 +10,7 @@ import (
 	"gopkg.in/op/go-logging.v1"
 )
 
-func initCommand(cmd *cobra.Command, args []string) (firstFileIndex int, err error) {
+func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 	cmd.SilenceUsage = true
 
 	fileInfo, _ := os.Stdout.Stat()
@@ -19,11 +19,17 @@ func initCommand(cmd *cobra.Command, args []string) (firstFileIndex int, err err
 		colorsEnabled = true
 	}
 
-	firstFileIndex = -1
-	if !nullInput && len(args) == 1 {
-		firstFileIndex = 0
-	} else if len(args) > 1 {
-		firstFileIndex = 1
+	expression, args, err := processArgs(args)
+	if err != nil {
+		return "", nil, err
+	}
+
+	if splitFileExpFile != "" {
+		splitExpressionBytes, err := os.ReadFile(splitFileExpFile)
+		if err != nil {
+			return "", nil, err
+		}
+		splitFileExp = string(splitExpressionBytes)
 	}
 
 	// backwards compatibility
@@ -31,19 +37,23 @@ func initCommand(cmd *cobra.Command, args []string) (firstFileIndex int, err err
 		outputFormat = "json"
 	}
 
-	if writeInplace && (firstFileIndex == -1) {
-		return 0, fmt.Errorf("write inplace flag only applicable when giving an expression and at least one file")
+	if writeInplace && (len(args) == 0 || args[0] == "-") {
+		return "", nil, fmt.Errorf("write inplace flag only applicable when giving an expression and at least one file")
+	}
+
+	if frontMatter != "" && len(args) == 0 {
+		return "", nil, fmt.Errorf("front matter flag only applicable when giving an expression and at least one file")
 	}
 
 	if writeInplace && splitFileExp != "" {
-		return 0, fmt.Errorf("write inplace cannot be used with split file")
+		return "", nil, fmt.Errorf("write inplace cannot be used with split file")
 	}
 
-	if nullInput && len(args) > 1 {
-		return 0, fmt.Errorf("cannot pass files in when using null-input flag")
+	if nullInput && len(args) > 0 {
+		return "", nil, fmt.Errorf("cannot pass files in when using null-input flag")
 	}
 
-	return firstFileIndex, nil
+	return expression, args, nil
 }
 
 func configureDecoder() (yqlib.Decoder, error) {
@@ -53,9 +63,15 @@ func configureDecoder() (yqlib.Decoder, error) {
 	}
 	switch yqlibInputFormat {
 	case yqlib.XMLInputFormat:
-		return yqlib.NewXMLDecoder(xmlAttributePrefix, xmlContentName), nil
+		return yqlib.NewXMLDecoder(xmlAttributePrefix, xmlContentName, xmlStrictMode, xmlKeepNamespace, xmlUseRawToken), nil
 	case yqlib.PropertiesInputFormat:
 		return yqlib.NewPropertiesDecoder(), nil
+	case yqlib.JsonInputFormat:
+		return yqlib.NewJSONDecoder(), nil
+	case yqlib.CSVObjectInputFormat:
+		return yqlib.NewCSVObjectDecoder(','), nil
+	case yqlib.TSVObjectInputFormat:
+		return yqlib.NewCSVObjectDecoder('\t'), nil
 	}
 
 	return yqlib.NewYamlDecoder(), nil
@@ -81,9 +97,9 @@ func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yq
 func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
 	switch format {
 	case yqlib.JSONOutputFormat:
-		return yqlib.NewJONEncoder(indent)
+		return yqlib.NewJSONEncoder(indent, colorsEnabled)
 	case yqlib.PropsOutputFormat:
-		return yqlib.NewPropertiesEncoder()
+		return yqlib.NewPropertiesEncoder(unwrapScalar)
 	case yqlib.CSVOutputFormat:
 		return yqlib.NewCsvEncoder(',')
 	case yqlib.TSVOutputFormat:
@@ -116,12 +132,15 @@ func maybeFile(str string) bool {
 	return result
 }
 
-func processStdInArgs(pipingStdin bool, args []string) []string {
+func processStdInArgs(args []string) []string {
+	stat, _ := os.Stdin.Stat()
+	pipingStdin := (stat.Mode() & os.ModeCharDevice) == 0
+
 	// if we've been given a file, don't automatically
 	// read from stdin.
 	// this happens if there is more than one argument
 	// or only one argument and its a file
-	if !pipingStdin || len(args) > 1 || (len(args) > 0 && maybeFile(args[0])) {
+	if nullInput || !pipingStdin || len(args) > 1 || (len(args) > 0 && maybeFile(args[0])) {
 		return args
 	}
 
@@ -137,7 +156,7 @@ func processStdInArgs(pipingStdin bool, args []string) []string {
 	return append(args, "-")
 }
 
-func processArgs(pipingStdin bool, originalArgs []string) (string, []string, error) {
+func processArgs(originalArgs []string) (string, []string, error) {
 	expression := forceExpression
 	if expressionFile != "" {
 		expressionBytes, err := os.ReadFile(expressionFile)
@@ -147,7 +166,7 @@ func processArgs(pipingStdin bool, originalArgs []string) (string, []string, err
 		expression = string(expressionBytes)
 	}
 
-	args := processStdInArgs(pipingStdin, originalArgs)
+	args := processStdInArgs(originalArgs)
 	yqlib.GetLogger().Debugf("processed args: %v", args)
 	if expression == "" && len(args) > 0 && args[0] != "-" && !maybeFile(args[0]) {
 		yqlib.GetLogger().Debug("assuming expression is '%v'", args[0])

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.23.1"
+	Version = "4.28.1"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

examples/data1.yaml

@@ -1,2 +1,5 @@
-a: mike
-b: [t, f]
+volumes:
+  beehive-conf:
+    driver: local
+services:
+  - beehive

examples/data2.yaml

@@ -1,2 +1,5 @@
-c:
-  d: hamster
+# --------------------------------------------------
+# It's a test with comment
+# --------------------------------------------------
+groups:
+  - name: d

examples/sample_objects.csv

@@ -0,0 +1,3 @@
+name,numberOfCats,likesApples,height
+Gary,1,true,168.8
+Samantha's Rabbit,2,false,-188.8

github-action/Dockerfile

@@ -2,7 +2,8 @@ FROM mikefarah/yq:4
 
 COPY entrypoint.sh /entrypoint.sh
 
-# this seems to be the default user in github actions
-USER 1001
+# github action recommendation is to run as root.
+# https://docs.github.com/en/actions/creating-actions/dockerfile-support-for-github-actions#user
+USER root
 
 ENTRYPOINT ["/entrypoint.sh"]

github-action/entrypoint.sh

@@ -2,8 +2,11 @@
 set -e
 echo "::debug::\$cmd: $1"
 RESULT=$(eval "$1")
-RESULT="${RESULT//'%'/'%25'}"
-RESULT="${RESULT//$'\n'/'%0A'}"
-RESULT="${RESULT//$'\r'/'%0D'}"
 echo "::debug::\$RESULT: $RESULT"
-echo ::set-output name=result::"$RESULT"
+# updating from 
+# https://github.com/orgs/community/discussions/26288#discussioncomment-3876281
+# https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-output-parameter
+delimiter=$(cat /proc/sys/kernel/random/uuid)
+echo "result<<${delimiter}" >> "${GITHUB_OUTPUT}"
+echo "${RESULT}" >> "${GITHUB_OUTPUT}"
+echo "${delimiter}" >> "${GITHUB_OUTPUT}"

go.mod

@@ -2,28 +2,30 @@ module github.com/mikefarah/yq/v4
 
 require (
 	github.com/a8m/envsubst v1.3.0
-	github.com/elliotchance/orderedmap v1.4.0
+	github.com/alecthomas/participle/v2 v2.0.0-beta.5
+	github.com/alecthomas/repr v0.1.1
+	github.com/dimchansky/utfbom v1.1.1
+	github.com/elliotchance/orderedmap v1.5.0
 	github.com/fatih/color v1.13.0
+	github.com/goccy/go-json v0.9.11
 	github.com/goccy/go-yaml v1.9.5
 	github.com/jinzhu/copier v0.3.5
 	github.com/magiconair/properties v1.8.6
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
-	github.com/spf13/cobra v1.4.0
-	github.com/timtadh/lexmachine v0.2.2
-	golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
+	github.com/spf13/cobra v1.6.0
+	golang.org/x/net v0.0.0-20220708220712-1185a9018129
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
+	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/timtadh/data-structures v0.5.3 // indirect
-	golang.org/x/sys v0.0.0-20211205182925-97ca703d548d // indirect
+	golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 )
 
-go 1.17
+go 1.19

go.sum

@@ -1,10 +1,17 @@
 github.com/a8m/envsubst v1.3.0 h1:GmXKmVssap0YtlU3E230W98RWtWCyIZzjtf1apWWyAg=
 github.com/a8m/envsubst v1.3.0/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
-github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/alecthomas/assert/v2 v2.0.3 h1:WKqJODfOiQG0nEJKFKzDIG3E29CN2/4zR9XGJzKIkbg=
+github.com/alecthomas/participle/v2 v2.0.0-beta.5 h1:y6dsSYVb1G5eK6mgmy+BgI3Mw35a3WghArZ/Hbebrjo=
+github.com/alecthomas/participle/v2 v2.0.0-beta.5/go.mod h1:RC764t6n4L8D8ITAJv0qdokritYSNR3wV5cVwmIEaMM=
+github.com/alecthomas/repr v0.1.1 h1:87P60cSmareLAxMc4Hro0r2RBY4ROm0dYwkJNpS4pPs=
+github.com/alecthomas/repr v0.1.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/elliotchance/orderedmap v1.4.0 h1:wZtfeEONCbx6in1CZyE6bELEt/vFayMvsxqI5SgsR+A=
-github.com/elliotchance/orderedmap v1.4.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
+github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
+github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
+github.com/elliotchance/orderedmap v1.5.0 h1:1IsExUsjv5XNBD3ZdC7jkAAqLWOOKdbPTmkHx63OsBg=
+github.com/elliotchance/orderedmap v1.5.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
 github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
@@ -12,10 +19,13 @@ github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvSc
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
+github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk=
+github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/goccy/go-yaml v1.9.5 h1:Eh/+3uk9kLxG4koCX6lRMAPS1OaMSAi+FJcya0INdB0=
 github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
-github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
+github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg=
 github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
@@ -33,48 +43,40 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q=
-github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g=
+github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI=
+github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/timtadh/data-structures v0.5.3 h1:F2tEjoG9qWIyUjbvXVgJqEOGJPMIiYn7U5W5mE+i/vQ=
-github.com/timtadh/data-structures v0.5.3/go.mod h1:9R4XODhJ8JdWFEI8P/HJKqxuJctfBQw6fDibMQny2oU=
-github.com/timtadh/lexmachine v0.2.2 h1:g55RnjdYazm5wnKv59pwFcBJHOyvTPfDEoz21s4PHmY=
-github.com/timtadh/lexmachine v0.2.2/go.mod h1:GBJvD5OAfRn/gnp92zb9KTgHLB7akKyxmVivoYCcjQI=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d h1:LO7XpTYMwTqxjLcGWPijK3vRXg1aWdlNOVOHRq45d7c=
-golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20220708220712-1185a9018129 h1:vucSRfWwTsoXro7P+3Cjlr6flUMtzCwzlvkxEQtHHB0=
+golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211205182925-97ca703d548d h1:FjkYO/PPp4Wi0EAUOVLxePm7qVW4r4ctbWpURyuOD0E=
-golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e h1:NHvCuwuS43lGnYhten69ZWqi2QOj/CiDNcKbVqwVoew=
+golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0=
+golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW8s2qTSe3wGBtvo0MbVQG/c5k8RE=
 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

how-it-works.md

@@ -19,7 +19,7 @@ with an expression:
 .a = .b
 ```
 
-Like math expression - operator precedence is important. 
+Like math expressions - operator precedence is important. 
 
 The `=` operator takes two arguments, a `lhs` expression, which in this case is `.a` and `rhs` expression which is `.b`. 
 
@@ -37,7 +37,7 @@ The `=` operator then pipes the 'root' context through the `rhs` expression of `
 dog
 ```
 
-Both sides have now been evaluated, so now the operator copies across the value from the RHS (`.b`) to the the LHS (`.a`), and it returns the now updated context:
+Both sides have now been evaluated, so now the operator copies across the value from the RHS (`.b`) to the LHS (`.a`), and it returns the now updated context:
 
 ```yaml
 a: dog
@@ -47,7 +47,7 @@ b: dog
 
 ## Complex assignment, operator precedence rules
 
-Just like math expression - `yq` expression have an order of precedence. The pipe `|` operator has a low order of precedence, so operators with higher precedence will get evalated first. 
+Just like math expressions - `yq` expressions have an order of precedence. The pipe `|` operator has a low order of precedence, so operators with higher precedence will get evaluated first. 
 
 Most of the time, this is intuitively what you'd want, for instance `.a = "cat" | .b = "dog"` is effectively: `(.a = "cat") | (.b = "dog")`.
 
@@ -66,7 +66,7 @@ Lets say you had:
 Lets say you wanted to update the `sally` entry to have fruit: 'mango'. The _incorrect_ way to do that is:
 `.[] | select(.name == "sally") | .fruit = "mango"`.
 
-Becasue `|` has a low operator precedence, this will be evaluated (_incorrectly_) as : `(.[]) | (select(.name == "sally")) | (.fruit = "mango")`. What you'll see is only the updated segment returned:
+Because `|` has a low operator precedence, this will be evaluated (_incorrectly_) as : `(.[]) | (select(.name == "sally")) | (.fruit = "mango")`. What you'll see is only the updated segment returned:
 
 ```yaml
 name: sally

pkg/yqlib/candidate_node.go

@@ -14,14 +14,15 @@ type CandidateNode struct {
 	Parent *CandidateNode // parent node
 	Key    *yaml.Node     // node key, if this is a value from a map (or index in an array)
 
-	LeadingContent string
+	LeadingContent  string
+	TrailingContent string
 
 	Path      []interface{} /// the path we took to get to this node
 	Document  uint          // the document index of this node
 	Filename  string
 	FileIndex int
 	// when performing op against all nodes given, this will treat all the nodes as one
-	// (e.g. top level cross document merge). This property does not propegate to child nodes.
+	// (e.g. top level cross document merge). This property does not propagate to child nodes.
 	EvaluateTogether bool
 	IsMapKey         bool
 }
@@ -96,6 +97,13 @@ func (n *CandidateNode) CreateReplacement(node *yaml.Node) *CandidateNode {
 	}
 }
 
+func (n *CandidateNode) CreateReplacementWithDocWrappers(node *yaml.Node) *CandidateNode {
+	replacement := n.CreateReplacement(node)
+	replacement.LeadingContent = n.LeadingContent
+	replacement.TrailingContent = n.TrailingContent
+	return replacement
+}
+
 func (n *CandidateNode) createChildPath(path interface{}) []interface{} {
 	if path == nil {
 		newPath := make([]interface{}, len(n.Path))
@@ -149,7 +157,7 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignP
 	n.Node.Kind = other.Node.Kind
 
 	// don't clobber custom tags...
-	if strings.HasPrefix(n.Node.Tag, "!!") || n.Node.Tag == "" {
+	if prefs.ClobberCustomTags || strings.HasPrefix(n.Node.Tag, "!!") || n.Node.Tag == "" {
 		n.Node.Tag = other.Node.Tag
 	}
 
@@ -169,6 +177,9 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignP
 	if other.Node.FootComment != "" {
 		n.Node.FootComment = other.Node.FootComment
 	}
+	if other.TrailingContent != "" {
+		n.TrailingContent = other.TrailingContent
+	}
 	if other.Node.HeadComment != "" {
 		n.Node.HeadComment = other.Node.HeadComment
 	}

pkg/yqlib/compare_operators.go

@@ -0,0 +1,133 @@
+package yqlib
+
+import (
+	"fmt"
+	"strconv"
+	"time"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+type compareTypePref struct {
+	OrEqual bool
+	Greater bool
+}
+
+func compareOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	log.Debugf("-- compareOperator")
+	prefs := expressionNode.Operation.Preferences.(compareTypePref)
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, compare(prefs), true)
+}
+
+func compare(prefs compareTypePref) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+		log.Debugf("-- compare cross function")
+		if lhs == nil && rhs == nil {
+			owner := &CandidateNode{}
+			return createBooleanCandidate(owner, prefs.OrEqual), nil
+		} else if lhs == nil {
+			log.Debugf("lhs nil, but rhs is not")
+			return createBooleanCandidate(rhs, false), nil
+		} else if rhs == nil {
+			log.Debugf("rhs nil, but rhs is not")
+			return createBooleanCandidate(lhs, false), nil
+		}
+
+		lhs.Node = unwrapDoc(lhs.Node)
+		rhs.Node = unwrapDoc(rhs.Node)
+
+		switch lhs.Node.Kind {
+		case yaml.MappingNode:
+			return nil, fmt.Errorf("maps not yet supported for comparison")
+		case yaml.SequenceNode:
+			return nil, fmt.Errorf("arrays not yet supported for comparison")
+		default:
+			if rhs.Node.Kind != yaml.ScalarNode {
+				return nil, fmt.Errorf("%v (%v) cannot be subtracted from %v", rhs.Node.Tag, rhs.Path, lhs.Node.Tag)
+			}
+			target := lhs.CreateReplacement(&yaml.Node{})
+			boolV, err := compareScalars(context, prefs, lhs.Node, rhs.Node)
+
+			return createBooleanCandidate(target, boolV), err
+		}
+	}
+}
+
+func compareDateTime(layout string, prefs compareTypePref, lhs *yaml.Node, rhs *yaml.Node) (bool, error) {
+	lhsTime, err := time.Parse(layout, lhs.Value)
+	if err != nil {
+		return false, err
+	}
+
+	rhsTime, err := time.Parse(layout, rhs.Value)
+	if err != nil {
+		return false, err
+	}
+
+	if prefs.OrEqual && lhsTime.Equal(rhsTime) {
+		return true, nil
+	}
+	if prefs.Greater {
+		return lhsTime.After(rhsTime), nil
+	}
+	return lhsTime.Before(rhsTime), nil
+
+}
+
+func compareScalars(context Context, prefs compareTypePref, lhs *yaml.Node, rhs *yaml.Node) (bool, error) {
+	lhsTag := guessTagFromCustomType(lhs)
+	rhsTag := guessTagFromCustomType(rhs)
+
+	isDateTime := lhs.Tag == "!!timestamp"
+	// if the lhs is a string, it might be a timestamp in a custom format.
+	if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
+		_, err := time.Parse(context.GetDateTimeLayout(), lhs.Value)
+		isDateTime = err == nil
+	}
+	if isDateTime {
+		return compareDateTime(context.GetDateTimeLayout(), prefs, lhs, rhs)
+	} else if lhsTag == "!!int" && rhsTag == "!!int" {
+		_, lhsNum, err := parseInt64(lhs.Value)
+		if err != nil {
+			return false, err
+		}
+		_, rhsNum, err := parseInt64(rhs.Value)
+		if err != nil {
+			return false, err
+		}
+
+		if prefs.OrEqual && lhsNum == rhsNum {
+			return true, nil
+		}
+		if prefs.Greater {
+			return lhsNum > rhsNum, nil
+		}
+		return lhsNum < rhsNum, nil
+	} else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") {
+		lhsNum, err := strconv.ParseFloat(lhs.Value, 64)
+		if err != nil {
+			return false, err
+		}
+		rhsNum, err := strconv.ParseFloat(rhs.Value, 64)
+		if err != nil {
+			return false, err
+		}
+		if prefs.OrEqual && lhsNum == rhsNum {
+			return true, nil
+		}
+		if prefs.Greater {
+			return lhsNum > rhsNum, nil
+		}
+		return lhsNum < rhsNum, nil
+	} else if lhsTag == "!!str" && rhsTag == "!!str" {
+		if prefs.OrEqual && lhs.Value == rhs.Value {
+			return true, nil
+		}
+		if prefs.Greater {
+			return lhs.Value > rhs.Value, nil
+		}
+		return lhs.Value < rhs.Value, nil
+	}
+
+	return false, fmt.Errorf("%v not yet supported for comparison", lhs.Tag)
+}

pkg/yqlib/compare_operators_test.go

@@ -0,0 +1,371 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var compareOperatorScenarios = []expressionScenario{
+	// ints, not equal
+	{
+		description: "Compare numbers (>)",
+		document:    "a: 5\nb: 4",
+		expression:  ".a > .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 5\nb: 4",
+		expression: ".a < .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:     true,
+		description: "Compare integers (>=)",
+		document:    "a: 5\nb: 4",
+		expression:  ".a >= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 5\nb: 4",
+		expression: ".a <= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+
+	// ints, equal
+	{
+		skipDoc:     true,
+		description: "Compare equal numbers (>)",
+		document:    "a: 5\nb: 5",
+		expression:  ".a > .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 5\nb: 5",
+		expression: ".a < .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		description: "Compare equal numbers (>=)",
+		document:    "a: 5\nb: 5",
+		expression:  ".a >= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 5\nb: 5",
+		expression: ".a <= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+
+	// floats, not equal
+	{
+		skipDoc:    true,
+		document:   "a: 5.2\nb: 4.1",
+		expression: ".a > .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 5.2\nb: 4.1",
+		expression: ".a < .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 5.2\nb: 4.1",
+		expression: ".a >= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 5.5\nb: 4.1",
+		expression: ".a <= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+
+	// floats, equal
+	{
+		skipDoc:    true,
+		document:   "a: 5.5\nb: 5.5",
+		expression: ".a > .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 5.5\nb: 5.5",
+		expression: ".a < .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 5.1\nb: 5.1",
+		expression: ".a >= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 5.1\nb: 5.1",
+		expression: ".a <= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+
+	// strings, not equal
+	{
+		description:    "Compare strings",
+		subdescription: "Compares strings by their bytecode.",
+		document:       "a: zoo\nb: apple",
+		expression:     ".a > .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: zoo\nb: apple",
+		expression: ".a < .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: zoo\nb: apple",
+		expression: ".a >= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: zoo\nb: apple",
+		expression: ".a <= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+
+	// strings, equal
+	{
+		skipDoc:    true,
+		document:   "a: cat\nb: cat",
+		expression: ".a > .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: cat\nb: cat",
+		expression: ".a < .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: cat\nb: cat",
+		expression: ".a >= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: cat\nb: cat",
+		expression: ".a <= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+
+	// datetime, not equal
+	{
+		description:    "Compare date times",
+		subdescription: "You can compare date times. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
+		document:       "a: 2021-01-01T03:10:00Z\nb: 2020-01-01T03:10:00Z",
+		expression:     ".a > .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 2021-01-01T03:10:00Z\nb: 2020-01-01T03:10:00Z",
+		expression: ".a < .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 2021-01-01T03:10:00Z\nb: 2020-01-01T03:10:00Z",
+		expression: ".a >= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 2021-01-01T03:10:00Z\nb: 2020-01-01T03:10:00Z",
+		expression: ".a <= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+
+	// datetime, equal
+	{
+		skipDoc:    true,
+		document:   "a: 2021-01-01T03:10:00Z\nb: 2021-01-01T03:10:00Z",
+		expression: ".a > .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 2021-01-01T03:10:00Z\nb: 2021-01-01T03:10:00Z",
+		expression: ".a < .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 2021-01-01T03:10:00Z\nb: 2021-01-01T03:10:00Z",
+		expression: ".a >= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a: 2021-01-01T03:10:00Z\nb: 2021-01-01T03:10:00Z",
+		expression: ".a <= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+	// both null
+	{
+		description: "Both sides are null: > is false",
+		expression:  ".a > .b",
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: ".a < .b",
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		description: "Both sides are null: >= is true",
+		expression:  ".a >= .b",
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: ".a <= .b",
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
+
+	// one null
+	{
+		skipDoc:     true,
+		description: "One side is null: > is false",
+		document:    `a: 5`,
+		expression:  ".a > .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: 5`,
+		expression: ".a < .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:     true,
+		description: "One side is null: >= is false",
+		document:    `a: 5`,
+		expression:  ".a >= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: 5`,
+		expression: ".a <= .b",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: 5`,
+		expression: ".b <= .a",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `a: 5`,
+		expression: ".b < .a",
+		expected: []string{
+			"D0, P[a], (!!bool)::false\n",
+		},
+	},
+}
+
+func TestCompareOperatorScenarios(t *testing.T) {
+	for _, tt := range compareOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentOperatorScenarios(t, "compare", compareOperatorScenarios)
+}

pkg/yqlib/csv_test.go

@@ -0,0 +1,273 @@
+package yqlib
+
+import (
+	"bufio"
+	"fmt"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+const csvSimple = `name,numberOfCats,likesApples,height
+Gary,1,true,168.8
+Samantha's Rabbit,2,false,-188.8
+`
+
+const expectedUpdatedSimpleCsv = `name,numberOfCats,likesApples,height
+Gary,3,true,168.8
+Samantha's Rabbit,2,false,-188.8
+`
+
+const csvSimpleShort = `Name,Number of Cats
+Gary,1
+Samantha's Rabbit,2
+`
+
+const tsvSimple = `name	numberOfCats	likesApples	height
+Gary	1	true	168.8
+Samantha's Rabbit	2	false	-188.8
+`
+
+const expectedYamlFromCSV = `- name: Gary
+  numberOfCats: 1
+  likesApples: true
+  height: 168.8
+- name: Samantha's Rabbit
+  numberOfCats: 2
+  likesApples: false
+  height: -188.8
+`
+
+const expectedYamlFromCSVMissingData = `- name: Gary
+  numberOfCats: 1
+  height: 168.8
+- name: Samantha's Rabbit
+  height: -188.8
+  likesApples: false
+`
+
+const csvSimpleMissingData = `name,numberOfCats,height
+Gary,1,168.8
+Samantha's Rabbit,,-188.8
+`
+
+const csvTestSimpleYaml = `- [i, like, csv]
+- [because, excel, is, cool]`
+
+const expectedSimpleCsv = `i,like,csv
+because,excel,is,cool
+`
+
+const tsvTestExpectedSimpleCsv = `i	like	csv
+because	excel	is	cool
+`
+
+var csvScenarios = []formatScenario{
+	{
+		description:  "Encode CSV simple",
+		input:        csvTestSimpleYaml,
+		expected:     expectedSimpleCsv,
+		scenarioType: "encode-csv",
+	},
+	{
+		description:  "Encode TSV simple",
+		input:        csvTestSimpleYaml,
+		expected:     tsvTestExpectedSimpleCsv,
+		scenarioType: "encode-tsv",
+	},
+	{
+		description:  "Encode Empty",
+		skipDoc:      true,
+		input:        `[]`,
+		expected:     "",
+		scenarioType: "encode-csv",
+	},
+	{
+		description:  "Comma in value",
+		skipDoc:      true,
+		input:        `["comma, in, value", things]`,
+		expected:     "\"comma, in, value\",things\n",
+		scenarioType: "encode-csv",
+	},
+	{
+		description:  "Encode array of objects to csv",
+		input:        expectedYamlFromCSV,
+		expected:     csvSimple,
+		scenarioType: "encode-csv",
+	},
+	{
+		description:    "Encode array of objects to custom csv format",
+		subdescription: "Add the header row manually, then the we convert each object into an array of values - resulting in an array of arrays. Pick the columns and call the header whatever you like.",
+		input:          expectedYamlFromCSV,
+		expected:       csvSimpleShort,
+		expression:     `[["Name", "Number of Cats"]] +  [.[] | [.name, .numberOfCats ]]`,
+		scenarioType:   "encode-csv",
+	},
+	{
+		description:    "Encode array of objects to csv - missing fields behaviour",
+		subdescription: "First entry is used to determine the headers, and it is missing 'likesApples', so it is not included in the csv. Second entry does not have 'numberOfCats' so that is blank",
+		input:          expectedYamlFromCSVMissingData,
+		expected:       csvSimpleMissingData,
+		scenarioType:   "encode-csv",
+	},
+	{
+		description:    "Parse CSV into an array of objects",
+		subdescription: "First row is assumed to be the header row.",
+		input:          csvSimple,
+		expected:       expectedYamlFromCSV,
+		scenarioType:   "decode-csv-object",
+	},
+	{
+		description:    "Parse TSV into an array of objects",
+		subdescription: "First row is assumed to be the header row.",
+		input:          tsvSimple,
+		expected:       expectedYamlFromCSV,
+		scenarioType:   "decode-tsv-object",
+	},
+	{
+		description:  "Round trip",
+		input:        csvSimple,
+		expected:     expectedUpdatedSimpleCsv,
+		expression:   `(.[] | select(.name == "Gary") | .numberOfCats) = 3`,
+		scenarioType: "roundtrip-csv",
+	},
+}
+
+func testCSVScenario(t *testing.T, s formatScenario) {
+	switch s.scenarioType {
+	case "encode-csv":
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder(',')), s.description)
+	case "encode-tsv":
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder('\t')), s.description)
+	case "decode-csv-object":
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, true, true)), s.description)
+	case "decode-tsv-object":
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, true, true)), s.description)
+	case "roundtrip-csv":
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewCsvEncoder(',')), s.description)
+	default:
+		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
+	}
+}
+
+func documentCSVDecodeObjectScenario(w *bufio.Writer, s formatScenario, formatType string) {
+	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
+
+	if s.subdescription != "" {
+		writeOrPanic(w, s.subdescription)
+		writeOrPanic(w, "\n\n")
+	}
+
+	writeOrPanic(w, fmt.Sprintf("Given a sample.%v file of:\n", formatType))
+	writeOrPanic(w, fmt.Sprintf("```%v\n%v\n```\n", formatType, s.input))
+
+	writeOrPanic(w, "then\n")
+	writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=%v sample.%v\n```\n", formatType, formatType))
+	writeOrPanic(w, "will output\n")
+
+	separator := ','
+	if formatType == "tsv" {
+		separator = '\t'
+	}
+
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n",
+		processFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, true, true))),
+	)
+}
+
+func documentCSVEncodeScenario(w *bufio.Writer, s formatScenario, formatType string) {
+	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
+
+	if s.subdescription != "" {
+		writeOrPanic(w, s.subdescription)
+		writeOrPanic(w, "\n\n")
+	}
+
+	writeOrPanic(w, "Given a sample.yml file of:\n")
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input))
+
+	writeOrPanic(w, "then\n")
+
+	expression := s.expression
+
+	if expression != "" {
+		writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=%v '%v' sample.yml\n```\n", formatType, expression))
+	} else {
+		writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=%v sample.yml\n```\n", formatType))
+	}
+	writeOrPanic(w, "will output\n")
+
+	separator := ','
+	if formatType == "tsv" {
+		separator = '\t'
+	}
+
+	writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
+		processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder(separator))),
+	)
+}
+
+func documentCSVRoundTripScenario(w *bufio.Writer, s formatScenario, formatType string) {
+	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
+
+	if s.subdescription != "" {
+		writeOrPanic(w, s.subdescription)
+		writeOrPanic(w, "\n\n")
+	}
+
+	writeOrPanic(w, fmt.Sprintf("Given a sample.%v file of:\n", formatType))
+	writeOrPanic(w, fmt.Sprintf("```%v\n%v\n```\n", formatType, s.input))
+
+	writeOrPanic(w, "then\n")
+
+	expression := s.expression
+
+	if expression != "" {
+		writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=%v -o=%v '%v' sample.%v\n```\n", formatType, formatType, expression, formatType))
+	} else {
+		writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=%v -o=%v sample.%v\n```\n", formatType, formatType, formatType))
+	}
+	writeOrPanic(w, "will output\n")
+
+	separator := ','
+	if formatType == "tsv" {
+		separator = '\t'
+	}
+
+	writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
+		processFormatScenario(s, NewCSVObjectDecoder(separator), NewCsvEncoder(separator))),
+	)
+}
+
+func documentCSVScenario(t *testing.T, w *bufio.Writer, i interface{}) {
+	s := i.(formatScenario)
+	if s.skipDoc {
+		return
+	}
+	switch s.scenarioType {
+	case "encode-csv":
+		documentCSVEncodeScenario(w, s, "csv")
+	case "encode-tsv":
+		documentCSVEncodeScenario(w, s, "tsv")
+	case "decode-csv-object":
+		documentCSVDecodeObjectScenario(w, s, "csv")
+	case "decode-tsv-object":
+		documentCSVDecodeObjectScenario(w, s, "tsv")
+	case "roundtrip-csv":
+		documentCSVRoundTripScenario(w, s, "csv")
+
+	default:
+		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
+	}
+}
+
+func TestCSVScenarios(t *testing.T) {
+	for _, tt := range csvScenarios {
+		testCSVScenario(t, tt)
+	}
+	genericScenarios := make([]interface{}, len(csvScenarios))
+	for i, s := range csvScenarios {
+		genericScenarios[i] = s
+	}
+	documentScenarios(t, "usage", "csv-tsv", genericScenarios, documentCSVScenario)
+}

pkg/yqlib/decoder.go

@@ -14,6 +14,9 @@ const (
 	XMLInputFormat
 	PropertiesInputFormat
 	Base64InputFormat
+	JsonInputFormat
+	CSVObjectInputFormat
+	TSVObjectInputFormat
 )
 
 type Decoder interface {
@@ -29,6 +32,12 @@ func InputFormatFromString(format string) (InputFormat, error) {
 		return XMLInputFormat, nil
 	case "props", "p":
 		return PropertiesInputFormat, nil
+	case "json", "ndjson", "j":
+		return JsonInputFormat, nil
+	case "csv", "c":
+		return CSVObjectInputFormat, nil
+	case "tsv", "t":
+		return TSVObjectInputFormat, nil
 	default:
 		return 0, fmt.Errorf("unknown format '%v' please use [yaml|xml|props]", format)
 	}

pkg/yqlib/decoder_base64.go

@@ -9,9 +9,10 @@ import (
 )
 
 type base64Decoder struct {
-	reader   io.Reader
-	finished bool
-	encoding base64.Encoding
+	reader       io.Reader
+	finished     bool
+	readAnything bool
+	encoding     base64.Encoding
 }
 
 func NewBase64Decoder() Decoder {
@@ -20,6 +21,7 @@ func NewBase64Decoder() Decoder {
 
 func (dec *base64Decoder) Init(reader io.Reader) {
 	dec.reader = reader
+	dec.readAnything = false
 	dec.finished = false
 }
 
@@ -35,8 +37,15 @@ func (dec *base64Decoder) Decode(rootYamlNode *yaml.Node) error {
 	}
 	if buf.Len() == 0 {
 		dec.finished = true
-		return io.EOF
+
+		// if we've read _only_ an empty string, lets return that
+		// otherwise if we've already read some bytes, and now we get
+		// an empty string, then we are done.
+		if dec.readAnything {
+			return io.EOF
+		}
 	}
+	dec.readAnything = true
 	rootYamlNode.Kind = yaml.ScalarNode
 	rootYamlNode.Tag = "!!str"
 	rootYamlNode.Value = buf.String()

pkg/yqlib/decoder_csv_object.go

@@ -0,0 +1,80 @@
+package yqlib
+
+import (
+	"encoding/csv"
+	"errors"
+	"io"
+
+	"github.com/dimchansky/utfbom"
+	yaml "gopkg.in/yaml.v3"
+)
+
+type csvObjectDecoder struct {
+	separator rune
+	reader    csv.Reader
+	finished  bool
+}
+
+func NewCSVObjectDecoder(separator rune) Decoder {
+	return &csvObjectDecoder{separator: separator}
+}
+
+func (dec *csvObjectDecoder) Init(reader io.Reader) {
+	cleanReader, enc := utfbom.Skip(reader)
+	log.Debugf("Detected encoding: %s\n", enc)
+	dec.reader = *csv.NewReader(cleanReader)
+	dec.reader.Comma = dec.separator
+	dec.finished = false
+}
+
+func (dec *csvObjectDecoder) convertToYamlNode(content string) *yaml.Node {
+	node, err := parseSnippet(content)
+	if err != nil {
+		return createScalarNode(content, content)
+	}
+	return node
+}
+
+func (dec *csvObjectDecoder) createObject(headerRow []string, contentRow []string) *yaml.Node {
+	objectNode := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
+
+	for i, header := range headerRow {
+		objectNode.Content = append(
+			objectNode.Content,
+			createScalarNode(header, header),
+			dec.convertToYamlNode(contentRow[i]))
+	}
+	return objectNode
+}
+
+func (dec *csvObjectDecoder) Decode(rootYamlNode *yaml.Node) error {
+	if dec.finished {
+		return io.EOF
+	}
+	headerRow, err := dec.reader.Read()
+	log.Debugf(": headerRow%v", headerRow)
+	if err != nil {
+		return err
+	}
+
+	rootArray := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+
+	contentRow, err := dec.reader.Read()
+
+	for err == nil && len(contentRow) > 0 {
+		log.Debugf("Adding contentRow: %v", contentRow)
+		rootArray.Content = append(rootArray.Content, dec.createObject(headerRow, contentRow))
+		contentRow, err = dec.reader.Read()
+		log.Debugf("Read next contentRow: %v, %v", contentRow, err)
+	}
+	if !errors.Is(err, io.EOF) {
+		return err
+	}
+
+	log.Debugf("finished, contentRow%v", contentRow)
+	log.Debugf("err: %v", err)
+
+	rootYamlNode.Kind = yaml.DocumentNode
+	rootYamlNode.Content = []*yaml.Node{rootArray}
+	return nil
+}

pkg/yqlib/decoder_json.go

@@ -0,0 +1,82 @@
+package yqlib
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/goccy/go-json"
+	yaml "gopkg.in/yaml.v3"
+)
+
+type jsonDecoder struct {
+	decoder json.Decoder
+}
+
+func NewJSONDecoder() Decoder {
+	return &jsonDecoder{}
+}
+
+func (dec *jsonDecoder) Init(reader io.Reader) {
+	dec.decoder = *json.NewDecoder(reader)
+}
+
+func (dec *jsonDecoder) Decode(rootYamlNode *yaml.Node) error {
+
+	var dataBucket orderedMap
+	log.Debug("going to decode")
+	err := dec.decoder.Decode(&dataBucket)
+	if err != nil {
+		return err
+	}
+	node, err := dec.convertToYamlNode(&dataBucket)
+
+	if err != nil {
+		return err
+	}
+	rootYamlNode.Kind = yaml.DocumentNode
+	rootYamlNode.Content = []*yaml.Node{node}
+	return nil
+}
+
+func (dec *jsonDecoder) convertToYamlNode(data *orderedMap) (*yaml.Node, error) {
+	if data.kv == nil {
+		switch rawData := data.altVal.(type) {
+		case nil:
+			return createScalarNode(nil, "null"), nil
+		case float64, float32:
+			// json decoder returns ints as float.
+			return parseSnippet(fmt.Sprintf("%v", rawData))
+		case int, int64, int32, string, bool:
+			return createScalarNode(rawData, fmt.Sprintf("%v", rawData)), nil
+		case []*orderedMap:
+			return dec.parseArray(rawData)
+		default:
+			return nil, fmt.Errorf("unrecognised type :( %v", rawData)
+		}
+	}
+
+	var yamlMap = &yaml.Node{Kind: yaml.MappingNode}
+	for _, keyValuePair := range data.kv {
+		yamlValue, err := dec.convertToYamlNode(&keyValuePair.V)
+		if err != nil {
+			return nil, err
+		}
+		yamlMap.Content = append(yamlMap.Content, createScalarNode(keyValuePair.K, keyValuePair.K), yamlValue)
+	}
+	return yamlMap, nil
+
+}
+
+func (dec *jsonDecoder) parseArray(dataArray []*orderedMap) (*yaml.Node, error) {
+
+	var yamlMap = &yaml.Node{Kind: yaml.SequenceNode}
+
+	for _, value := range dataArray {
+		yamlValue, err := dec.convertToYamlNode(value)
+		if err != nil {
+			return nil, err
+		}
+		yamlMap.Content = append(yamlMap.Content, yamlValue)
+	}
+	return yamlMap, nil
+}

pkg/yqlib/decoder_xml.go

@@ -2,6 +2,7 @@ package yqlib
 
 import (
 	"encoding/xml"
+	"errors"
 	"io"
 	"strings"
 	"unicode"
@@ -12,25 +13,37 @@ import (
 
 type xmlDecoder struct {
 	reader          io.Reader
+	readAnything    bool
 	attributePrefix string
 	contentName     string
+	strictMode      bool
+	keepNamespace   bool
+	useRawToken     bool
 	finished        bool
 }
 
-func NewXMLDecoder(attributePrefix string, contentName string) Decoder {
+func NewXMLDecoder(attributePrefix string, contentName string, strictMode bool, keepNamespace bool, useRawToken bool) Decoder {
 	if contentName == "" {
 		contentName = "content"
 	}
-	return &xmlDecoder{attributePrefix: attributePrefix, contentName: contentName, finished: false}
+	return &xmlDecoder{
+		attributePrefix: attributePrefix,
+		contentName:     contentName,
+		finished:        false,
+		strictMode:      strictMode,
+		keepNamespace:   keepNamespace,
+		useRawToken:     useRawToken,
+	}
 }
 
 func (dec *xmlDecoder) Init(reader io.Reader) {
 	dec.reader = reader
+	dec.readAnything = false
 	dec.finished = false
 }
 
 func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yaml.Node, error) {
-	yamlNode := &yaml.Node{Kind: yaml.SequenceNode}
+	yamlNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
 	for _, child := range nodes {
 		yamlChild, err := dec.convertToYamlNode(child)
 		if err != nil {
@@ -51,7 +64,7 @@ func (dec *xmlDecoder) processComment(c string) string {
 
 func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
 	log.Debug("createMap: headC: %v, footC: %v", n.HeadComment, n.FootComment)
-	yamlNode := &yaml.Node{Kind: yaml.MappingNode}
+	yamlNode := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
 
 	if len(n.Data) > 0 {
 		label := dec.contentName
@@ -135,8 +148,11 @@ func (dec *xmlDecoder) Decode(rootYamlNode *yaml.Node) error {
 		return err
 	} else if firstNode.Tag == "!!null" {
 		dec.finished = true
-		return io.EOF
+		if dec.readAnything {
+			return io.EOF
+		}
 	}
+	dec.readAnything = true
 	rootYamlNode.Kind = yaml.DocumentNode
 	rootYamlNode.Content = []*yaml.Node{firstNode}
 	dec.finished = true
@@ -189,7 +205,7 @@ type element struct {
 // of the map keys.
 func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
 	xmlDec := xml.NewDecoder(dec.reader)
-
+	xmlDec.Strict = dec.strictMode
 	// That will convert the charset if the provided XML is non-UTF-8
 	xmlDec.CharsetReader = charset.NewReaderLabel
 
@@ -199,8 +215,18 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
 		n:      root,
 	}
 
+	getToken := func() (xml.Token, error) {
+		if dec.useRawToken {
+			return xmlDec.RawToken()
+		}
+		return xmlDec.Token()
+	}
+
 	for {
-		t, _ := xmlDec.Token()
+		t, e := getToken()
+		if e != nil && !errors.Is(e, io.EOF) {
+			return e
+		}
 		if t == nil {
 			break
 		}
@@ -218,6 +244,11 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
 
 			// Extract attributes as children
 			for _, a := range se.Attr {
+				if dec.keepNamespace {
+					if a.Name.Space != "" {
+						a.Name.Local = a.Name.Space + ":" + a.Name.Local
+					}
+				}
 				elem.n.AddChild(dec.attributePrefix+a.Name.Local, &xmlNode{Data: a.Value})
 			}
 		case xml.CharData:

pkg/yqlib/doc/operators/add.md

@@ -135,7 +135,9 @@ a:
     b:
       - dog
       - mouse
-  a3: {b: [mouse]}
+  a3:
+    b:
+      - mouse
 ```
 
 ## String concatenation

pkg/yqlib/doc/operators/alternative-default-value.md

@@ -79,3 +79,36 @@ will output
 cat
 ```
 
+## Update or create - entity exists
+This initialises `a` if it's not present
+
+Given a sample.yml file of:
+```yaml
+a: 1
+```
+then
+```bash
+yq '(.a // (.a = 0)) += 1' sample.yml
+```
+will output
+```yaml
+a: 2
+```
+
+## Update or create - entity does not exist
+This initialises `a` if it's not present
+
+Given a sample.yml file of:
+```yaml
+b: camel
+```
+then
+```bash
+yq '(.a // (.a = 0)) += 1' sample.yml
+```
+will output
+```yaml
+b: camel
+a: 1
+```
+

pkg/yqlib/doc/operators/assign-update.md

@@ -7,6 +7,11 @@ Which will assign the LHS node values to the RHS node values. The RHS expression
 
 ### relative form: `|=`
 This will do a similar thing to the plain form, however, the RHS expression is run against _the LHS nodes_. This is useful for updating values based on old values, e.g. increment.
+
+
+### Flags
+- `c` clobber custom tags
+
 {% hint style="warning" %}
 Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
 
@@ -107,13 +112,13 @@ c: fieldC
 ```
 then
 ```bash
-yq '(.a, .c) = "potatoe"' sample.yml
+yq '(.a, .c) = "potato"' sample.yml
 ```
 will output
 ```yaml
-a: potatoe
+a: potato
 b: fieldB
-c: potatoe
+c: potato
 ```
 
 ## Update string value
@@ -151,7 +156,7 @@ a:
 ```
 
 ## Update deeply selected results
-Note that the LHS is wrapped in brackets! This is to ensure we dont first filter out the yaml and then update the snippet.
+Note that the LHS is wrapped in brackets! This is to ensure we don't first filter out the yaml and then update the snippet.
 
 Given a sample.yml file of:
 ```yaml
@@ -199,7 +204,8 @@ yq '.a.b |= "bogs"' sample.yml
 ```
 will output
 ```yaml
-{a: {b: bogs}}
+a:
+  b: bogs
 ```
 
 ## Update node value that has an anchor
@@ -229,6 +235,42 @@ yq '.a.b.[0] |= "bogs"' sample.yml
 ```
 will output
 ```yaml
-{a: {b: [bogs]}}
+a:
+  b:
+    - bogs
+```
+
+## Custom types are maintained by default
+Given a sample.yml file of:
+```yaml
+a: !cat meow
+b: !dog woof
+```
+then
+```bash
+yq '.a = .b' sample.yml
+```
+will output
+```yaml
+a: !cat woof
+b: !dog woof
+```
+
+## Custom types: clovver
+Use the `c` option to clobber custom tags
+
+Given a sample.yml file of:
+```yaml
+a: !cat meow
+b: !dog woof
+```
+then
+```bash
+yq '.a =c .b' sample.yml
+```
+will output
+```yaml
+a: !dog woof
+b: !dog woof
 ```
 

pkg/yqlib/doc/operators/boolean-operators.md

@@ -10,6 +10,12 @@ The `or` and `and` operators take two parameters and return a boolean result.
 
 These are most commonly used with the `select` operator to filter particular nodes.
 
+## Related Operators
+
+- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals)
+- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare)
+- select operator [here](https://mikefarah.gitbook.io/yq/operators/select)
+
 {% hint style="warning" %}
 Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
 

pkg/yqlib/doc/operators/comment-operators.md

@@ -1,6 +1,6 @@
 # Comment Operators
 
-Use these comment operators to set or retrieve comments.
+Use these comment operators to set or retrieve comments. Note that line comments on maps/arrays are actually set on the _key_ node as opposed to the _value_ (map/array). See below for examples.
 
 Like the `=` and `|=` assign operators, the same syntax applies when updating comments:
 
@@ -17,6 +17,8 @@ Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#
 {% endhint %}
 
 ## Set line comment
+Set the comment on the key node for more reliability (see below).
+
 Given a sample.yml file of:
 ```yaml
 a: cat
@@ -30,6 +32,24 @@ will output
 a: cat # single
 ```
 
+## Set line comment of a maps/arrays
+For maps and arrays, you need to set the line comment on the _key_ node. This will also work for scalars.
+
+Given a sample.yml file of:
+```yaml
+a:
+  b: things
+```
+then
+```bash
+yq '(.a | key) line_comment="single"' sample.yml
+```
+will output
+```yaml
+a: # single
+  b: things
+```
+
 ## Use update assign to perform relative updates
 Given a sample.yml file of:
 ```yaml
@@ -46,6 +66,121 @@ a: cat # cat
 b: dog # dog
 ```
 
+## Where is the comment - map key example
+The underlying yaml parser can assign comments in a document to surprising nodes. Use an expression like this to find where you comment is. 'p' indicates the path, 'isKey' is if the node is a map key (as opposed to a map value).
+From this, you can see the 'hello-world-comment' is actually on the 'hello' key
+
+Given a sample.yml file of:
+```yaml
+hello: # hello-world-comment
+  message: world
+```
+then
+```bash
+yq '[... | {"p": path | join("."), "isKey": is_key, "hc": headComment, "lc": lineComment, "fc": footComment}]' sample.yml
+```
+will output
+```yaml
+- p: ""
+  isKey: false
+  hc: ""
+  lc: ""
+  fc: ""
+- p: hello
+  isKey: true
+  hc: ""
+  lc: hello-world-comment
+  fc: ""
+- p: hello
+  isKey: false
+  hc: ""
+  lc: ""
+  fc: ""
+- p: hello.message
+  isKey: true
+  hc: ""
+  lc: ""
+  fc: ""
+- p: hello.message
+  isKey: false
+  hc: ""
+  lc: ""
+  fc: ""
+```
+
+## Retrieve comment - map key example
+From the previous example, we know that the comment is on the 'hello' _key_ as a lineComment
+
+Given a sample.yml file of:
+```yaml
+hello: # hello-world-comment
+  message: world
+```
+then
+```bash
+yq '.hello | key | line_comment' sample.yml
+```
+will output
+```yaml
+hello-world-comment
+```
+
+## Where is the comment - array example
+The underlying yaml parser can assign comments in a document to surprising nodes. Use an expression like this to find where you comment is. 'p' indicates the path, 'isKey' is if the node is a map key (as opposed to a map value).
+From this, you can see the 'under-name-comment' is actually on the first child
+
+Given a sample.yml file of:
+```yaml
+name:
+  # under-name-comment
+  - first-array-child
+```
+then
+```bash
+yq '[... | {"p": path | join("."), "isKey": is_key, "hc": headComment, "lc": lineComment, "fc": footComment}]' sample.yml
+```
+will output
+```yaml
+- p: ""
+  isKey: false
+  hc: ""
+  lc: ""
+  fc: ""
+- p: name
+  isKey: true
+  hc: ""
+  lc: ""
+  fc: ""
+- p: name
+  isKey: false
+  hc: ""
+  lc: ""
+  fc: ""
+- p: name.0
+  isKey: false
+  hc: under-name-comment
+  lc: ""
+  fc: ""
+```
+
+## Retrieve comment - array example
+From the previous example, we know that the comment is on the first child as a headComment
+
+Given a sample.yml file of:
+```yaml
+name:
+  # under-name-comment
+  - first-array-child
+```
+then
+```bash
+yq '.name[0] | headComment' sample.yml
+```
+will output
+```yaml
+under-name-comment
+```
+
 ## Set head comment
 Given a sample.yml file of:
 ```yaml
@@ -62,6 +197,25 @@ will output
 a: cat
 ```
 
+## Set head comment of a map entry
+Given a sample.yml file of:
+```yaml
+f: foo
+a:
+  b: cat
+```
+then
+```bash
+yq '(.a | key) head_comment="single"' sample.yml
+```
+will output
+```yaml
+f: foo
+# single
+a:
+  b: cat
+```
+
 ## Set foot comment, using an expression
 Given a sample.yml file of:
 ```yaml
@@ -74,7 +228,6 @@ yq '. foot_comment=.a' sample.yml
 will output
 ```yaml
 a: cat
-
 # cat
 ```
 
@@ -117,6 +270,7 @@ b:
 Given a sample.yml file of:
 ```yaml
 a: cat # meow
+# have a great day
 ```
 then
 ```bash

pkg/yqlib/doc/operators/compare.md

@@ -0,0 +1,106 @@
+# Compare Operators
+
+Comparison operators (`>`, `>=`, `<`, `<=`) can be used for comparing scalar values of the same time.
+
+The following types are currently supported:
+
+- numbers
+- strings
+- datetimes
+
+## Related Operators
+
+- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals)
+- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators)
+- select operator [here](https://mikefarah.gitbook.io/yq/operators/select)
+
+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
+## Compare numbers (>)
+Given a sample.yml file of:
+```yaml
+a: 5
+b: 4
+```
+then
+```bash
+yq '.a > .b' sample.yml
+```
+will output
+```yaml
+true
+```
+
+## Compare equal numbers (>=)
+Given a sample.yml file of:
+```yaml
+a: 5
+b: 5
+```
+then
+```bash
+yq '.a >= .b' sample.yml
+```
+will output
+```yaml
+true
+```
+
+## Compare strings
+Compares strings by their bytecode.
+
+Given a sample.yml file of:
+```yaml
+a: zoo
+b: apple
+```
+then
+```bash
+yq '.a > .b' sample.yml
+```
+will output
+```yaml
+true
+```
+
+## Compare date times
+You can compare date times. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.
+
+Given a sample.yml file of:
+```yaml
+a: 2021-01-01T03:10:00Z
+b: 2020-01-01T03:10:00Z
+```
+then
+```bash
+yq '.a > .b' sample.yml
+```
+will output
+```yaml
+true
+```
+
+## Both sides are null: > is false
+Running
+```bash
+yq --null-input '.a > .b'
+```
+will output
+```yaml
+false
+```
+
+## Both sides are null: >= is true
+Running
+```bash
+yq --null-input '.a >= .b'
+```
+will output
+```yaml
+true
+```
+

pkg/yqlib/doc/operators/create-collect-into-object.md

@@ -86,3 +86,16 @@ will output
 wrap: frog
 ```
 
+## Creating yaml from scratch with multiple objects
+Running
+```bash
+yq --null-input '(.a.b = "foo") | (.d.e = "bar")'
+```
+will output
+```yaml
+a:
+  b: foo
+d:
+  e: bar
+```
+

pkg/yqlib/doc/operators/datetime.md

@@ -13,11 +13,11 @@ To specify a custom parsing format, use the `with_dtf` operator. The first param
 yq 'with_dtf("myformat"; .a + "3h" | tz("Australia/Melbourne"))'
 ```
 
-See https://pkg.go.dev/time#pkg-constants for examples of formatting options.
+See the [library docs](https://pkg.go.dev/time#pkg-constants) for examples of formatting options.
 
 
 ## Timezones
-This uses golangs built in LoadLocation function to parse timezones strings. See https://pkg.go.dev/time#LoadLocation for more details.
+This uses golangs built in LoadLocation function to parse timezones strings. See the [library docs](https://pkg.go.dev/time#LoadLocation) for more details.
 
 
 ## Durations
@@ -156,7 +156,7 @@ a: 2021-01-01T03:10:00Z
 ```
 
 ## Date subtraction
-You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.
+You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/datetime#date-time-formattings) for more information.
 
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/operators/encode-decode.md

@@ -4,21 +4,21 @@ Encode operators will take the piped in object structure and encode it as a stri
 
 Note that you can optionally pass an indent value to the encode functions (see below).
 
-These operators are useful to process yaml documents that have stringified embeded yaml/json/props in them.
+These operators are useful to process yaml documents that have stringified embedded yaml/json/props in them.
 
 
 | Format | Decode (from string) | Encode (to string) |
 | --- | -- | --|
-| Yaml | from_yaml | to_yaml(i)/@yaml |
-| JSON | from_json | to_json(i)/@json |
-| Properties |  | to_props/@props |
-| CSV |  | to_csv/@csv |
-| TSV |  | to_tsv/@tsv |
-| XML | from_xml | to_xml(i)/@xml |
+| Yaml | from_yaml/@yamld | to_yaml(i)/@yaml |
+| JSON | from_json/@jsond | to_json(i)/@json |
+| Properties | from_props/@propsd  | to_props/@props |
+| CSV | from_csv/@csvd | to_csv/@csv |
+| TSV | from_tsv/@tsvd | to_tsv/@tsv |
+| XML | from_xml/@xmld | to_xml(i)/@xml |
 | Base64 | @base64d | @base64 |
 
 
-CSV and TSV format both accept either a single array or scalars (representing a single row), or an array of array of scalars (representing multiple rows). 
+See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) for accepted formats.
 
 XML uses the `--xml-attribute-prefix` and `xml-content-name` flags to identify attributes and content fields.
 
@@ -123,6 +123,60 @@ b: |
   cool = thing
 ```
 
+## Decode props encoded string
+Given a sample.yml file of:
+```yaml
+a: |-
+  cats=great
+  dogs=cool as well
+```
+then
+```bash
+yq '.a |= @propsd' sample.yml
+```
+will output
+```yaml
+a:
+  cats: great
+  dogs: cool as well
+```
+
+## Decode csv encoded string
+Given a sample.yml file of:
+```yaml
+a: |-
+  cats,dogs
+  great,cool as well
+```
+then
+```bash
+yq '.a |= @csvd' sample.yml
+```
+will output
+```yaml
+a:
+  - cats: great
+    dogs: cool as well
+```
+
+## Decode tsv encoded string
+Given a sample.yml file of:
+```yaml
+a: |-
+  cats	dogs
+  great	cool as well
+```
+then
+```bash
+yq '.a |= @tsvd' sample.yml
+```
+will output
+```yaml
+a:
+  - cats: great
+    dogs: cool as well
+```
+
 ## Encode value as yaml string
 Indent defaults to 2
 

pkg/yqlib/doc/operators/entries.md

@@ -107,6 +107,26 @@ KEY_a: 1
 KEY_b: 2
 ```
 
+## Custom sort map keys
+Use to_entries to convert to an array of key/value pairs, sort the array using sort/sort_by/etc, and convert it back.
+
+Given a sample.yml file of:
+```yaml
+a: 1
+c: 3
+b: 2
+```
+then
+```bash
+yq 'to_entries | sort_by(.key) | reverse | from_entries' sample.yml
+```
+will output
+```yaml
+c: 3
+b: 2
+a: 1
+```
+
 ## Use with_entries to filter the map
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/operators/equals.md

@@ -14,6 +14,13 @@ select(.a == .b)
 
 The not equals `!=` operator returns `false` if the LHS is equal to the RHS.
 
+## Related Operators
+
+- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare)
+- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators)
+- select operator [here](https://mikefarah.gitbook.io/yq/operators/select)
+
+
 {% hint style="warning" %}
 Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
 
@@ -74,7 +81,7 @@ true
 false
 ```
 
-## Dont match number
+## Don't match number
 Given a sample.yml file of:
 ```yaml
 - 3
@@ -102,7 +109,7 @@ will output
 true
 ```
 
-## Non exisitant key doesn't equal a value
+## Non existent key doesn't equal a value
 Given a sample.yml file of:
 ```yaml
 a: frog

pkg/yqlib/doc/operators/error.md

@@ -0,0 +1,56 @@
+# Error
+
+Use this operation to short-circuit expressions. Useful for validation.
+
+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
+
+`yq e <exp> <file>`
+{% endhint %}
+
+## Validate a particular value
+Given a sample.yml file of:
+```yaml
+a: hello
+```
+then
+```bash
+yq 'select(.a == "howdy") or error(".a [" + .a + "] is not howdy!")' sample.yml
+```
+will output
+```bash
+Error: .a [hello] is not howdy!
+```
+
+## Validate the environment variable is a number - invalid
+Running
+```bash
+numberOfCats="please" yq --null-input 'env(numberOfCats) | select(tag == "!!int") or error("numberOfCats is not a number :(")'
+```
+will output
+```bash
+Error: numberOfCats is not a number :(
+```
+
+## Validate the environment variable is a number - valid
+`with` can be a convenient way of encapsulating validation.
+
+Given a sample.yml file of:
+```yaml
+name: Bob
+favouriteAnimal: cat
+```
+then
+```bash
+numberOfCats="3" yq '
+	with(env(numberOfCats); select(tag == "!!int") or error("numberOfCats is not a number :(")) | 
+	.numPets = env(numberOfCats)
+' sample.yml
+```
+will output
+```yaml
+name: Bob
+favouriteAnimal: cat
+numPets: 3
+```
+

pkg/yqlib/doc/operators/headers/assign-update.md

@@ -6,4 +6,8 @@ This operator is used to update node values. It can be used in either the:
 Which will assign the LHS node values to the RHS node values. The RHS expression is run against the matching nodes in the pipeline.
 
 ### relative form: `|=`
-This will do a similar thing to the plain form, however, the RHS expression is run against _the LHS nodes_. This is useful for updating values based on old values, e.g. increment.
+This will do a similar thing to the plain form, however, the RHS expression is run against _the LHS nodes_. This is useful for updating values based on old values, e.g. increment.
+
+
+### Flags
+- `c` clobber custom tags

pkg/yqlib/doc/operators/headers/boolean-operators.md

@@ -9,3 +9,9 @@ The `or` and `and` operators take two parameters and return a boolean result.
 `any_c(condition)` and `all_c(condition)` are like `any` and `all` but they take a condition expression that is used against each element to determine if it's `true`. Note: in `jq` you can simply pass a condition to `any` or `all` and it simply works - `yq` isn't that clever..yet
 
 These are most commonly used with the `select` operator to filter particular nodes.
+
+## Related Operators
+
+- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals)
+- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare)
+- select operator [here](https://mikefarah.gitbook.io/yq/operators/select)

pkg/yqlib/doc/operators/headers/comment-operators.md

@@ -1,6 +1,6 @@
 # Comment Operators
 
-Use these comment operators to set or retrieve comments.
+Use these comment operators to set or retrieve comments. Note that line comments on maps/arrays are actually set on the _key_ node as opposed to the _value_ (map/array). See below for examples.
 
 Like the `=` and `|=` assign operators, the same syntax applies when updating comments:
 

pkg/yqlib/doc/operators/headers/compare.md

@@ -0,0 +1,15 @@
+# Compare Operators
+
+Comparison operators (`>`, `>=`, `<`, `<=`) can be used for comparing scalar values of the same time.
+
+The following types are currently supported:
+
+- numbers
+- strings
+- datetimes
+
+## Related Operators
+
+- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals)
+- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators)
+- select operator [here](https://mikefarah.gitbook.io/yq/operators/select)

pkg/yqlib/doc/operators/headers/datetime.md

@@ -13,11 +13,11 @@ To specify a custom parsing format, use the `with_dtf` operator. The first param
 yq 'with_dtf("myformat"; .a + "3h" | tz("Australia/Melbourne"))'
 ```
 
-See https://pkg.go.dev/time#pkg-constants for examples of formatting options.
+See the [library docs](https://pkg.go.dev/time#pkg-constants) for examples of formatting options.
 
 
 ## Timezones
-This uses golangs built in LoadLocation function to parse timezones strings. See https://pkg.go.dev/time#LoadLocation for more details.
+This uses golangs built in LoadLocation function to parse timezones strings. See the [library docs](https://pkg.go.dev/time#LoadLocation) for more details.
 
 
 ## Durations

pkg/yqlib/doc/operators/headers/encode-decode.md

@@ -4,21 +4,21 @@ Encode operators will take the piped in object structure and encode it as a stri
 
 Note that you can optionally pass an indent value to the encode functions (see below).
 
-These operators are useful to process yaml documents that have stringified embeded yaml/json/props in them.
+These operators are useful to process yaml documents that have stringified embedded yaml/json/props in them.
 
 
 | Format | Decode (from string) | Encode (to string) |
 | --- | -- | --|
-| Yaml | from_yaml | to_yaml(i)/@yaml |
-| JSON | from_json | to_json(i)/@json |
-| Properties |  | to_props/@props |
-| CSV |  | to_csv/@csv |
-| TSV |  | to_tsv/@tsv |
-| XML | from_xml | to_xml(i)/@xml |
+| Yaml | from_yaml/@yamld | to_yaml(i)/@yaml |
+| JSON | from_json/@jsond | to_json(i)/@json |
+| Properties | from_props/@propsd  | to_props/@props |
+| CSV | from_csv/@csvd | to_csv/@csv |
+| TSV | from_tsv/@tsvd | to_tsv/@tsv |
+| XML | from_xml/@xmld | to_xml(i)/@xml |
 | Base64 | @base64d | @base64 |
 
 
-CSV and TSV format both accept either a single array or scalars (representing a single row), or an array of array of scalars (representing multiple rows). 
+See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) for accepted formats.
 
 XML uses the `--xml-attribute-prefix` and `xml-content-name` flags to identify attributes and content fields.
 

pkg/yqlib/doc/operators/headers/equals.md

@@ -13,3 +13,10 @@ select(.a == .b)
 ```
 
 The not equals `!=` operator returns `false` if the LHS is equal to the RHS.
+
+## Related Operators
+
+- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare)
+- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators)
+- select operator [here](https://mikefarah.gitbook.io/yq/operators/select)
+

pkg/yqlib/doc/operators/headers/error.md

@@ -0,0 +1,3 @@
+# Error
+
+Use this operation to short-circuit expressions. Useful for validation.

pkg/yqlib/doc/operators/headers/multiply-merge.md

@@ -14,6 +14,7 @@ You can control how objects are merged by using one or more of the following fla
 - `d` deeply merge arrays
 - `?` only merge _existing_ fields
 - `n` only merge _new_ fields
+- `c` clobber custom tags
 
 
 ### Merge two files together
@@ -28,3 +29,9 @@ Note the use of `eval-all` to ensure all documents are loaded into memory.
 ```bash
 yq eval-all '. as $item ireduce ({}; . * $item )' *.yml
 ```
+
+# Merging complex arrays together by a key field
+By default - `yq` merge is naive. It merges maps when they match the key name, and arrays are merged either by appending them together, or merging the entries by their position in the array.
+
+For more complex array merging (e.g. merging items that match on a certain key) please see the example [here](https://mikefarah.gitbook.io/yq/operators/multiply-merge#merge-arrays-of-objects-together-matching-on-a-key)
+

pkg/yqlib/doc/operators/headers/path.md

@@ -1,5 +1,8 @@
 # Path
 
-The path operator can be used to get the traversal paths of matching nodes in an expression. The path is returned as an array, which if traversed in order will lead to the matching node.
+The `path` operator can be used to get the traversal paths of matching nodes in an expression. The path is returned as an array, which if traversed in order will lead to the matching node.
 
 You can get the key/index of matching nodes by using the `path` operator to return the path array then piping that through `.[-1]` to get the last element of that array, the key.
+
+Use `setpath` to set a value to the path array returned by `path`, and similarly `delpaths` for an array of path arrays.
+

pkg/yqlib/doc/operators/headers/reduce.md

@@ -19,4 +19,4 @@ On the RHS there is `<init>`, the starting value of the accumulator and `<block>
 ## yq vs jq syntax
 Reduce syntax in `yq` is a little different from `jq` - as `yq` (currently) isn't as sophisticated as `jq` and its only supports infix notation (e.g. a + b, where the operator is in the middle of the two parameters) - where as `jq` uses a mix of infix notation with _prefix_ notation (e.g. `reduce a b` is like writing `+ a b`).
 
-To that end, the reduce operator is called `ireduce` for backwards compatability if a `jq` like prefix version of `reduce` is ever added.
+To that end, the reduce operator is called `ireduce` for backwards compatibility if a `jq` like prefix version of `reduce` is ever added.

pkg/yqlib/doc/operators/headers/select.md

@@ -1,3 +1,9 @@
 # Select
 
 Select is used to filter arrays and maps by a boolean expression.
+
+## Related Operators
+
+- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals)
+- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare)
+- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators)

pkg/yqlib/doc/operators/headers/sort-keys.md

@@ -11,3 +11,6 @@ diff file1.yml file2.yml
 ```
 
 Note that `yq` does not yet consider anchors when sorting by keys - this may result in invalid yaml documents if your are using merge anchors.
+
+For more advanced sorting, using `to_entries` to convert the map to an array, then sort/process the array as you like (e.g. using `sort_by`) and convert back to a map using `from_entries`.
+See [here](https://mikefarah.gitbook.io/yq/operators/entries#custom-sort-map-keys) for an example. 

pkg/yqlib/doc/operators/headers/string-operators.md

@@ -1,7 +1,9 @@
 # String Operators
 
 ## RegEx
-This uses golangs native regex functions under the hood - See https://github.com/google/re2/wiki/Syntax for the supported syntax.
+This uses golangs native regex functions under the hood - See their [docs](https://github.com/google/re2/wiki/Syntax) for the supported syntax.
+
+Case insensitive tip: prefix the regex with `(?i)` - e.g. `test("(?i)cats)"`.
 
 ### match(regEx)
 This operator returns the substring match details of the given regEx.
@@ -9,7 +11,7 @@ This operator returns the substring match details of the given regEx.
 ### capture(regEx)
 Capture returns named RegEx capture groups in a map. Can be more convenient than `match` depending on what you are doing.
 
-# test(regEx)
+## test(regEx)
 Returns true if the string matches the RegEx, false otherwise.
 
 ## sub(regEx, replacement)

pkg/yqlib/doc/operators/headers/unique.md

@@ -1,3 +1,4 @@
 # Unique
 
-This is used to filter out duplicated items in an array.
+This is used to filter out duplicated items in an array. Note that the original order of the array is maintained.
+

pkg/yqlib/doc/operators/keys.md

@@ -118,3 +118,43 @@ will output
 comment on key
 ```
 
+## Check node is a key
+Given a sample.yml file of:
+```yaml
+a:
+  b:
+    - cat
+  c: frog
+```
+then
+```bash
+yq '[... | { "p": path | join("."), "isKey": is_key, "tag": tag }]' sample.yml
+```
+will output
+```yaml
+- p: ""
+  isKey: false
+  tag: '!!map'
+- p: a
+  isKey: true
+  tag: '!!str'
+- p: a
+  isKey: false
+  tag: '!!map'
+- p: a.b
+  isKey: true
+  tag: '!!str'
+- p: a.b
+  isKey: false
+  tag: '!!seq'
+- p: a.b.0
+  isKey: false
+  tag: '!!str'
+- p: a.c
+  isKey: true
+  tag: '!!str'
+- p: a.c
+  isKey: false
+  tag: '!!str'
+```
+

pkg/yqlib/doc/operators/multiply-merge.md

@@ -14,6 +14,7 @@ You can control how objects are merged by using one or more of the following fla
 - `d` deeply merge arrays
 - `?` only merge _existing_ fields
 - `n` only merge _new_ fields
+- `c` clobber custom tags
 
 
 ### Merge two files together
@@ -29,6 +30,12 @@ Note the use of `eval-all` to ensure all documents are loaded into memory.
 yq eval-all '. as $item ireduce ({}; . * $item )' *.yml
 ```
 
+# Merging complex arrays together by a key field
+By default - `yq` merge is naive. It merges maps when they match the key name, and arrays are merged either by appending them together, or merging the entries by their position in the array.
+
+For more complex array merging (e.g. merging items that match on a certain key) please see the example [here](https://mikefarah.gitbook.io/yq/operators/multiply-merge#merge-arrays-of-objects-together-matching-on-a-key)
+
+
 {% hint style="warning" %}
 Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
 
@@ -475,3 +482,26 @@ b: !goat
   dog: woof
 ```
 
+## Custom types: clobber tags
+Use the `c` option to clobber custom tags. Note that the second tag is now used
+
+Given a sample.yml file of:
+```yaml
+a: !horse
+  cat: meow
+b: !goat
+  dog: woof
+```
+then
+```bash
+yq '.a *=c .b' sample.yml
+```
+will output
+```yaml
+a: !goat
+  cat: meow
+  dog: woof
+b: !goat
+  dog: woof
+```
+

pkg/yqlib/doc/operators/path.md

@@ -1,9 +1,12 @@
 # Path
 
-The path operator can be used to get the traversal paths of matching nodes in an expression. The path is returned as an array, which if traversed in order will lead to the matching node.
+The `path` operator can be used to get the traversal paths of matching nodes in an expression. The path is returned as an array, which if traversed in order will lead to the matching node.
 
 You can get the key/index of matching nodes by using the `path` operator to return the path array then piping that through `.[-1]` to get the last element of that array, the key.
 
+Use `setpath` to set a value to the path array returned by `path`, and similarly `delpaths` for an array of path arrays.
+
+
 {% hint style="warning" %}
 Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
 
@@ -98,3 +101,114 @@ will output
   value: frog
 ```
 
+## Set path
+Given a sample.yml file of:
+```yaml
+a:
+  b: cat
+```
+then
+```bash
+yq 'setpath(["a", "b"]; "things")' sample.yml
+```
+will output
+```yaml
+a:
+  b: things
+```
+
+## Set on empty document
+Running
+```bash
+yq --null-input 'setpath(["a", "b"]; "things")'
+```
+will output
+```yaml
+a:
+  b: things
+```
+
+## Set array path
+Given a sample.yml file of:
+```yaml
+a:
+  - cat
+  - frog
+```
+then
+```bash
+yq 'setpath(["a", 0]; "things")' sample.yml
+```
+will output
+```yaml
+a:
+  - things
+  - frog
+```
+
+## Set array path empty
+Running
+```bash
+yq --null-input 'setpath(["a", 0]; "things")'
+```
+will output
+```yaml
+a:
+  - things
+```
+
+## Delete path
+Notice delpaths takes an _array_ of paths.
+
+Given a sample.yml file of:
+```yaml
+a:
+  b: cat
+  c: dog
+  d: frog
+```
+then
+```bash
+yq 'delpaths([["a", "c"], ["a", "d"]])' sample.yml
+```
+will output
+```yaml
+a:
+  b: cat
+```
+
+## Delete array path
+Given a sample.yml file of:
+```yaml
+a:
+  - cat
+  - frog
+```
+then
+```bash
+yq 'delpaths([["a", 0]])' sample.yml
+```
+will output
+```yaml
+a:
+  - frog
+```
+
+## Delete - wrong parameter
+delpaths does not work with a single path array
+
+Given a sample.yml file of:
+```yaml
+a:
+  - cat
+  - frog
+```
+then
+```bash
+yq 'delpaths(["a", 0])' sample.yml
+```
+will output
+```bash
+Error: DELPATHS: expected entry [0] to be a sequence, but its a !!str. Note that delpaths takes an array of path arrays, e.g. [["a", "b"]]
+```
+

pkg/yqlib/doc/operators/reduce.md

@@ -19,7 +19,7 @@ On the RHS there is `<init>`, the starting value of the accumulator and `<block>
 ## yq vs jq syntax
 Reduce syntax in `yq` is a little different from `jq` - as `yq` (currently) isn't as sophisticated as `jq` and its only supports infix notation (e.g. a + b, where the operator is in the middle of the two parameters) - where as `jq` uses a mix of infix notation with _prefix_ notation (e.g. `reduce a b` is like writing `+ a b`).
 
-To that end, the reduce operator is called `ireduce` for backwards compatability if a `jq` like prefix version of `reduce` is ever added.
+To that end, the reduce operator is called `ireduce` for backwards compatibility if a `jq` like prefix version of `reduce` is ever added.
 
 {% hint style="warning" %}
 Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;

pkg/yqlib/doc/operators/select.md

@@ -2,6 +2,12 @@
 
 Select is used to filter arrays and maps by a boolean expression.
 
+## Related Operators
+
+- equals / not equals (`==`, `!=`) operators [here](https://mikefarah.gitbook.io/yq/operators/equals)
+- comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare)
+- boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators)
+
 {% hint style="warning" %}
 Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
 

pkg/yqlib/doc/operators/sort-keys.md

@@ -12,6 +12,9 @@ diff file1.yml file2.yml
 
 Note that `yq` does not yet consider anchors when sorting by keys - this may result in invalid yaml documents if your are using merge anchors.
 
+For more advanced sorting, using `to_entries` to convert the map to an array, then sort/process the array as you like (e.g. using `sort_by`) and convert back to a map using `from_entries`.
+See [here](https://mikefarah.gitbook.io/yq/operators/entries#custom-sort-map-keys) for an example. 
+
 {% hint style="warning" %}
 Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
 

pkg/yqlib/doc/operators/string-operators.md

@@ -1,7 +1,9 @@
 # String Operators
 
 ## RegEx
-This uses golangs native regex functions under the hood - See https://github.com/google/re2/wiki/Syntax for the supported syntax.
+This uses golangs native regex functions under the hood - See their [docs](https://github.com/google/re2/wiki/Syntax) for the supported syntax.
+
+Case insensitive tip: prefix the regex with `(?i)` - e.g. `test("(?i)cats)"`.
 
 ### match(regEx)
 This operator returns the substring match details of the given regEx.
@@ -9,7 +11,7 @@ This operator returns the substring match details of the given regEx.
 ### capture(regEx)
 Capture returns named RegEx capture groups in a map. Can be more convenient than `match` depending on what you are doing.
 
-# test(regEx)
+## test(regEx)
 Returns true if the string matches the RegEx, false otherwise.
 
 ## sub(regEx, replacement)
@@ -110,6 +112,26 @@ will output
 cat; meow; 1; ; true
 ```
 
+## Trim strings
+Given a sample.yml file of:
+```yaml
+- ' cat'
+- 'dog '
+- ' cow cow '
+- horse
+```
+then
+```bash
+yq '.[] | trim' sample.yml
+```
+will output
+```yaml
+cat
+dog
+cow cow
+horse
+```
+
 ## Match string
 Given a sample.yml file of:
 ```yaml
@@ -264,7 +286,7 @@ will output
 ```
 
 ## Test using regex
-Like jq'q equivalent, this works like match but only returns true/false instead of full match details
+Like jq's equivalent, this works like match but only returns true/false instead of full match details
 
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/operators/tag.md

@@ -31,6 +31,29 @@ will output
 !!seq
 ```
 
+## type is an alias for tag
+Given a sample.yml file of:
+```yaml
+a: cat
+b: 5
+c: 3.2
+e: true
+f: []
+```
+then
+```bash
+yq '.. | type' sample.yml
+```
+will output
+```yaml
+!!map
+!!str
+!!int
+!!float
+!!bool
+!!seq
+```
+
 ## Set custom tag
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/operators/unique.md

@@ -1,6 +1,7 @@
 # Unique
 
-This is used to filter out duplicated items in an array.
+This is used to filter out duplicated items in an array. Note that the original order of the array is maintained.
+
 
 {% hint style="warning" %}
 Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
@@ -9,10 +10,12 @@ Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#
 {% endhint %}
 
 ## Unique array of scalars (string/numbers)
+Note that unique maintains the original order of the array.
+
 Given a sample.yml file of:
 ```yaml
-- 1
 - 2
+- 1
 - 3
 - 2
 ```
@@ -22,8 +25,8 @@ yq 'unique' sample.yml
 ```
 will output
 ```yaml
-- 1
 - 2
+- 1
 - 3
 ```
 

pkg/yqlib/doc/operators/variable-operators.md

@@ -46,7 +46,7 @@ Example taken from [jq](https://stedolan.github.io/jq/manual/#Variable/SymbolicB
 Given a sample.yml file of:
 ```yaml
 "posts":
-  - "title": Frist psot
+  - "title": First post
     "author": anon
   - "title": A well-written article
     "author": person1
@@ -60,7 +60,7 @@ yq '.realnames as $names | .posts[] | {"title":.title, "author": $names[.author]
 ```
 will output
 ```yaml
-title: Frist psot
+title: First post
 author: Anonymous Coward
 title: A well-written article
 author: Person McPherson

pkg/yqlib/doc/usage/convert.md

@@ -1,8 +1,9 @@
 # JSON
 
-Encode and decode to and from JSON. Note that YAML is a _superset_ of JSON - so `yq` can read any json file without doing anything special.
+Encode and decode to and from JSON. Supports multiple JSON documents in a single file (e.g. NDJSON).
+
+Note that YAML is a superset of (single document) JSON - so you don't have to use the JSON parser to read JSON when there is only one JSON document in the input. You will probably want to pretty print the result in this case, to get idiomatic YAML styling.
 
-This means you don't need to 'convert' a JSON file to YAML - however if you want idiomatic YAML styling, then you can use the `-P/--prettyPrint` flag, see examples below.
 
 {% hint style="warning" %}
 Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
@@ -130,3 +131,129 @@ will output
 {"whatever":"cat"}
 ```
 
+## Roundtrip NDJSON
+Unfortunately the json encoder strips leading spaces of values.
+
+Given a sample.json file of:
+```json
+{"this": "is a multidoc json file"}
+{"each": ["line is a valid json document"]}
+{"a number": 4}
+
+```
+then
+```bash
+yq -p=json -o=json -I=0 sample.json
+```
+will output
+```yaml
+{"this":"is a multidoc json file"}
+{"each":["line is a valid json document"]}
+{"a number":4}
+```
+
+## Roundtrip multi-document JSON
+The NDJSON parser can also handle multiple multi-line json documents in a single file!
+
+Given a sample.json file of:
+```json
+{
+	"this": "is a multidoc json file"
+}
+{
+	"it": [
+		"has",
+		"consecutive",
+		"json documents"
+	]
+}
+{
+	"a number": 4
+}
+
+```
+then
+```bash
+yq -p=json -o=json -I=2 sample.json
+```
+will output
+```yaml
+{
+  "this": "is a multidoc json file"
+}
+{
+  "it": [
+    "has",
+    "consecutive",
+    "json documents"
+  ]
+}
+{
+  "a number": 4
+}
+```
+
+## Update a specific document in a multi-document json
+Documents are indexed by the `documentIndex` or `di` operator.
+
+Given a sample.json file of:
+```json
+{"this": "is a multidoc json file"}
+{"each": ["line is a valid json document"]}
+{"a number": 4}
+
+```
+then
+```bash
+yq -p=json -o=json -I=0 '(select(di == 1) | .each ) += "cool"' sample.json
+```
+will output
+```yaml
+{"this":"is a multidoc json file"}
+{"each":["line is a valid json document","cool"]}
+{"a number":4}
+```
+
+## Find and update a specific document in a multi-document json
+Use expressions as you normally would.
+
+Given a sample.json file of:
+```json
+{"this": "is a multidoc json file"}
+{"each": ["line is a valid json document"]}
+{"a number": 4}
+
+```
+then
+```bash
+yq -p=json -o=json -I=0 '(select(has("each")) | .each ) += "cool"' sample.json
+```
+will output
+```yaml
+{"this":"is a multidoc json file"}
+{"each":["line is a valid json document","cool"]}
+{"a number":4}
+```
+
+## Decode NDJSON
+Given a sample.json file of:
+```json
+{"this": "is a multidoc json file"}
+{"each": ["line is a valid json document"]}
+{"a number": 4}
+
+```
+then
+```bash
+yq -p=json sample.json
+```
+will output
+```yaml
+this: is a multidoc json file
+---
+each:
+    - line is a valid json document
+---
+a number: 4
+```
+

pkg/yqlib/doc/usage/csv-tsv.md

@@ -0,0 +1,214 @@
+# CSV
+Encode/Decode/Roundtrip CSV and TSV files.
+
+## Encode 
+Currently supports arrays of homogenous flat objects, that is: no nesting and it assumes the _first_ object has all the keys required:
+
+```yaml
+- name: Bobo
+  type: dog
+- name: Fifi
+  type: cat
+```
+
+As well as arrays of arrays of scalars (strings/numbers/booleans):
+
+```yaml
+- [Bobo, dog]
+- [Fifi, cat]
+```
+
+## Decode
+Decode assumes the first CSV/TSV row is the header row, and all rows beneath are the entries.
+The data will be coded into an array of objects, using the header rows as keys.
+
+```csv
+name,type
+Bobo,dog
+Fifi,cat
+```
+
+
+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified. 
+
+`yq e <exp> <file>`
+{% endhint %}
+
+## Encode CSV simple
+Given a sample.yml file of:
+```yaml
+- [i, like, csv]
+- [because, excel, is, cool]
+```
+then
+```bash
+yq -o=csv sample.yml
+```
+will output
+```csv
+i,like,csv
+because,excel,is,cool
+```
+
+## Encode TSV simple
+Given a sample.yml file of:
+```yaml
+- [i, like, csv]
+- [because, excel, is, cool]
+```
+then
+```bash
+yq -o=tsv sample.yml
+```
+will output
+```tsv
+i	like	csv
+because	excel	is	cool
+```
+
+## Encode array of objects to csv
+Given a sample.yml file of:
+```yaml
+- name: Gary
+  numberOfCats: 1
+  likesApples: true
+  height: 168.8
+- name: Samantha's Rabbit
+  numberOfCats: 2
+  likesApples: false
+  height: -188.8
+
+```
+then
+```bash
+yq -o=csv sample.yml
+```
+will output
+```csv
+name,numberOfCats,likesApples,height
+Gary,1,true,168.8
+Samantha's Rabbit,2,false,-188.8
+```
+
+## Encode array of objects to custom csv format
+Add the header row manually, then the we convert each object into an array of values - resulting in an array of arrays. Pick the columns and call the header whatever you like.
+
+Given a sample.yml file of:
+```yaml
+- name: Gary
+  numberOfCats: 1
+  likesApples: true
+  height: 168.8
+- name: Samantha's Rabbit
+  numberOfCats: 2
+  likesApples: false
+  height: -188.8
+
+```
+then
+```bash
+yq -o=csv '[["Name", "Number of Cats"]] +  [.[] | [.name, .numberOfCats ]]' sample.yml
+```
+will output
+```csv
+Name,Number of Cats
+Gary,1
+Samantha's Rabbit,2
+```
+
+## Encode array of objects to csv - missing fields behaviour
+First entry is used to determine the headers, and it is missing 'likesApples', so it is not included in the csv. Second entry does not have 'numberOfCats' so that is blank
+
+Given a sample.yml file of:
+```yaml
+- name: Gary
+  numberOfCats: 1
+  height: 168.8
+- name: Samantha's Rabbit
+  height: -188.8
+  likesApples: false
+
+```
+then
+```bash
+yq -o=csv sample.yml
+```
+will output
+```csv
+name,numberOfCats,height
+Gary,1,168.8
+Samantha's Rabbit,,-188.8
+```
+
+## Parse CSV into an array of objects
+First row is assumed to be the header row.
+
+Given a sample.csv file of:
+```csv
+name,numberOfCats,likesApples,height
+Gary,1,true,168.8
+Samantha's Rabbit,2,false,-188.8
+
+```
+then
+```bash
+yq -p=csv sample.csv
+```
+will output
+```yaml
+- name: Gary
+  numberOfCats: 1
+  likesApples: true
+  height: 168.8
+- name: Samantha's Rabbit
+  numberOfCats: 2
+  likesApples: false
+  height: -188.8
+```
+
+## Parse TSV into an array of objects
+First row is assumed to be the header row.
+
+Given a sample.tsv file of:
+```tsv
+name	numberOfCats	likesApples	height
+Gary	1	true	168.8
+Samantha's Rabbit	2	false	-188.8
+
+```
+then
+```bash
+yq -p=tsv sample.tsv
+```
+will output
+```yaml
+- name: Gary
+  numberOfCats: 1
+  likesApples: true
+  height: 168.8
+- name: Samantha's Rabbit
+  numberOfCats: 2
+  likesApples: false
+  height: -188.8
+```
+
+## Round trip
+Given a sample.csv file of:
+```csv
+name,numberOfCats,likesApples,height
+Gary,1,true,168.8
+Samantha's Rabbit,2,false,-188.8
+
+```
+then
+```bash
+yq -p=csv -o=csv '(.[] | select(.name == "Gary") | .numberOfCats) = 3' sample.csv
+```
+will output
+```csv
+name,numberOfCats,likesApples,height
+Gary,3,true,168.8
+Samantha's Rabbit,2,false,-188.8
+```
+

pkg/yqlib/doc/usage/headers/convert.md

@@ -1,5 +1,6 @@
 # JSON
 
-Encode and decode to and from JSON. Note that YAML is a _superset_ of JSON - so `yq` can read any json file without doing anything special.
+Encode and decode to and from JSON. Supports multiple JSON documents in a single file (e.g. NDJSON).
+
+Note that YAML is a superset of (single document) JSON - so you don't have to use the JSON parser to read JSON when there is only one JSON document in the input. You will probably want to pretty print the result in this case, to get idiomatic YAML styling.
 
-This means you don't need to 'convert' a JSON file to YAML - however if you want idiomatic YAML styling, then you can use the `-P/--prettyPrint` flag, see examples below.

pkg/yqlib/doc/usage/headers/csv-tsv.md

@@ -0,0 +1,30 @@
+# CSV
+Encode/Decode/Roundtrip CSV and TSV files.
+
+## Encode 
+Currently supports arrays of homogenous flat objects, that is: no nesting and it assumes the _first_ object has all the keys required:
+
+```yaml
+- name: Bobo
+  type: dog
+- name: Fifi
+  type: cat
+```
+
+As well as arrays of arrays of scalars (strings/numbers/booleans):
+
+```yaml
+- [Bobo, dog]
+- [Fifi, cat]
+```
+
+## Decode
+Decode assumes the first CSV/TSV row is the header row, and all rows beneath are the entries.
+The data will be coded into an array of objects, using the header rows as keys.
+
+```csv
+name,type
+Bobo,dog
+Fifi,cat
+```
+

pkg/yqlib/doc/usage/headers/properties.md

@@ -1,5 +1,5 @@
 # Properties
 
-Encode to a property file (decode not yet supported). Line comments on value nodes will be copied across.
+Encode/Decode/Roundtrip to/from a property file. Line comments on value nodes will be copied across.
 
 By default, empty maps and arrays are not encoded - see below for an example on how to encode a value for these.

pkg/yqlib/doc/usage/properties.md

@@ -1,6 +1,6 @@
 # Properties
 
-Encode to a property file (decode not yet supported). Line comments on value nodes will be copied across.
+Encode/Decode/Roundtrip to/from a property file. Line comments on value nodes will be copied across.
 
 By default, empty maps and arrays are not encoded - see below for an example on how to encode a value for these.
 
@@ -17,7 +17,7 @@ Given a sample.yml file of:
 ```yaml
 # block comments don't come through
 person: # neither do comments on maps
-    name: Mike # comments on values appear
+    name: Mike Wazowski # comments on values appear
     pets: 
     - cat # comments on array values appear
     food: [pizza] # comments on arrays do not
@@ -32,7 +32,36 @@ yq -o=props sample.yml
 will output
 ```properties
 # comments on values appear
-person.name = Mike
+person.name = Mike Wazowski
+
+# comments on array values appear
+person.pets.0 = cat
+person.food.0 = pizza
+```
+
+## Encode properties: scalar encapsulation
+Note that string values with blank characters in them are encapsulated with double quotes
+
+Given a sample.yml file of:
+```yaml
+# block comments don't come through
+person: # neither do comments on maps
+    name: Mike Wazowski # comments on values appear
+    pets: 
+    - cat # comments on array values appear
+    food: [pizza] # comments on arrays do not
+emptyArray: []
+emptyMap: []
+
+```
+then
+```bash
+yq -o=props --unwrapScalar=false sample.yml
+```
+will output
+```properties
+# comments on values appear
+person.name = "Mike Wazowski"
 
 # comments on array values appear
 person.pets.0 = cat
@@ -44,7 +73,7 @@ Given a sample.yml file of:
 ```yaml
 # block comments don't come through
 person: # neither do comments on maps
-    name: Mike # comments on values appear
+    name: Mike Wazowski # comments on values appear
     pets: 
     - cat # comments on array values appear
     food: [pizza] # comments on arrays do not
@@ -58,7 +87,7 @@ yq -o=props '... comments = ""' sample.yml
 ```
 will output
 ```properties
-person.name = Mike
+person.name = Mike Wazowski
 person.pets.0 = cat
 person.food.0 = pizza
 ```
@@ -70,7 +99,7 @@ Given a sample.yml file of:
 ```yaml
 # block comments don't come through
 person: # neither do comments on maps
-    name: Mike # comments on values appear
+    name: Mike Wazowski # comments on values appear
     pets: 
     - cat # comments on array values appear
     food: [pizza] # comments on arrays do not
@@ -85,7 +114,7 @@ yq -o=props '(.. | select( (tag == "!!map" or tag =="!!seq") and length == 0)) =
 will output
 ```properties
 # comments on values appear
-person.name = Mike
+person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = cat
@@ -98,7 +127,7 @@ emptyMap =
 Given a sample.properties file of:
 ```properties
 # comments on values appear
-person.name = Mike
+person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = cat
@@ -112,7 +141,7 @@ yq -p=props sample.properties
 will output
 ```yaml
 person:
-    name: Mike # comments on values appear
+    name: Mike Wazowski # comments on values appear
     pets:
         - cat # comments on array values appear
     food:
@@ -123,7 +152,7 @@ person:
 Given a sample.properties file of:
 ```properties
 # comments on values appear
-person.name = Mike
+person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = cat
@@ -137,7 +166,7 @@ yq -p=props -o=props '.person.pets.0 = "dog"' sample.properties
 will output
 ```properties
 # comments on values appear
-person.name = Mike
+person.name = Mike Wazowski
 
 # comments on array values appear
 person.pets.0 = dog

pkg/yqlib/doc/usage/xml.md

@@ -120,6 +120,31 @@ cat:
   +legs: "4"
 ```
 
+## Parse xml: custom dtd
+DTD entities are ignored.
+
+Given a sample.xml file of:
+```xml
+
+<?xml version="1.0"?>
+<!DOCTYPE root [
+<!ENTITY writer "Blah.">
+<!ENTITY copyright "Blah">
+]>
+<root>
+    <item>&writer;&copyright;</item>
+</root>
+```
+then
+```bash
+yq -p=xml '.' sample.xml
+```
+will output
+```yaml
+root:
+  item: '&writer;&copyright;'
+```
+
 ## Parse xml: with comments
 A best attempt is made to preserve comments.
 
@@ -167,6 +192,52 @@ cat:
 # after cat
 ```
 
+## Parse xml: keep attribute namespace
+Given a sample.xml file of:
+```xml
+
+<?xml version="1.0"?>
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
+</map>
+
+```
+then
+```bash
+yq -p=xml -o=xml --xml-keep-namespace '.' sample.xml
+```
+will output
+```xml
+<map xmlns="some-namespace" xmlns:xsi="some-instance" some-instance:schemaLocation="some-url"></map>
+```
+
+instead of
+```xml
+<map xmlns="some-namespace" xsi="some-instance" schemaLocation="some-url"></map>
+```
+
+## Parse xml: keep raw attribute namespace
+Given a sample.xml file of:
+```xml
+
+<?xml version="1.0"?>
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
+</map>
+
+```
+then
+```bash
+yq -p=xml -o=xml --xml-keep-namespace --xml-raw-token '.' sample.xml
+```
+will output
+```xml
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url"></map>
+```
+
+instead of
+```xml
+<map xmlns="some-namespace" xsi="some-instance" schemaLocation="some-url"></map>
+```
+
 ## Encode xml: simple
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/encoder.go

@@ -66,8 +66,13 @@ func (o *orderedMap) UnmarshalJSON(data []byte) error {
 		}
 		return nil
 	case '[':
-		var arr []orderedMap
-		return json.Unmarshal(data, &arr)
+		var res []*orderedMap
+		if err := json.Unmarshal(data, &res); err != nil {
+			return err
+		}
+		o.altVal = res
+		o.kv = nil
+		return nil
 	}
 
 	return json.Unmarshal(data, &o.altVal)

pkg/yqlib/encoder_csv.go

@@ -13,7 +13,7 @@ type csvEncoder struct {
 }
 
 func NewCsvEncoder(separator rune) Encoder {
-	return &csvEncoder{separator}
+	return &csvEncoder{separator: separator}
 }
 
 func (e *csvEncoder) CanHandleAliases() bool {
@@ -41,6 +41,67 @@ func (e *csvEncoder) encodeRow(csvWriter *csv.Writer, contents []*yaml.Node) err
 	return csvWriter.Write(stringValues)
 }
 
+func (e *csvEncoder) encodeArrays(csvWriter *csv.Writer, content []*yaml.Node) error {
+	for i, child := range content {
+
+		if child.Kind != yaml.SequenceNode {
+			return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag)
+		}
+		err := e.encodeRow(csvWriter, child.Content)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (e *csvEncoder) extractHeader(child *yaml.Node) ([]*yaml.Node, error) {
+	if child.Kind != yaml.MappingNode {
+		return nil, fmt.Errorf("csv object encoding only works for arrays of flat objects (string key => string/numbers/boolean value), child[0] is a %v", child.Tag)
+	}
+	mapKeys := getMapKeys(child)
+	return mapKeys.Content, nil
+}
+
+func (e *csvEncoder) createChildRow(child *yaml.Node, headers []*yaml.Node) []*yaml.Node {
+	childRow := make([]*yaml.Node, 0)
+	for _, header := range headers {
+		keyIndex := findKeyInMap(child, header)
+		value := createScalarNode(nil, "")
+		if keyIndex != -1 {
+			value = child.Content[keyIndex+1]
+		}
+		childRow = append(childRow, value)
+	}
+	return childRow
+
+}
+
+func (e *csvEncoder) encodeObjects(csvWriter *csv.Writer, content []*yaml.Node) error {
+	headers, err := e.extractHeader(content[0])
+	if err != nil {
+		return nil
+	}
+
+	err = e.encodeRow(csvWriter, headers)
+	if err != nil {
+		return nil
+	}
+
+	for i, child := range content {
+		if child.Kind != yaml.MappingNode {
+			return fmt.Errorf("csv object encoding only works for arrays of flat objects (string key => string/numbers/boolean value), child[%v] is a %v", i, child.Tag)
+		}
+		row := e.createChildRow(child, headers)
+		err = e.encodeRow(csvWriter, row)
+		if err != nil {
+			return err
+		}
+
+	}
+	return nil
+}
+
 func (e *csvEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error {
 	csvWriter := csv.NewWriter(writer)
 	csvWriter.Comma = e.separator
@@ -56,15 +117,10 @@ func (e *csvEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error {
 		return e.encodeRow(csvWriter, node.Content)
 	}
 
-	for i, child := range node.Content {
-
-		if child.Kind != yaml.SequenceNode {
-			return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag)
-		}
-		err := e.encodeRow(csvWriter, child.Content)
-		if err != nil {
-			return err
-		}
+	if node.Content[0].Kind == yaml.MappingNode {
+		return e.encodeObjects(csvWriter, node.Content)
 	}
-	return nil
+
+	return e.encodeArrays(csvWriter, node.Content)
+
 }

pkg/yqlib/encoder_csv_test.go

@@ -1,60 +0,0 @@
-package yqlib
-
-import (
-	"bufio"
-	"bytes"
-	"strings"
-	"testing"
-
-	"github.com/mikefarah/yq/v4/test"
-)
-
-func yamlToCsv(sampleYaml string, separator rune) string {
-	var output bytes.Buffer
-	writer := bufio.NewWriter(&output)
-
-	var jsonEncoder = NewCsvEncoder(separator)
-	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
-	if err != nil {
-		panic(err)
-	}
-	node := inputs.Front().Value.(*CandidateNode).Node
-	err = jsonEncoder.Encode(writer, node)
-	if err != nil {
-		panic(err)
-	}
-	writer.Flush()
-
-	return strings.TrimSuffix(output.String(), "\n")
-}
-
-var sampleYaml = `["apple", apple2, "comma, in, value", "new
-line", 3, 3.40, true, "tab	here"]`
-
-var sampleYamlArray = "[" + sampleYaml + ", [bob, cat, meow, puss]]"
-
-func TestCsvEncoderEmptyArray(t *testing.T) {
-	var actualCsv = yamlToCsv(`[]`, ',')
-	test.AssertResult(t, "", actualCsv)
-}
-
-func TestCsvEncoder(t *testing.T) {
-	var expectedCsv = `apple,apple2,"comma, in, value",new line,3,3.40,true,tab	here`
-
-	var actualCsv = yamlToCsv(sampleYaml, ',')
-	test.AssertResult(t, expectedCsv, actualCsv)
-}
-
-func TestCsvEncoderArrayOfArrays(t *testing.T) {
-	var actualCsv = yamlToCsv(sampleYamlArray, ',')
-	var expectedCsv = "apple,apple2,\"comma, in, value\",new line,3,3.40,true,tab	here\nbob,cat,meow,puss"
-	test.AssertResult(t, expectedCsv, actualCsv)
-}
-
-func TestTsvEncoder(t *testing.T) {
-
-	var expectedCsv = `apple	apple2	comma, in, value	new line	3	3.40	true	"tab	here"`
-
-	var actualCsv = yamlToCsv(sampleYaml, '\t')
-	test.AssertResult(t, expectedCsv, actualCsv)
-}

pkg/yqlib/encoder_json.go

@@ -1,14 +1,16 @@
 package yqlib
 
 import (
-	"encoding/json"
+	"bytes"
 	"io"
 
+	"github.com/goccy/go-json"
 	yaml "gopkg.in/yaml.v3"
 )
 
 type jsonEncoder struct {
 	indentString string
+	colorise     bool
 }
 
 func mapKeysToStrings(node *yaml.Node) {
@@ -26,14 +28,14 @@ func mapKeysToStrings(node *yaml.Node) {
 	}
 }
 
-func NewJONEncoder(indent int) Encoder {
+func NewJSONEncoder(indent int, colorise bool) Encoder {
 	var indentString = ""
 
 	for index := 0; index < indent; index++ {
 		indentString = indentString + " "
 	}
 
-	return &jsonEncoder{indentString}
+	return &jsonEncoder{indentString, colorise}
 }
 
 func (je *jsonEncoder) CanHandleAliases() bool {
@@ -49,7 +51,14 @@ func (je *jsonEncoder) PrintLeadingContent(writer io.Writer, content string) err
 }
 
 func (je *jsonEncoder) Encode(writer io.Writer, node *yaml.Node) error {
-	var encoder = json.NewEncoder(writer)
+
+	destination := writer
+	tempBuffer := bytes.NewBuffer(nil)
+	if je.colorise {
+		destination = tempBuffer
+	}
+
+	var encoder = json.NewEncoder(destination)
 	encoder.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
 	encoder.SetIndent("", je.indentString)
 
@@ -60,5 +69,12 @@ func (je *jsonEncoder) Encode(writer io.Writer, node *yaml.Node) error {
 	if errorDecoding != nil {
 		return errorDecoding
 	}
-	return encoder.Encode(dataBucket)
+	err := encoder.Encode(dataBucket)
+	if err != nil {
+		return err
+	}
+	if je.colorise {
+		return colorizeAndPrint(tempBuffer.Bytes(), writer)
+	}
+	return nil
 }

pkg/yqlib/encoder_properties.go

@@ -12,10 +12,13 @@ import (
 )
 
 type propertiesEncoder struct {
+	unwrapScalar bool
 }
 
-func NewPropertiesEncoder() Encoder {
-	return &propertiesEncoder{}
+func NewPropertiesEncoder(unwrapScalar bool) Encoder {
+	return &propertiesEncoder{
+		unwrapScalar: unwrapScalar,
+	}
 }
 
 func (pe *propertiesEncoder) CanHandleAliases() bool {
@@ -75,7 +78,13 @@ func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yaml.Node,
 	p.SetComment(path, headAndLineComment(node))
 	switch node.Kind {
 	case yaml.ScalarNode:
-		_, _, err := p.Set(path, node.Value)
+		var nodeValue string
+		if pe.unwrapScalar || !strings.Contains(node.Value, " ") {
+			nodeValue = node.Value
+		} else {
+			nodeValue = fmt.Sprintf("%q", node.Value)
+		}
+		_, _, err := p.Set(path, nodeValue)
 		return err
 	case yaml.DocumentNode:
 		return pe.doEncode(p, node.Content[0], path)

pkg/yqlib/encoder_properties_test.go

@@ -9,11 +9,11 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
-func yamlToProps(sampleYaml string) string {
+func yamlToProps(sampleYaml string, unwrapScalar bool) string {
 	var output bytes.Buffer
 	writer := bufio.NewWriter(&output)
 
-	var propsEncoder = NewPropertiesEncoder()
+	var propsEncoder = NewPropertiesEncoder(unwrapScalar)
 	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
 		panic(err)
@@ -28,51 +28,100 @@ func yamlToProps(sampleYaml string) string {
 	return strings.TrimSuffix(output.String(), "\n")
 }
 
-func TestPropertiesEncoderSimple(t *testing.T) {
+func TestPropertiesEncoderSimple_Unwrapped(t *testing.T) {
 	var sampleYaml = `a: 'bob cool'`
 
 	var expectedProps = `a = bob cool`
-	var actualProps = yamlToProps(sampleYaml)
+	var actualProps = yamlToProps(sampleYaml, true)
 	test.AssertResult(t, expectedProps, actualProps)
 }
 
-func TestPropertiesEncoderSimpleWithComments(t *testing.T) {
+func TestPropertiesEncoderSimple_Wrapped(t *testing.T) {
+	var sampleYaml = `a: 'bob cool'`
+
+	var expectedProps = `a = "bob cool"`
+	var actualProps = yamlToProps(sampleYaml, false)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderSimpleWithComments_Unwrapped(t *testing.T) {
 	var sampleYaml = `a: 'bob cool' # line`
 
 	var expectedProps = `# line
 a = bob cool`
-	var actualProps = yamlToProps(sampleYaml)
+	var actualProps = yamlToProps(sampleYaml, true)
 	test.AssertResult(t, expectedProps, actualProps)
 }
 
-func TestPropertiesEncoderDeep(t *testing.T) {
+func TestPropertiesEncoderSimpleWithComments_Wrapped(t *testing.T) {
+	var sampleYaml = `a: 'bob cool' # line`
+
+	var expectedProps = `# line
+a = "bob cool"`
+	var actualProps = yamlToProps(sampleYaml, false)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderDeep_Unwrapped(t *testing.T) {
 	var sampleYaml = `a: 
   b: "bob cool"
 `
 
 	var expectedProps = `a.b = bob cool`
-	var actualProps = yamlToProps(sampleYaml)
+	var actualProps = yamlToProps(sampleYaml, true)
 	test.AssertResult(t, expectedProps, actualProps)
 }
 
-func TestPropertiesEncoderDeepWithComments(t *testing.T) {
+func TestPropertiesEncoderDeep_Wrapped(t *testing.T) {
+	var sampleYaml = `a: 
+  b: "bob cool"
+`
+
+	var expectedProps = `a.b = "bob cool"`
+	var actualProps = yamlToProps(sampleYaml, false)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderDeepWithComments_Unwrapped(t *testing.T) {
 	var sampleYaml = `a:  # a thing
   b: "bob cool" # b thing
 `
 
 	var expectedProps = `# b thing
 a.b = bob cool`
-	var actualProps = yamlToProps(sampleYaml)
+	var actualProps = yamlToProps(sampleYaml, true)
 	test.AssertResult(t, expectedProps, actualProps)
 }
 
-func TestPropertiesEncoderArray(t *testing.T) {
+func TestPropertiesEncoderDeepWithComments_Wrapped(t *testing.T) {
+	var sampleYaml = `a:  # a thing
+  b: "bob cool" # b thing
+`
+
+	var expectedProps = `# b thing
+a.b = "bob cool"`
+	var actualProps = yamlToProps(sampleYaml, false)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderArray_Unwrapped(t *testing.T) {
 	var sampleYaml = `a: 
   b: [{c: dog}, {c: cat}]
 `
 
 	var expectedProps = `a.b.0.c = dog
 a.b.1.c = cat`
-	var actualProps = yamlToProps(sampleYaml)
+	var actualProps = yamlToProps(sampleYaml, true)
+	test.AssertResult(t, expectedProps, actualProps)
+}
+
+func TestPropertiesEncoderArray_Wrapped(t *testing.T) {
+	var sampleYaml = `a: 
+  b: [{c: dog named jim}, {c: cat named jam}]
+`
+
+	var expectedProps = `a.b.0.c = "dog named jim"
+a.b.1.c = "cat named jam"`
+	var actualProps = yamlToProps(sampleYaml, false)
 	test.AssertResult(t, expectedProps, actualProps)
 }

pkg/yqlib/encoder_test.go

@@ -13,7 +13,7 @@ func yamlToJSON(sampleYaml string, indent int) string {
 	var output bytes.Buffer
 	writer := bufio.NewWriter(&output)
 
-	var jsonEncoder = NewJONEncoder(indent)
+	var jsonEncoder = NewJSONEncoder(indent, false)
 	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder())
 	if err != nil {
 		panic(err)

pkg/yqlib/encoder_xml.go

@@ -9,7 +9,7 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-var XMLPreferences = xmlPreferences{AttributePrefix: "+", ContentName: "+content"}
+var XMLPreferences = xmlPreferences{AttributePrefix: "+", ContentName: "+content", StrictMode: false, UseRawToken: false}
 
 type xmlEncoder struct {
 	attributePrefix string

pkg/yqlib/expression_parser.go

@@ -21,7 +21,7 @@ type expressionParserImpl struct {
 }
 
 func newExpressionParser() ExpressionParserInterface {
-	return &expressionParserImpl{newExpressionTokeniser(), newExpressionPostFixer()}
+	return &expressionParserImpl{newParticipleLexer(), newExpressionPostFixer()}
 }
 
 func (p *expressionParserImpl) ParseExpression(expression string) (*ExpressionNode, error) {
@@ -70,7 +70,7 @@ func (p *expressionParserImpl) createExpressionTree(postFixPath []*Operation) (*
 		stack = append(stack, &newNode)
 	}
 	if len(stack) != 1 {
-		return nil, fmt.Errorf("Bad expression, please check expression syntax")
+		return nil, fmt.Errorf("bad expression, please check expression syntax")
 	}
 	return stack[0], nil
 }

pkg/yqlib/expression_parser_test.go

@@ -11,33 +11,38 @@ func getExpressionParser() ExpressionParserInterface {
 	return ExpressionParser
 }
 
+func TestParserNoMatchingCloseBracket(t *testing.T) {
+	_, err := getExpressionParser().ParseExpression(".cat | with(.;.bob")
+	test.AssertResultComplex(t, "bad expression - probably missing close bracket on WITH", err.Error())
+}
+
 func TestParserNoMatchingCloseCollect(t *testing.T) {
 	_, err := getExpressionParser().ParseExpression("[1,2")
-	test.AssertResultComplex(t, "Bad expression, could not find matching `]`", err.Error())
+	test.AssertResultComplex(t, "bad expression, could not find matching `]`", err.Error())
 }
 func TestParserNoMatchingCloseObjectInCollect(t *testing.T) {
 	_, err := getExpressionParser().ParseExpression(`[{"b": "c"]`)
-	test.AssertResultComplex(t, "Bad expression, could not find matching `}`", err.Error())
+	test.AssertResultComplex(t, "bad expression, could not find matching `}`", err.Error())
 }
 
 func TestParserNoMatchingCloseInCollect(t *testing.T) {
 	_, err := getExpressionParser().ParseExpression(`[(.a]`)
-	test.AssertResultComplex(t, "Bad expression, could not find matching `)`", err.Error())
+	test.AssertResultComplex(t, "bad expression, could not find matching `)`", err.Error())
 }
 
 func TestParserNoMatchingCloseCollectObject(t *testing.T) {
 	_, err := getExpressionParser().ParseExpression(`{"a": "b"`)
-	test.AssertResultComplex(t, "Bad expression, could not find matching `}`", err.Error())
+	test.AssertResultComplex(t, "bad expression, could not find matching `}`", err.Error())
 }
 
 func TestParserNoMatchingCloseCollectInCollectObject(t *testing.T) {
 	_, err := getExpressionParser().ParseExpression(`{"b": [1}`)
-	test.AssertResultComplex(t, "Bad expression, could not find matching `]`", err.Error())
+	test.AssertResultComplex(t, "bad expression, could not find matching `]`", err.Error())
 }
 
 func TestParserNoMatchingCloseBracketInCollectObject(t *testing.T) {
 	_, err := getExpressionParser().ParseExpression(`{"b": (1}`)
-	test.AssertResultComplex(t, "Bad expression, could not find matching `)`", err.Error())
+	test.AssertResultComplex(t, "bad expression, could not find matching `)`", err.Error())
 }
 
 func TestParserNoArgsForTwoArgOp(t *testing.T) {
@@ -72,5 +77,5 @@ func TestParserOneArgForOneArgOp(t *testing.T) {
 
 func TestParserExtraArgs(t *testing.T) {
 	_, err := getExpressionParser().ParseExpression("sortKeys(.) explode(.)")
-	test.AssertResultComplex(t, "Bad expression, please check expression syntax", err.Error())
+	test.AssertResultComplex(t, "bad expression, please check expression syntax", err.Error())
 }

pkg/yqlib/expression_postfix.go

@@ -27,11 +27,11 @@ func popOpToResult(opStack []*token, result []*Operation) ([]*token, []*Operatio
 
 func validateNoOpenTokens(token *token) error {
 	if token.TokenType == openCollect {
-		return fmt.Errorf(("Bad expression, could not find matching `]`"))
+		return fmt.Errorf(("bad expression, could not find matching `]`"))
 	} else if token.TokenType == openCollectObject {
-		return fmt.Errorf(("Bad expression, could not find matching `}`"))
+		return fmt.Errorf(("bad expression, could not find matching `}`"))
 	} else if token.TokenType == openBracket {
-		return fmt.Errorf(("Bad expression, could not find matching `)`"))
+		return fmt.Errorf(("bad expression, could not find matching `)`"))
 	}
 	return nil
 }
@@ -50,7 +50,7 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
 			log.Debugf("put %v onto the opstack", currentToken.toString(true))
 		case closeCollect, closeCollectObject:
 			var opener tokenType = openCollect
-			var collectOperator *operationType = collectOpType
+			var collectOperator = collectOpType
 			if currentToken.TokenType == closeCollectObject {
 				opener = openCollectObject
 				collectOperator = collectObjectOpType
@@ -76,7 +76,7 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
 			// on the close op - move it to the traverse array op
 			// allows for .["cat"]?
 			prefs := traversePreferences{}
-			closeTokenMatch := string(currentToken.Match.Bytes)
+			closeTokenMatch := currentToken.Match
 			if closeTokenMatch[len(closeTokenMatch)-1:] == "?" {
 				prefs.OptionalTraverse = true
 			}
@@ -104,7 +104,7 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
 				opStack, result = popOpToResult(opStack, result)
 			}
 			if len(opStack) == 0 {
-				return nil, errors.New("Bad path expression, got close brackets without matching opening bracket")
+				return nil, errors.New("bad expression, got close brackets without matching opening bracket")
 			}
 			// now we should have ( as the last element on the opStack, get rid of it
 			opStack = opStack[0 : len(opStack)-1]
@@ -124,6 +124,14 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
 	}
 
 	log.Debugf("opstackLen: %v", len(opStack))
+	if len(opStack) > 0 {
+		log.Debugf("opstack:")
+		for _, token := range opStack {
+			log.Debugf("- %v", token.toString(true))
+		}
+
+		return nil, fmt.Errorf("bad expression - probably missing close bracket on %v", opStack[len(opStack)-1].toString(false))
+	}
 
 	if log.IsEnabledFor(logging.DEBUG) {
 		log.Debugf("PostFix Result:")

pkg/yqlib/expression_processing_test.go

@@ -312,7 +312,7 @@ var pathTests = []struct {
 	},
 }
 
-var tokeniser = newExpressionTokeniser()
+var tokeniser = newParticipleLexer()
 var postFixer = newExpressionPostFixer()
 
 func TestPathParsing(t *testing.T) {

pkg/yqlib/expression_tokeniser.go

@@ -1,674 +0,0 @@
-package yqlib
-
-import (
-	"fmt"
-	"regexp"
-	"strconv"
-	"strings"
-
-	lex "github.com/timtadh/lexmachine"
-	"github.com/timtadh/lexmachine/machines"
-)
-
-func skip(*lex.Scanner, *machines.Match) (interface{}, error) {
-	return nil, nil
-}
-
-type tokenType uint32
-
-const (
-	operationToken = 1 << iota
-	openBracket
-	closeBracket
-	openCollect
-	closeCollect
-	openCollectObject
-	closeCollectObject
-	traverseArrayCollect
-)
-
-type token struct {
-	TokenType            tokenType
-	Operation            *Operation
-	AssignOperation      *Operation      // e.g. tag (GetTag) op becomes AssignTag if '=' follows it
-	CheckForPostTraverse bool            // e.g. [1]cat should really be [1].cat
-	Match                *machines.Match // match that created this token
-
-}
-
-func (t *token) toString(detail bool) string {
-	if t.TokenType == operationToken {
-		if detail {
-			return fmt.Sprintf("%v (%v)", t.Operation.toString(), t.Operation.OperationType.Precedence)
-		}
-		return t.Operation.toString()
-	} else if t.TokenType == openBracket {
-		return "("
-	} else if t.TokenType == closeBracket {
-		return ")"
-	} else if t.TokenType == openCollect {
-		return "["
-	} else if t.TokenType == closeCollect {
-		return "]"
-	} else if t.TokenType == openCollectObject {
-		return "{"
-	} else if t.TokenType == closeCollectObject {
-		return "}"
-	} else if t.TokenType == traverseArrayCollect {
-		return ".["
-
-	} else {
-		return "NFI"
-	}
-}
-
-func pathToken(wrapped bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-		prefs := traversePreferences{}
-
-		if value[len(value)-1:] == "?" {
-			prefs.OptionalTraverse = true
-			value = value[:len(value)-1]
-		}
-
-		value = value[1:]
-		if wrapped {
-			value = unwrap(value)
-		}
-		log.Debug("PathToken %v", value)
-		op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
-	}
-}
-
-func opToken(op *operationType) lex.Action {
-	return opTokenWithPrefs(op, nil, nil)
-}
-
-func opAssignableToken(opType *operationType, assignOpType *operationType) lex.Action {
-	return opTokenWithPrefs(opType, assignOpType, nil)
-}
-
-func assignOpToken(updateAssign bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		log.Debug("assignOpToken %v", string(m.Bytes))
-		value := string(m.Bytes)
-		prefs := assignPreferences{DontOverWriteAnchor: true}
-		op := &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, UpdateAssign: updateAssign, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func multiplyWithPrefs(op *operationType) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		prefs := multiplyPreferences{}
-		options := string(m.Bytes)
-		if strings.Contains(options, "+") {
-			prefs.AppendArrays = true
-		}
-		if strings.Contains(options, "?") {
-			prefs.TraversePrefs = traversePreferences{DontAutoCreate: true}
-		}
-		if strings.Contains(options, "n") {
-			prefs.AssignPrefs = assignPreferences{OnlyWriteNull: true}
-		}
-		if strings.Contains(options, "d") {
-			prefs.DeepMergeArrays = true
-		}
-		prefs.TraversePrefs.DontFollowAlias = true
-		op := &Operation{OperationType: op, Value: multiplyOpType.Type, StringValue: options, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func opTokenWithPrefs(op *operationType, assignOpType *operationType, preferences interface{}) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		log.Debug("opTokenWithPrefs %v", string(m.Bytes))
-		value := string(m.Bytes)
-		op := &Operation{OperationType: op, Value: op.Type, StringValue: value, Preferences: preferences}
-		var assign *Operation
-		if assignOpType != nil {
-			assign = &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, Preferences: preferences}
-		}
-		return &token{TokenType: operationToken, Operation: op, AssignOperation: assign}, nil
-	}
-}
-
-func hasOptionParameter(value string, option string) bool {
-	parameterParser := regexp.MustCompile(`.*\([^\)]*\)`)
-	matches := parameterParser.FindStringSubmatch(value)
-	if len(matches) == 0 {
-		return false
-	}
-	parameterString := matches[0]
-	optionParser := regexp.MustCompile(fmt.Sprintf("\\b%v\\b", option))
-	return len(optionParser.FindStringSubmatch(parameterString)) > 0
-}
-
-func extractNumberParameter(value string) (int, error) {
-	parameterParser := regexp.MustCompile(`.*\(([0-9]+)\)`)
-	matches := parameterParser.FindStringSubmatch(value)
-	var indent, errParsingInt = strconv.ParseInt(matches[1], 10, 32)
-	if errParsingInt != nil {
-		return 0, errParsingInt
-	}
-	return int(indent), nil
-}
-
-func envSubstWithOptions() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-		noEmpty := hasOptionParameter(value, "ne")
-		noUnset := hasOptionParameter(value, "nu")
-		failFast := hasOptionParameter(value, "ff")
-		envsubstOpType.Type = "ENVSUBST"
-		prefs := envOpPreferences{NoUnset: noUnset, NoEmpty: noEmpty, FailFast: failFast}
-		if noEmpty {
-			envsubstOpType.Type = envsubstOpType.Type + "_NO_EMPTY"
-		}
-		if noUnset {
-			envsubstOpType.Type = envsubstOpType.Type + "_NO_UNSET"
-		}
-
-		op := &Operation{OperationType: envsubstOpType, Value: envsubstOpType.Type, StringValue: value, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func flattenWithDepth() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-		var depth, errParsingInt = extractNumberParameter(value)
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-
-		prefs := flattenPreferences{depth: depth}
-		op := &Operation{OperationType: flattenOpType, Value: flattenOpType.Type, StringValue: value, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func encodeWithIndent(outputFormat PrinterOutputFormat) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-		var indent, errParsingInt = extractNumberParameter(value)
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-
-		prefs := encoderPreferences{format: outputFormat, indent: indent}
-		op := &Operation{OperationType: encodeOpType, Value: encodeOpType.Type, StringValue: value, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func assignAllCommentsOp(updateAssign bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		log.Debug("assignAllCommentsOp %v", string(m.Bytes))
-		value := string(m.Bytes)
-		op := &Operation{
-			OperationType: assignCommentOpType,
-			Value:         assignCommentOpType.Type,
-			StringValue:   value,
-			UpdateAssign:  updateAssign,
-			Preferences:   commentOpPreferences{LineComment: true, HeadComment: true, FootComment: true},
-		}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func literalToken(pType tokenType, checkForPost bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &token{TokenType: pType, CheckForPostTraverse: checkForPost, Match: m}, nil
-	}
-}
-
-func unwrap(value string) string {
-	return value[1 : len(value)-1]
-}
-
-func numberValue() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		var numberString = string(m.Bytes)
-		var number, errParsingInt = strconv.ParseInt(numberString, 10, 64)
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-
-		return &token{TokenType: operationToken, Operation: createValueOperation(number, numberString)}, nil
-	}
-}
-
-func hexValue() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		var originalString = string(m.Bytes)
-		var numberString = originalString[2:]
-		log.Debugf("numberString: %v", numberString)
-		var number, errParsingInt = strconv.ParseInt(numberString, 16, 64)
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-
-		return &token{TokenType: operationToken, Operation: createValueOperation(number, originalString)}, nil
-	}
-}
-
-func floatValue() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		var numberString = string(m.Bytes)
-		var number, errParsingInt = strconv.ParseFloat(numberString, 64)
-		if errParsingInt != nil {
-			return nil, errParsingInt
-		}
-		return &token{TokenType: operationToken, Operation: createValueOperation(number, numberString)}, nil
-	}
-}
-
-func booleanValue(val bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &token{TokenType: operationToken, Operation: createValueOperation(val, string(m.Bytes))}, nil
-	}
-}
-
-func stringValue(wrapped bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-		if wrapped {
-			value = unwrap(value)
-		}
-		value = strings.ReplaceAll(value, "\\\"", "\"")
-		return &token{TokenType: operationToken, Operation: createValueOperation(value, value)}, nil
-	}
-}
-
-func getVariableOpToken() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-
-		value = value[1:]
-
-		getVarOperation := createValueOperation(value, value)
-		getVarOperation.OperationType = getVariableOpType
-
-		return &token{TokenType: operationToken, Operation: getVarOperation, CheckForPostTraverse: true}, nil
-	}
-}
-
-func envOp(strenv bool) lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		value := string(m.Bytes)
-		preferences := envOpPreferences{}
-
-		if strenv {
-			// strenv( )
-			value = value[7 : len(value)-1]
-			preferences.StringValue = true
-		} else {
-			//env( )
-			value = value[4 : len(value)-1]
-		}
-
-		envOperation := createValueOperation(value, value)
-		envOperation.OperationType = envOpType
-		envOperation.Preferences = preferences
-
-		return &token{TokenType: operationToken, Operation: envOperation}, nil
-	}
-}
-
-func nullValue() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &token{TokenType: operationToken, Operation: createValueOperation(nil, string(m.Bytes))}, nil
-	}
-}
-
-func selfToken() lex.Action {
-	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		op := &Operation{OperationType: selfReferenceOpType}
-		return &token{TokenType: operationToken, Operation: op}, nil
-	}
-}
-
-func initLexer() (*lex.Lexer, error) {
-	lexer := lex.NewLexer()
-	lexer.Add([]byte(`\(`), literalToken(openBracket, false))
-	lexer.Add([]byte(`\)`), literalToken(closeBracket, true))
-
-	lexer.Add([]byte(`\.\[`), literalToken(traverseArrayCollect, false))
-	lexer.Add([]byte(`\.\.`), opTokenWithPrefs(recursiveDescentOpType, nil, recursiveDescentPreferences{RecurseArray: true,
-		TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: false}}))
-
-	lexer.Add([]byte(`\.\.\.`), opTokenWithPrefs(recursiveDescentOpType, nil, recursiveDescentPreferences{RecurseArray: true,
-		TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: true}}))
-
-	lexer.Add([]byte(`,`), opToken(unionOpType))
-	lexer.Add([]byte(`:\s*`), opToken(createMapOpType))
-	lexer.Add([]byte(`length`), opToken(lengthOpType))
-	lexer.Add([]byte(`line`), opToken(lineOpType))
-	lexer.Add([]byte(`column`), opToken(columnOpType))
-
-	lexer.Add([]byte(`eval`), opToken(evalOpType))
-
-	lexer.Add([]byte(`map`), opToken(mapOpType))
-	lexer.Add([]byte(`map_values`), opToken(mapValuesOpType))
-	lexer.Add([]byte(`pick`), opToken(pickOpType))
-
-	lexer.Add([]byte(`flatten\([0-9]+\)`), flattenWithDepth())
-	lexer.Add([]byte(`flatten`), opTokenWithPrefs(flattenOpType, nil, flattenPreferences{depth: -1}))
-
-	lexer.Add([]byte(`format_datetime`), opToken(formatDateTimeOpType))
-	lexer.Add([]byte(`now`), opToken(nowOpType))
-	lexer.Add([]byte(`tz`), opToken(tzOpType))
-	lexer.Add([]byte(`with_dtf`), opToken(withDtFormatOpType))
-
-	lexer.Add([]byte(`toyaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat))
-	lexer.Add([]byte(`to_yaml\([0-9]+\)`), encodeWithIndent(YamlOutputFormat))
-
-	lexer.Add([]byte(`toxml\([0-9]+\)`), encodeWithIndent(XMLOutputFormat))
-	lexer.Add([]byte(`to_xml\([0-9]+\)`), encodeWithIndent(XMLOutputFormat))
-
-	lexer.Add([]byte(`tojson\([0-9]+\)`), encodeWithIndent(JSONOutputFormat))
-	lexer.Add([]byte(`to_json\([0-9]+\)`), encodeWithIndent(JSONOutputFormat))
-
-	lexer.Add([]byte(`toyaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
-	lexer.Add([]byte(`to_yaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
-	// 0 indent doesn't work with yaml.
-	lexer.Add([]byte(`@yaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
-
-	lexer.Add([]byte(`tojson`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JSONOutputFormat, indent: 2}))
-	lexer.Add([]byte(`to_json`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JSONOutputFormat, indent: 2}))
-	lexer.Add([]byte(`@json`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JSONOutputFormat, indent: 0}))
-
-	lexer.Add([]byte(`toprops`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2}))
-	lexer.Add([]byte(`to_props`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2}))
-	lexer.Add([]byte(`@props`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2}))
-
-	lexer.Add([]byte(`tocsv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: CSVOutputFormat}))
-	lexer.Add([]byte(`to_csv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: CSVOutputFormat}))
-	lexer.Add([]byte(`@csv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: CSVOutputFormat}))
-
-	lexer.Add([]byte(`totsv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: TSVOutputFormat}))
-	lexer.Add([]byte(`to_tsv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: TSVOutputFormat}))
-	lexer.Add([]byte(`@tsv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: TSVOutputFormat}))
-
-	lexer.Add([]byte(`toxml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: XMLOutputFormat}))
-	lexer.Add([]byte(`to_xml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: XMLOutputFormat, indent: 2}))
-	lexer.Add([]byte(`@xml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: XMLOutputFormat, indent: 0}))
-
-	lexer.Add([]byte(`@base64`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: Base64OutputFormat}))
-	lexer.Add([]byte(`@base64d`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: Base64InputFormat}))
-
-	lexer.Add([]byte(`fromyaml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
-	lexer.Add([]byte(`fromjson`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
-	lexer.Add([]byte(`fromxml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: XMLInputFormat}))
-
-	lexer.Add([]byte(`from_yaml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
-	lexer.Add([]byte(`from_json`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: YamlInputFormat}))
-	lexer.Add([]byte(`from_xml`), opTokenWithPrefs(decodeOpType, nil, decoderPreferences{format: XMLInputFormat}))
-
-	lexer.Add([]byte(`sortKeys`), opToken(sortKeysOpType))
-	lexer.Add([]byte(`sort_keys`), opToken(sortKeysOpType))
-
-	lexer.Add([]byte(`load`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewYamlDecoder()}))
-
-	lexer.Add([]byte(`xmlload`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName)}))
-	lexer.Add([]byte(`load_xml`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName)}))
-	lexer.Add([]byte(`loadxml`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewXMLDecoder(XMLPreferences.AttributePrefix, XMLPreferences.ContentName)}))
-
-	lexer.Add([]byte(`load_base64`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewBase64Decoder()}))
-
-	lexer.Add([]byte(`load_props`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewPropertiesDecoder()}))
-	lexer.Add([]byte(`loadprops`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false, decoder: NewPropertiesDecoder()}))
-
-	lexer.Add([]byte(`strload`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: true}))
-	lexer.Add([]byte(`load_str`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: true}))
-	lexer.Add([]byte(`loadstr`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: true}))
-
-	lexer.Add([]byte(`select`), opToken(selectOpType))
-	lexer.Add([]byte(`has`), opToken(hasOpType))
-	lexer.Add([]byte(`unique`), opToken(uniqueOpType))
-	lexer.Add([]byte(`unique_by`), opToken(uniqueByOpType))
-	lexer.Add([]byte(`group_by`), opToken(groupByOpType))
-	lexer.Add([]byte(`explode`), opToken(explodeOpType))
-	lexer.Add([]byte(`or`), opToken(orOpType))
-	lexer.Add([]byte(`and`), opToken(andOpType))
-	lexer.Add([]byte(`not`), opToken(notOpType))
-	lexer.Add([]byte(`ireduce`), opToken(reduceOpType))
-	lexer.Add([]byte(`;`), opToken(blockOpType))
-	lexer.Add([]byte(`\/\/`), opToken(alternativeOpType))
-
-	lexer.Add([]byte(`documentIndex`), opToken(getDocumentIndexOpType))
-	lexer.Add([]byte(`document_index`), opToken(getDocumentIndexOpType))
-
-	lexer.Add([]byte(`di`), opToken(getDocumentIndexOpType))
-
-	lexer.Add([]byte(`splitDoc`), opToken(splitDocumentOpType))
-	lexer.Add([]byte(`split_doc`), opToken(splitDocumentOpType))
-
-	lexer.Add([]byte(`join`), opToken(joinStringOpType))
-	lexer.Add([]byte(`sub`), opToken(subStringOpType))
-	lexer.Add([]byte(`match`), opToken(matchOpType))
-	lexer.Add([]byte(`capture`), opToken(captureOpType))
-	lexer.Add([]byte(`test`), opToken(testOpType))
-
-	lexer.Add([]byte(`upcase`), opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: true}))
-	lexer.Add([]byte(`ascii_upcase`), opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: true}))
-
-	lexer.Add([]byte(`downcase`), opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: false}))
-	lexer.Add([]byte(`ascii_downcase`), opTokenWithPrefs(changeCaseOpType, nil, changeCasePrefs{ToUpperCase: false}))
-
-	lexer.Add([]byte(`sort`), opToken(sortOpType))
-	lexer.Add([]byte(`sort_by`), opToken(sortByOpType))
-	lexer.Add([]byte(`reverse`), opToken(reverseOpType))
-
-	lexer.Add([]byte(`any`), opToken(anyOpType))
-	lexer.Add([]byte(`any_c`), opToken(anyConditionOpType))
-	lexer.Add([]byte(`all`), opToken(allOpType))
-	lexer.Add([]byte(`all_c`), opToken(allConditionOpType))
-	lexer.Add([]byte(`contains`), opToken(containsOpType))
-
-	lexer.Add([]byte(`split`), opToken(splitStringOpType))
-
-	lexer.Add([]byte(`parent`), opToken(getParentOpType))
-	lexer.Add([]byte(`key`), opToken(getKeyOpType))
-	lexer.Add([]byte(`keys`), opToken(keysOpType))
-
-	lexer.Add([]byte(`style`), opAssignableToken(getStyleOpType, assignStyleOpType))
-
-	lexer.Add([]byte(`tag`), opAssignableToken(getTagOpType, assignTagOpType))
-	lexer.Add([]byte(`anchor`), opAssignableToken(getAnchorOpType, assignAnchorOpType))
-	lexer.Add([]byte(`alias`), opAssignableToken(getAliasOptype, assignAliasOpType))
-	lexer.Add([]byte(`filename`), opToken(getFilenameOpType))
-
-	lexer.Add([]byte(`fileIndex`), opToken(getFileIndexOpType))
-	lexer.Add([]byte(`file_index`), opToken(getFileIndexOpType))
-
-	lexer.Add([]byte(`fi`), opToken(getFileIndexOpType))
-	lexer.Add([]byte(`path`), opToken(getPathOpType))
-	lexer.Add([]byte(`to_entries`), opToken(toEntriesOpType))
-	lexer.Add([]byte(`from_entries`), opToken(fromEntriesOpType))
-	lexer.Add([]byte(`with_entries`), opToken(withEntriesOpType))
-
-	lexer.Add([]byte(`with`), opToken(withOpType))
-
-	lexer.Add([]byte(`lineComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}))
-	lexer.Add([]byte(`line_comment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}))
-
-	lexer.Add([]byte(`headComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{HeadComment: true}))
-	lexer.Add([]byte(`head_comment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{HeadComment: true}))
-
-	lexer.Add([]byte(`footComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{FootComment: true}))
-	lexer.Add([]byte(`foot_comment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{FootComment: true}))
-
-	lexer.Add([]byte(`comments\s*=`), assignAllCommentsOp(false))
-	lexer.Add([]byte(`comments\s*\|=`), assignAllCommentsOp(true))
-
-	lexer.Add([]byte(`collect`), opToken(collectOpType))
-
-	lexer.Add([]byte(`\s*==\s*`), opToken(equalsOpType))
-	lexer.Add([]byte(`\s*!=\s*`), opToken(notEqualsOpType))
-	lexer.Add([]byte(`\s*=\s*`), assignOpToken(false))
-
-	lexer.Add([]byte(`del`), opToken(deleteChildOpType))
-
-	lexer.Add([]byte(`\s*\|=\s*`), assignOpToken(true))
-
-	lexer.Add([]byte("( |\t|\n|\r)+"), skip)
-
-	lexer.Add([]byte(`\."[^ "]+"\??`), pathToken(true))
-	lexer.Add([]byte(`\.[^ ;\}\{\:\[\],\|\.\[\(\)=\n]+\??`), pathToken(false))
-	lexer.Add([]byte(`\.`), selfToken())
-
-	lexer.Add([]byte(`\|`), opToken(pipeOpType))
-
-	lexer.Add([]byte(`0[xX][0-9A-Fa-f]+`), hexValue())
-	lexer.Add([]byte(`-?\d+(\.\d+)`), floatValue())
-	lexer.Add([]byte(`-?[1-9](\.\d+)?[Ee][-+]?\d+`), floatValue())
-	lexer.Add([]byte(`-?\d+`), numberValue())
-
-	lexer.Add([]byte(`[Tt][Rr][Uu][Ee]`), booleanValue(true))
-	lexer.Add([]byte(`[Ff][Aa][Ll][Ss][Ee]`), booleanValue(false))
-
-	lexer.Add([]byte(`[Nn][Uu][Ll][Ll]`), nullValue())
-	lexer.Add([]byte(`~`), nullValue())
-
-	lexer.Add([]byte(`"([^"\\]*(\\.[^"\\]*)*)"`), stringValue(true))
-	lexer.Add([]byte(`strenv\([^\)]+\)`), envOp(true))
-	lexer.Add([]byte(`env\([^\)]+\)`), envOp(false))
-
-	lexer.Add([]byte(`envsubst\((ne|nu|ff| |,)+\)`), envSubstWithOptions())
-	lexer.Add([]byte(`envsubst`), opToken(envsubstOpType))
-
-	lexer.Add([]byte(`\[`), literalToken(openCollect, false))
-	lexer.Add([]byte(`\]\??`), literalToken(closeCollect, true))
-	lexer.Add([]byte(`\{`), literalToken(openCollectObject, false))
-	lexer.Add([]byte(`\}`), literalToken(closeCollectObject, true))
-	lexer.Add([]byte(`\*=[\+|\?dn]*`), multiplyWithPrefs(multiplyAssignOpType))
-	lexer.Add([]byte(`\*[\+|\?dn]*`), multiplyWithPrefs(multiplyOpType))
-
-	lexer.Add([]byte(`\+`), opToken(addOpType))
-	lexer.Add([]byte(`\+=`), opToken(addAssignOpType))
-
-	lexer.Add([]byte(`\-`), opToken(subtractOpType))
-	lexer.Add([]byte(`\-=`), opToken(subtractAssignOpType))
-	lexer.Add([]byte(`\$[a-zA-Z_-0-9]+`), getVariableOpToken())
-	lexer.Add([]byte(`as`), opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{}))
-	lexer.Add([]byte(`ref`), opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{IsReference: true}))
-
-	err := lexer.CompileNFA()
-	if err != nil {
-		return nil, err
-	}
-	return lexer, nil
-}
-
-type expressionTokeniser interface {
-	Tokenise(expression string) ([]*token, error)
-}
-
-type expressionTokeniserImpl struct {
-	lexer *lex.Lexer
-}
-
-func newExpressionTokeniser() expressionTokeniser {
-	var lexer, err = initLexer()
-	if err != nil {
-		panic(err)
-	}
-	return &expressionTokeniserImpl{lexer}
-}
-
-func (p *expressionTokeniserImpl) Tokenise(expression string) ([]*token, error) {
-	scanner, err := p.lexer.Scanner([]byte(expression))
-
-	if err != nil {
-		return nil, fmt.Errorf("parsing expression: %w", err)
-	}
-	var tokens []*token
-	for tok, err, eof := scanner.Next(); !eof; tok, err, eof = scanner.Next() {
-
-		if tok != nil {
-			currentToken := tok.(*token)
-			log.Debugf("Tokenising %v", currentToken.toString(true))
-			tokens = append(tokens, currentToken)
-		}
-		if err != nil {
-			return nil, fmt.Errorf("parsing expression: %w", err)
-		}
-	}
-	var postProcessedTokens = make([]*token, 0)
-
-	skipNextToken := false
-
-	for index := range tokens {
-		if skipNextToken {
-			skipNextToken = false
-		} else {
-			postProcessedTokens, skipNextToken = p.handleToken(tokens, index, postProcessedTokens)
-		}
-	}
-
-	return postProcessedTokens, nil
-}
-
-func (p *expressionTokeniserImpl) handleToken(tokens []*token, index int, postProcessedTokens []*token) (tokensAccum []*token, skipNextToken bool) {
-	skipNextToken = false
-	currentToken := tokens[index]
-
-	log.Debug("processing %v", currentToken.toString(true))
-
-	if currentToken.TokenType == traverseArrayCollect {
-		// `.[exp]`` works by creating a traversal array of [self, exp] and piping that into the traverse array operator
-		//need to put a traverse array then a collect currentToken
-		// do this by adding traverse then converting currentToken to collect
-
-		log.Debug("  adding self")
-		op := &Operation{OperationType: selfReferenceOpType, StringValue: "SELF"}
-		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
-
-		log.Debug("  adding traverse array")
-		op = &Operation{OperationType: traverseArrayOpType, StringValue: "TRAVERSE_ARRAY"}
-		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
-
-		currentToken = &token{TokenType: openCollect}
-
-	}
-
-	if index != len(tokens)-1 && currentToken.AssignOperation != nil &&
-		tokens[index+1].TokenType == operationToken &&
-		tokens[index+1].Operation.OperationType == assignOpType {
-		log.Debug("  its an update assign")
-		currentToken.Operation = currentToken.AssignOperation
-		currentToken.Operation.UpdateAssign = tokens[index+1].Operation.UpdateAssign
-		skipNextToken = true
-	}
-
-	log.Debug("  adding token to the fixed list")
-	postProcessedTokens = append(postProcessedTokens, currentToken)
-
-	if index != len(tokens)-1 &&
-		((currentToken.TokenType == openCollect && tokens[index+1].TokenType == closeCollect) ||
-			(currentToken.TokenType == openCollectObject && tokens[index+1].TokenType == closeCollectObject)) {
-		log.Debug("  adding empty")
-		op := &Operation{OperationType: emptyOpType, StringValue: "EMPTY"}
-		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
-	}
-
-	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
-		((tokens[index+1].TokenType == operationToken && (tokens[index+1].Operation.OperationType == traversePathOpType)) ||
-			(tokens[index+1].TokenType == traverseArrayCollect)) {
-		log.Debug("  adding pipe because the next thing is traverse")
-		op := &Operation{OperationType: shortPipeOpType, Value: "PIPE", StringValue: "."}
-		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
-	}
-	if index != len(tokens)-1 && currentToken.CheckForPostTraverse &&
-		tokens[index+1].TokenType == openCollect {
-
-		log.Debug("  adding traverArray because next is opencollect")
-		op := &Operation{OperationType: traverseArrayOpType}
-		postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op})
-	}
-	return postProcessedTokens, skipNextToken
-}

pkg/yqlib/file_utils.go

@@ -39,7 +39,7 @@ func copyFileContents(src, dst string) (err error) {
 		return err
 	}
 	defer safelyCloseFile(in)
-	out, err := os.Create(dst)
+	out, err := os.Create(dst) // #nosec
 	if err != nil {
 		return err
 	}