Fixing from review

Added CSV roundtrip
Adding CSV decoder
2026-08-24 16:39:58 +08:00 · 2022-08-01 10:16:59 +10:00 · 2022-08-01 10:05:26 +10:00 · 2022-07-31 17:35:36 +10:00 · 2022-07-31 17:31:27 +10:00
374 changed files with 5644 additions and 20989 deletions
@@ -38,11 +38,11 @@ jobs:

    steps:
    - name: Checkout repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3

    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
+      uses: github/codeql-action/init@v2
      with:
        languages: ${{ matrix.language }}
        # If you wish to specify custom queries, you can do so here or in a config file.
@@ -53,7 +53,7 @@ jobs:
    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
    # If this step fails, then you should remove it and run the build manually (see below)
    - name: Autobuild
-      uses: github/codeql-action/autobuild@v3
+      uses: github/codeql-action/autobuild@v2

    # ℹ️ Command-line programs to run using the OS shell.
    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -67,4 +67,4 @@ jobs:
    #   make release

    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      uses: github/codeql-action/analyze@v2
@@ -14,16 +14,16 @@ jobs:
      IMAGE_NAME: mikefarah/yq
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3

      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v2
        with:
          platforms: all

      - name: Set up Docker Buildx
        id: buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v2
        with:
          version: latest

@@ -45,7 +45,7 @@ jobs:
          docker buildx build \
            --label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
            --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
-            --label "org.opencontainers.image.description=yq is a portable command-line data file processor" \
+            --label "org.opencontainers.image.description=yq is a portable command-line YAML processor" \
            --label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
            --label "org.opencontainers.image.licenses=MIT" \
            --label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \
@@ -60,23 +60,3 @@ jobs:
            -t "${IMAGE_NAME}:4" \
            -t "${IMAGE_NAME}:latest" \
            .
-          
-          cd github-action
-          docker buildx build \
-            --label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
-            --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
-            --label "org.opencontainers.image.description=yq is a portable command-line data file processor" \
-            --label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
-            --label "org.opencontainers.image.licenses=MIT" \
-            --label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \
-            --label "org.opencontainers.image.source=https://github.com/mikefarah/yq" \
-            --label "org.opencontainers.image.title=yq" \
-            --label "org.opencontainers.image.url=https://mikefarah.gitbook.io/yq/" \
-            --label "org.opencontainers.image.version=${IMAGE_VERSION}" \
-            --platform "${PLATFORMS}" \
-            --pull \
-            --push \
-            -t "${IMAGE_NAME}:${IMAGE_VERSION}-githubaction" \
-            -t "${IMAGE_NAME}:4-githubaction" \
-            -t "${IMAGE_NAME}:latest-githubaction" \
-            .
@@ -10,14 +10,14 @@ jobs:
    runs-on: ubuntu-latest
    steps:

-    - name: Set up Go
-      uses: actions/setup-go@v5
+    - name: Set up Go 1.18
+      uses: actions/setup-go@v3
      with:
-        go-version: '^1.20'
+        go-version: 1.18
      id: go

    - name: Check out code into the Go module directory
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3

    - name: Get dependencies
      run: |
@@ -2,26 +2,26 @@ name: Release YQ
 on:
  push:
    tags:
-      - 'v4.*'
+      - 'v*'
      - 'draft-*'

 jobs:
  publishGitRelease:
    runs-on: ubuntu-latest
    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-go@v5
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
        with:
-          go-version: '^1.20'
-          check-latest: true
+          go-version: '^1.18'
      - name: Compile man page markup
        id: gen-man-page-md
        run: |
          ./scripts/generate-man-page-md.sh
+          echo "::set-output name=man-page-md::man.md"

      - name: Get the version
        id: get_version
-        run: echo "VERSION=${GITHUB_REF##*/}" >> "${GITHUB_OUTPUT}"
+        run: echo ::set-output name=VERSION::${GITHUB_REF##*/}

      - name: Generate man page
        uses: docker://pandoc/core:2.14.2
@@ -35,7 +35,7 @@ jobs:
            --variable=header:"yq (https://github.com/mikefarah/yq/) version ${{ steps.get_version.outputs.VERSION }}"
            --variable=author:"Mike Farah"
            --output=yq.1
-            man.md
+            ${{ steps.gen-man-page-md.outputs.man-page-md }}

      - name: Cross compile
        run: |
@@ -1,23 +0,0 @@
-name: Release Snap
-
-on:
-  release:
-    types: [released]
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-jobs:
-    buildSnap:
-      environment: snap
-      runs-on: ubuntu-latest
-      steps:
-        - uses: actions/checkout@v4
-        - uses: snapcore/action-build@v1
-          id: build
-        - uses: snapcore/action-publish@v1
-          env:
-            SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.STORE_LOGIN }}
-          with:
-            snap: ${{ steps.build.outputs.snap }}
-            release: stable
@@ -1,8 +1,5 @@
 name: Test Yq Action
-on: 
-  push:
-  workflow_dispatch:
-
+on: [push]
 permissions:
  contents: read

@@ -13,16 +10,13 @@ jobs:
    runs-on: ubuntu-latest

    steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v3
      - name: Get test
-        id: get_value
        uses: mikefarah/yq@master
        with:
-          cmd: yq '.test' examples/multiline-text.yaml
-      - name: Multiline test
-        run: echo "### It was [${{ steps.get_value.outputs.result }}]" >> $GITHUB_STEP_SUMMARY
+          cmd: yq eval '.a' examples/sample.yaml
      - name: Write inplace test
        id: lookupSdkVersion
        uses: mikefarah/yq@master
        with:
-          cmd: yq -i '.b.c = 5' examples/sample.yaml
+          cmd: yq eval -i '.b.c = 5' examples/sample.yaml
@@ -41,8 +41,6 @@ yq*.snap

 test.yml
 test*.yml
-test*.xml
-test*.toml
 test*.yaml
 0.yml
 1.yml
@@ -58,9 +56,4 @@ _build
 debian/files

 # intellij
-/.idea
-
-# vscode
-.vscode
-
-yq3
+/.idea
@@ -20,15 +20,12 @@ linters:
    - unparam
 linters-settings:
  depguard:
-    rules:
-      prevent_unmaintained_packages:
-        list-mode: lax
-        files:
-          - $all
-          - "!$test"
-        deny:
-          - pkg: io/ioutil
-            desc: "replaced by io and os packages since Go 1.16: https://tip.golang.org/doc/go1.16#ioutil"
+    list-type: blacklist
+    include-go-root: true
+    packages:
+      - io/ioutil
+    packages-with-error-message:
+      - io/ioutil: "The 'io/ioutil' package is deprecated. Use corresponding 'os' or 'io' functions instead."
 issues:
  exclude-rules:
  - linters:
@@ -1,6 +1,6 @@
 # Development

-1. Install (Golang)[https://golang.org/]
+1. Install (golang)[https://golang.org/]
 1. Run  `scripts/devtools.sh` to install the required devtools
 2. Run `make [local] vendor` to install the vendor dependencies
 2. Run `make [local] test` to ensure you can run the existing tests
@@ -12,7 +12,7 @@

 # Documentation

-The documentation is a bit of a mixed bag (sorry in advance, I do plan on simplifying it...) - with some parts automatically generated and stiched together and some statically defined.
+The documentation is a bit of a mixed bag (sorry in advanced, I do plan on simplifying it...) - with some parts automatically generated and stiched together and some statically defined.

 Documentation is written in markdown, and is published in the 'gitbook' branch.

@@ -1,10 +1,10 @@
-FROM golang:1.22.1 as builder
+FROM golang:1.18.4 as builder

 WORKDIR /go/src/mikefarah/yq

 COPY . .

-RUN CGO_ENABLED=0 go build -ldflags "-s -w" .
+RUN CGO_ENABLED=0 go build .
 # RUN ./scripts/test.sh -- this too often times out in the github pipeline.
 RUN ./scripts/acceptance.sh

@@ -1,4 +1,4 @@
-FROM golang:1.22.1
+FROM golang:1.18.4

 COPY scripts/devtools.sh /opt/devtools.sh

@@ -6,7 +6,5 @@ RUN set -e -x \
    && /opt/devtools.sh
 ENV PATH=/go/bin:$PATH

-RUN apt-get update && apt-get install -y npm && npm install -g npx cspell@latest
-
 ENV CGO_ENABLED 0
 ENV GOPATH /go:/yq
@@ -84,12 +84,8 @@ format: vendor
 	${ENGINERUN} bash ./scripts/format.sh


-.PHONY: spelling
-spelling: format
-	${ENGINERUN} bash ./scripts/spelling.sh
-
 .PHONY: secure
-secure: spelling
+secure: format
 	${ENGINERUN} bash ./scripts/secure.sh

 .PHONY: check
@@ -7,7 +7,6 @@ export GIT_DESCRIBE = $(shell git describe --tags --always)
 LDFLAGS :=
 LDFLAGS += -X main.GitCommit=${GIT_COMMIT}${GIT_DIRTY}
 LDFLAGS += -X main.GitDescribe=${GIT_DESCRIBE}
-LDFLAGS += -w -s

 GITHUB_TOKEN ?=

@@ -3,10 +3,31 @@
 ![Build](https://github.com/mikefarah/yq/workflows/Build/badge.svg)  ![Docker Pulls](https://img.shields.io/docker/pulls/mikefarah/yq.svg) ![Github Releases (by Release)](https://img.shields.io/github/downloads/mikefarah/yq/total.svg) ![Go Report](https://goreportcard.com/badge/github.com/mikefarah/yq) ![CodeQL](https://github.com/mikefarah/yq/workflows/CodeQL/badge.svg)


-a lightweight and portable command-line YAML, JSON and XML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json, xml, properties, csv and tsv. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.
+a lightweight and portable command-line YAML, JSON and XML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json and xml. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.

 yq is written in go - so you can download a dependency free binary for your platform and you are good to go! If you prefer there are a variety of package managers that can be used as well as Docker and Podman, all listed below.

+## Notice for v4.x versions prior to 4.18.1
+Since 4.18.1, yq's 'eval/e' command is the _default_ command and no longer needs to be specified.
+
+Older versions will still need to specify 'eval/e'.
+
+Similarly, '-' is no longer required as a filename to read from STDIN (unless reading from one or more files).
+
+TLDR:
+
+Prior to 4.18.1 
+```bash
+yq e '.cool' - < file.yaml
+```
+
+4.18+ 
+```bash
+yq '.cool' < file.yaml
+```
+
+When merging multiple files together, `eval-all/ea` is still required to tell `yq` to run the expression against all the document at once.
+
 ## Quick Usage Guide

 Read a value:
@@ -19,7 +40,7 @@ Pipe from STDIN:
 yq '.a.b[0].c' < file.yaml
 ```

-Update a yaml file, in place
+Update a yaml file, inplace
 ```bash
 yq -i '.a.b[0].c = "cool"' file.yaml
 ```
@@ -31,10 +52,6 @@ NAME=mike yq -i '.a.b[0].c = strenv(NAME)' file.yaml

 Merge multiple files
 ```bash
-# merge two files
-yq -n 'load("file1.yaml") * load("file2.yaml")'
-
-# merge using globs:
 # note the use of `ea` to evaluate all the files at once
 # instead of in sequence
 yq ea '. as $item ireduce ({}; . * $item )' path/to/*.yml
@@ -49,29 +66,19 @@ yq -i '
 ' file.yaml
 ```

-Find and update an item in an array:
-```bash
-yq '(.[] | select(.name == "foo") | .address) = "12 cat st"'
-```
-
 Convert JSON to YAML
 ```bash
-yq -Poy sample.json
+yq -P sample.json
 ```

-See [recipes](https://mikefarah.gitbook.io/yq/recipes) for more examples and the [documentation](https://mikefarah.gitbook.io/yq/) for more information.
-
-Take a look at the discussions for [common questions](https://github.com/mikefarah/yq/discussions/categories/q-a), and [cool ideas](https://github.com/mikefarah/yq/discussions/categories/show-and-tell)
+See the [documentation](https://mikefarah.gitbook.io/yq/) for more examples.

 ## Install

 ### [Download the latest binary](https://github.com/mikefarah/yq/releases/latest)

 ### wget
-Use wget to download, gzipped pre-compiled binaries:
-
-
-For instance, VERSION=v4.2.0 and BINARY=yq_linux_amd64
+Use wget to download the pre-compiled binaries:

 #### Compressed via tar.gz
 ```bash
@@ -86,12 +93,7 @@ wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -O /
    chmod +x /usr/bin/yq
 ```

-#### Latest version
-
-```bash
-wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq &&\
-    chmod +x /usr/bin/yq
-```
+For instance, VERSION=v4.2.0 and BINARY=yq_linux_amd64

 ### MacOS / Linux via Homebrew:
 Using [Homebrew](https://brew.sh/)
@@ -205,27 +207,12 @@ RUN apk add --no-cache tzdata
 USER yq
 ```

-#### Podman with SELinux
-
-If you are using podman with SELinux, you will need to set the shared volume flag `:z` on the volume mount:
-
-```
-v "${PWD}":/workdir:z
-```
-
 ### GitHub Action
 ```
  - name: Set foobar to cool
    uses: mikefarah/yq@master
    with:
      cmd: yq -i '.foo.bar = "cool"' 'config.yml'
-  - name: Get an entry with a variable that might contain dots or spaces
-    id: get_username
-    uses: mikefarah/yq@master
-    with:
-      cmd: yq '.all.children.["${{ matrix.ip_address }}"].username' ops/inventories/production.yml
-  - name: Reuse a variable obtained in another step
-    run: echo ${{ steps.get_username.outputs.result }}
 ```

 See https://mikefarah.gitbook.io/yq/usage/github-action for more.
@@ -238,19 +225,7 @@ go install github.com/mikefarah/yq/v4@latest
 ## Community Supported Installation methods
 As these are supported by the community :heart: - however, they may be out of date with the officially supported releases.

-_Please note that the Debian package (previously supported by @rmescandon) is no longer maintained. Please use an alternative installation method._
-
-
-### Nix
-
-```
-nix profile install nixpkgs#yq-go
-```
-
-See [here](https://search.nixos.org/packages?channel=unstable&show=yq-go&from=0&size=50&sort=relevance&type=packages&query=yq-go)
-
-
-### Webi
+# Webi

 ```
 webi yq
@@ -266,9 +241,6 @@ pacman -S go-yq
 ```

 ### Windows:
-
-Using [Chocolatey](https://chocolatey.org)
-
 [![Chocolatey](https://img.shields.io/chocolatey/v/yq.svg)](https://chocolatey.org/packages/yq)
 [![Chocolatey](https://img.shields.io/chocolatey/dt/yq.svg)](https://chocolatey.org/packages/yq)
 ```
@@ -276,16 +248,6 @@ choco install yq
 ```
 Supported by @chillum (https://chocolatey.org/packages/yq)

-Using [scoop](https://scoop.sh/)
-```
-scoop install main/yq
-```
-
-Using [winget](https://learn.microsoft.com/en-us/windows/package-manager/)
-```
-winget install --id MikeFarah.yq
-```
-
 ### Mac:
 Using [MacPorts](https://www.macports.org/)
 ```
@@ -303,6 +265,15 @@ Supported by Tuan Hoang
 https://pkgs.alpinelinux.org/package/edge/community/x86/yq


+### On Ubuntu 16.04 or higher from Debian package:
+```sh
+sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64
+sudo add-apt-repository ppa:rmescandon/yq
+sudo apt update
+sudo apt install yq -y
+```
+Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
+
 ## Features
 - [Detailed documentation with many examples](https://mikefarah.gitbook.io/yq/)
 - Written in portable go, so you can download a lovely dependency free binary
@@ -314,15 +285,15 @@ https://pkgs.alpinelinux.org/package/edge/community/x86/yq
 - [Deeply data structures](https://mikefarah.gitbook.io/yq/operators/traverse-read)
 - [Sort keys](https://mikefarah.gitbook.io/yq/operators/sort-keys)
 - Manipulate yaml [comments](https://mikefarah.gitbook.io/yq/operators/comment-operators), [styling](https://mikefarah.gitbook.io/yq/operators/style), [tags](https://mikefarah.gitbook.io/yq/operators/tag) and [anchors and aliases](https://mikefarah.gitbook.io/yq/operators/anchor-and-alias-operators).
- [Update in place](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags)
+- [Update inplace](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags)
 - [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/operators/select#select-and-update-matching-values-in-map)
 - Keeps yaml formatting and comments when updating (though there are issues with whitespace)
 - [Decode/Encode base64 data](https://mikefarah.gitbook.io/yq/operators/encode-decode)
 - [Load content from other files](https://mikefarah.gitbook.io/yq/operators/load)
- [Convert to/from json/ndjson](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
+- [Convert to/from json](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
 - [Convert to/from xml](https://mikefarah.gitbook.io/yq/v/v4.x/usage/xml)
 - [Convert to/from properties](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties)
- [Convert to/from csv/tsv](https://mikefarah.gitbook.io/yq/usage/csv-tsv)
+- [Convert to csv/tsv](https://mikefarah.gitbook.io/yq/usage/csv-tsv)
 - [General shell completion scripts (bash/zsh/fish/powershell)](https://mikefarah.gitbook.io/yq/v/v4.x/commands/shell-completion)
 - [Reduce](https://mikefarah.gitbook.io/yq/operators/reduce) to merge multiple files or sum an array or other fancy things.
 - [Github Action](https://mikefarah.gitbook.io/yq/usage/github-action) to use in your automated pipeline (thanks @devorbitus)
@@ -341,7 +312,7 @@ Examples:
 # yq defaults to 'eval' command if no command is specified. See "yq eval --help" for more examples.
 yq '.stuff' < myfile.yml # outputs the data at the "stuff" node from "myfile.yml"

-yq -i '.stuff = "foo"' myfile.yml # update myfile.yml in place
+yq -i '.stuff = "foo"' myfile.yml # update myfile.yml inplace


 Available Commands:
@@ -358,7 +329,7 @@ Flags:
      --header-preprocess             Slurp any header comments and separators before processing expression. (default true)
  -h, --help                          help for yq
  -I, --indent int                    sets indent level for output (default 2)
-  -i, --inplace                       update the file in place of first file given.
+  -i, --inplace                       update the file inplace of first file given.
  -p, --input-format string           [yaml|y|xml|x] parse format for input. Note that json is a subset of yaml. (default "yaml")
  -M, --no-colors                     force print with no colors
  -N, --no-doc                        Don't print document separators (---)
@@ -377,6 +348,5 @@ Use "yq [command] --help" for more information about a command.
 ## Known Issues / Missing Features
 - `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details)
 - Powershell has its own...[opinions on quoting yq](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks#quotes-in-windows-powershell)
- "yes", "no" were dropped as boolean values in the yaml 1.2 standard - which is the standard yq assumes.

 See [tips and tricks](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks) for more common problems and solutions.
@@ -3,25 +3,25 @@
 testWriteInPlacePipeIn() {
  result=$(./yq e -i -n '.a' 2>&1)
  assertEquals 1 $?
-  assertEquals "Error: write in place flag only applicable when giving an expression and at least one file" "$result"
+  assertEquals "Error: write inplace flag only applicable when giving an expression and at least one file" "$result"
 }

 testWriteInPlacePipeInEvalall() {
  result=$(./yq ea -i -n '.a' 2>&1)
  assertEquals 1 $?
-  assertEquals "Error: write in place flag only applicable when giving an expression and at least one file" "$result"
+  assertEquals "Error: write inplace flag only applicable when giving an expression and at least one file" "$result"
 }

 testWriteInPlaceWithSplit() {
  result=$(./yq e -s "cat" -i '.a = "thing"' test.yml 2>&1)
  assertEquals 1 $?
-  assertEquals "Error: write in place cannot be used with split file" "$result"
+  assertEquals "Error: write inplace cannot be used with split file" "$result"
 }

 testWriteInPlaceWithSplitEvalAll() {
  result=$(./yq ea -s "cat" -i '.a = "thing"' test.yml 2>&1)
  assertEquals 1 $?
-  assertEquals "Error: write in place cannot be used with split file" "$result"
+  assertEquals "Error: write inplace cannot be used with split file" "$result"
 }

 testNullWithFiles() {
@@ -96,17 +96,6 @@ testBasicExpressionFromFile() {
  assertEquals '{"xyz":"meow","cool":"frog"}' "$X"
 }

-testBasicExpressionFromFileDos() {
-  ./yq -n ".xyz = 123" > test.yml
-  echo '.xyz = "meow" | .cool = "frog"' | sed 's/$'"/`echo \\\r`/" > instructions.txt
-  
-  X=$(./yq --from-file instructions.txt test.yml -o=j -I=0)
-  assertEquals '{"xyz":"meow","cool":"frog"}' "$X"
-
-  X=$(./yq ea --from-file instructions.txt test.yml -o=j -I=0)
-  assertEquals '{"xyz":"meow","cool":"frog"}' "$X"
-}
-
 testBasicGitHubAction() {
  ./yq -n ".a = 123" > test.yml
  X=$(cat /dev/null | ./yq test.yml)
@@ -154,7 +143,7 @@ testBasicCatWithFilesNoDash() {
 }

 # when the nullinput flag is used
-# don't automatically read STDIN (this breaks github actions)
+# dont automatically read STDIN (this breaks github actions)
 testBasicCreateFileGithubAction() {
  cat /dev/null | ./yq -n ".a = 123" > test.yml
 }
@@ -313,7 +302,7 @@ testBasicExitStatusNoEval() {
  assertEquals 1 "$?"
 }

-testBasicExtractFieldWithSeparator() {
+testBasicExtractFieldWithSeperator() {
    cat >test.yml <<EOL
 ---
 name: chart-name
@@ -323,7 +312,7 @@ EOL
  assertEquals "chart-name" "$X"
 }

-testBasicExtractMultipleFieldWithSeparator() {
+testBasicExtractMultipleFieldWithSeperator() {
    cat >test.yml <<EOL
 ---
 name: chart-name
@@ -362,12 +351,6 @@ EOM
  assertEquals "$expected" "$X"
 }

-testBasicClosedStdIn() {
-  cat >test.yml <<EOL
-a: 1
-EOL
-  X=$(./yq e '.a' test.yml <&-)
-  assertEquals "1" "$X"
-}
+

 source ./scripts/shunit2
@@ -9,7 +9,7 @@ EOL

 testEmptyEval() {
  X=$(./yq e test.yml)
-  expected="# comment"
+  expected=$(cat test.yml)
  assertEquals 0 $?
  assertEquals "$expected" "$X"
 }
@@ -1,21 +0,0 @@
-#!/bin/bash
-
-setUp() {
-  rm test*.yml || true
-  cat >test.yml <<EOL
-# comment
-EOL
-}
-
-testStringInterpolation() {
-    X=$(./yq -n '"Mike \(3 + 4)"')
-    assertEquals "Mike 7" "$X"
-}
-
-testNoStringInterpolation() {
-    X=$(./yq --string-interpolation=f -n '"Mike \(3 + 4)"')
-    assertEquals "Mike \(3 + 4)" "$X"
-}
-
-
-source ./scripts/shunit2
@@ -1,43 +0,0 @@
-#!/bin/bash
-
-setUp() {
-  rm test*.yml || true
-  
-}
-
-testLineCountFirstLineComment() {
-  cat >test.yml <<EOL
-#test123 
-abc: 123 
-test123: 123123 
-#comment 
-lalilu: lalilu
-EOL
-
-  X=$(./yq '.lalilu | line' --header-preprocess=false < test.yml)
-  assertEquals "5" "$X"
-}
-
-testArrayOfDocs() {
-  cat >test.yml <<EOL
---
-# leading comment doc 1
-a: 1
---
-# leading comment doc 2
-a: 2
-EOL
-
-read -r -d '' expected << EOM
- # leading comment doc 1
-  a: 1
- # leading comment doc 2
-  a: 2
-EOM
-
-  X=$(./yq ea '[.]' --header-preprocess=false < test.yml)
-  assertEquals "$expected" "$X"
-
-}
-
-source ./scripts/shunit2
@@ -1,249 +0,0 @@
-#!/bin/bash
-
-setUp() {
-  rm test*.yml 2>/dev/null || true
-  rm test*.toml 2>/dev/null || true
-  rm test*.tfstate 2>/dev/null || true
-  rm test*.json 2>/dev/null || true
-  rm test*.properties 2>/dev/null || true
-  rm test*.csv 2>/dev/null || true
-  rm test*.tsv 2>/dev/null || true
-  rm test*.xml 2>/dev/null || true
-}
-
-testInputJson() {
-  cat >test.json <<EOL
-{ "mike" : { "things": "cool" } }
-EOL
-
-  read -r -d '' expected << EOM
-{
-  "mike": {
-    "things": "cool"
-  }
-}
-EOM
-
-  X=$(./yq test.json)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea test.json)
-  assertEquals "$expected" "$X"
-}
-
-testInputToml() {
-  cat >test.toml <<EOL
-[owner]
-name = "Tom Preston-Werner"
-dob = 1979-05-27T07:32:00-08:00
-EOL
-
-  read -r -d '' expected << EOM
-owner:
-  name: Tom Preston-Werner
-  dob: 1979-05-27T07:32:00-08:00
-EOM
-
-  X=$(./yq -oy test.toml)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea -oy test.toml)
-  assertEquals "$expected" "$X"
-}
-
-testInputTfstate() {
-  cat >test.tfstate <<EOL
-{ "mike" : { "things": "cool" } }
-EOL
-
-  read -r -d '' expected << EOM
-{"mike": {"things": "cool"}}
-EOM
-
-  X=$(./yq test.tfstate)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea test.tfstate)
-  assertEquals "$expected" "$X"
-}
-
-testInputJsonOutputYaml() {
-  cat >test.json <<EOL
-{ "mike" : { "things": "cool" } }
-EOL
-
-  read -r -d '' expected << EOM
-mike:
-  things: cool
-EOM
-
-  X=$(./yq test.json -oy)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea test.json -oy)
-  assertEquals "$expected" "$X"
-}
-
-testInputProperties() {
-  cat >test.properties <<EOL
-mike.things = hello
-EOL
-
-  read -r -d '' expected << EOM
-mike.things = hello
-EOM
-
-  X=$(./yq e test.properties)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq test.properties)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea test.properties)
-  assertEquals "$expected" "$X"
-}
-
-testInputPropertiesGitHubAction() {
-  cat >test.properties <<EOL
-mike.things = hello
-EOL
-
-  read -r -d '' expected << EOM
-mike.things = hello
-EOM
-
-  X=$(cat /dev/null | ./yq e test.properties)
-  assertEquals "$expected" "$X"
-
-  X=$(cat /dev/null | ./yq ea test.properties)
-  assertEquals "$expected" "$X"
-}
-
-testInputCSV() {
-  cat >test.csv <<EOL
-fruit,yumLevel
-apple,5
-banana,4
-EOL
-
-  read -r -d '' expected << EOM
-fruit,yumLevel
-apple,5
-banana,4
-EOM
-
-  X=$(./yq e test.csv)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea test.csv)
-  assertEquals "$expected" "$X"
-}
-
-testInputCSVUTF8() {
-  read -r -d '' expected << EOM
-id,first,last
-1,john,smith
-1,jane,smith
-EOM
-
-  X=$(./yq utf8.csv)
-  assertEquals "$expected" "$X"
-}
-
-testInputTSV() {
-  cat >test.tsv <<EOL
-fruit	yumLevel
-apple	5
-banana	4
-EOL
-
-  read -r -d '' expected << EOM
-fruit	yumLevel
-apple	5
-banana	4
-EOM
-
-  X=$(./yq e test.tsv)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea test.tsv)
-  assertEquals "$expected" "$X"
-}
-
-
-
-
-testInputXml() {
-  cat >test.xml <<EOL
-<cat legs="4">BiBi</cat>
-EOL
-
-  read -r -d '' expected << EOM
-<cat legs="4">BiBi</cat>
-EOM
-
-  X=$(./yq e test.xml)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea test.xml)
-  assertEquals "$expected" "$X"
-}
-
-testInputXmlNamespaces() {
-  cat >test.xml <<EOL
-<?xml version="1.0"?>
-<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
-</map>
-EOL
-
-  read -r -d '' expected << EOM
-<?xml version="1.0"?>
-<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url"></map>
-EOM
-
-  X=$(./yq e test.xml)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea test.xml)
-  assertEquals "$expected" "$X"
-}
-
-
-
-testInputXmlStrict() {
-  cat >test.xml <<EOL
-<?xml version="1.0"?>
-<!DOCTYPE root [
-<!ENTITY writer "Catherine.">
-<!ENTITY copyright "(r) Great">
-]>
-<root>
-    <item>&writer;&copyright;</item>
-</root>
-EOL
-
-  X=$(./yq --xml-strict-mode test.xml  2>&1)
-  assertEquals 1 $?
-  assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X"
-
-  X=$(./yq ea --xml-strict-mode test.xml  2>&1)
-  assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X"
-}
-
-testInputXmlGithubAction() {
-  cat >test.xml <<EOL
-<cat legs="4">BiBi</cat>
-EOL
-
-  read -r -d '' expected << EOM
-<cat legs="4">BiBi</cat>
-EOM
-
-  X=$(cat /dev/null | ./yq e test.xml)
-  assertEquals "$expected" "$X"
-
-  X=$(cat /dev/null | ./yq ea test.xml)
-  assertEquals "$expected" "$X"
-}
-
-source ./scripts/shunit2
@@ -63,75 +63,6 @@ EOM
  assertEquals "$expected" "$X"
 }

-testInputCSVCustomSeparator() {
-  cat >test.csv <<EOL
-fruit;yumLevel
-apple;5
-banana;4
-EOL
-
-  read -r -d '' expected << EOM
- fruit: apple
-  yumLevel: 5
- fruit: banana
-  yumLevel: 4
-EOM
-
-  X=$(./yq -p=csv --csv-separator ";" test.csv)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea -p=csv --csv-separator ";" test.csv)
-  assertEquals "$expected" "$X"
-}
-
-testInputCSVNoAuto() {
-  cat >test.csv <<EOL
-thing1
-name: cat
-EOL
-
-  read -r -d '' expected << EOM
- thing1: 'name: cat'
-EOM
-
-  X=$(./yq --csv-auto-parse=f test.csv -oy)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea --csv-auto-parse=f test.csv -oy)
-  assertEquals "$expected" "$X"
-}
-
-testInputTSVNoAuto() {
-  cat >test.tsv <<EOL
-thing1
-name: cat
-EOL
-
-  read -r -d '' expected << EOM
- thing1: 'name: cat'
-EOM
-
-  X=$(./yq --tsv-auto-parse=f test.tsv -oy)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea --tsv-auto-parse=f test.tsv -oy)
-  assertEquals "$expected" "$X"
-}
-
-testInputCSVUTF8() {
-  read -r -d '' expected << EOM
- id: 1
-  first: john
-  last: smith
- id: 1
-  first: jane
-  last: smith
-EOM
-
-  X=$(./yq -p=csv utf8.csv)
-  assertEquals "$expected" "$X"
-}
-
 testInputTSV() {
  cat >test.tsv <<EOL
 fruit	yumLevel
@@ -164,7 +95,7 @@ EOL
  read -r -d '' expected << EOM
 cat:
  +content: BiBi
-  +@legs: "4"
+  +legs: "4"
 EOM

  X=$(./yq e -p=xml test.yml)
@@ -175,44 +106,23 @@ EOM
 }

 testInputXmlNamespaces() {
-  cat >test.xml <<EOL
+  cat >test.yml <<EOL
 <?xml version="1.0"?>
 <map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
 </map>
 EOL

  read -r -d '' expected << EOM
-+p_xml: version="1.0"
 map:
-  +@xmlns: some-namespace
-  +@xmlns:xsi: some-instance
-  +@xsi:schemaLocation: some-url
+  +xmlns: some-namespace
+  +xmlns:xsi: some-instance
+  +xsi:schemaLocation: some-url
 EOM

-  X=$(./yq e -p=xml test.xml)
+  X=$(./yq e -p=xml test.yml)
  assertEquals "$expected" "$X"

-  X=$(./yq ea -p=xml test.xml)
-  assertEquals "$expected" "$X"
-}
-
-testInputXmlRoundtrip() {
-  cat >test.yml <<EOL
-<?xml version="1.0"?>
-<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
-<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">Meow</map>
-EOL
-
-  read -r -d '' expected << EOM
-<?xml version="1.0"?>
-<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
-<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">Meow</map>
-EOM
-
-  X=$(./yq -p=xml -o=xml test.yml)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea -p=xml -o=xml test.yml)
+  X=$(./yq ea -p=xml test.yml)
  assertEquals "$expected" "$X"
 }

@@ -229,11 +139,11 @@ testInputXmlStrict() {
 </root>
 EOL

-  X=$(./yq -p=xml --xml-strict-mode test.yml -o=xml 2>&1)
+  X=$(./yq -p=xml --xml-strict-mode test.yml 2>&1)
  assertEquals 1 $?
  assertEquals "Error: bad file 'test.yml': XML syntax error on line 7: invalid character entity &writer;" "$X"

-  X=$(./yq ea -p=xml --xml-strict-mode test.yml -o=xml 2>&1)
+  X=$(./yq ea -p=xml --xml-strict-mode test.yml 2>&1)
  assertEquals "Error: bad file 'test.yml': XML syntax error on line 7: invalid character entity &writer;" "$X"
 }

@@ -245,7 +155,7 @@ EOL
  read -r -d '' expected << EOM
 cat:
  +content: BiBi
-  +@legs: "4"
+  +legs: "4"
 EOM

  X=$(cat /dev/null | ./yq e -p=xml test.yml)
@@ -1,8 +1,5 @@
 #!/bin/bash

-
-# examples where header-preprocess is required
-
 setUp() {
  rm test*.yml || true
  cat >test.yml <<EOL
@@ -11,7 +8,7 @@ a: test
 EOL
 }

-testLeadingSeparatorWithDoc() {
+testLeadingSeperatorWithDoc() {
  cat >test.yml <<EOL
 # hi peeps
 # cool
@@ -34,94 +31,18 @@ EOM
  assertEquals "$expected" "$X"
 }

-
-testLeadingSeparatorWithNewlinesNewDoc() {
-  cat >test.yml <<EOL
-# hi peeps
-# cool
-
-
---
-a: test
---
-b: cool
-EOL
-
-  read -r -d '' expected << EOM
-# hi peeps
-# cool
-
-
---
-a: thing
---
-b: cool
-EOM
-
-  X=$(./yq e '(select(di == 0) | .a) = "thing"' - < test.yml)
-  assertEquals "$expected" "$X"
-}
-
-testLeadingSeparatorWithNewlinesMoreComments() {
-  cat >test.yml <<EOL
-# hi peeps
-# cool
-
---
-# great
-
-a: test
---
-b: cool
-EOL
-
-  read -r -d '' expected << EOM
-# hi peeps
-# cool
-
---
-# great
-
-a: thing
---
-b: cool
-EOM
-
-  X=$(./yq e '(select(di == 0) | .a) = "thing"' - < test.yml)
-  assertEquals "$expected" "$X"
-}
-
-
-testLeadingSeparatorWithDirective() {
-  cat >test.yml <<EOL
-%YAML 1.1
---
-this: should really work
-EOL
-
-  read -r -d '' expected << EOM
-%YAML 1.1
---
-this: should really work
-EOM
-
-  X=$(./yq < test.yml)
-  assertEquals "$expected" "$X"
-}
-
-
-testLeadingSeparatorPipeIntoEvalSeq() {
+testLeadingSeperatorPipeIntoEvalSeq() {
  X=$(./yq e - < test.yml)
  expected=$(cat test.yml)
  assertEquals "$expected" "$X"
 }

-testLeadingSeparatorExtractField() {
+testLeadingSeperatorExtractField() {
  X=$(./yq e '.a' - < test.yml)
  assertEquals "test" "$X"
 }

-testLeadingSeparatorExtractFieldWithCommentsAfterSep() {
+testLeadingSeperatorExtractFieldWithCommentsAfterSep() {
  cat >test.yml <<EOL
 ---
 # hi peeps
@@ -132,7 +53,7 @@ EOL
  assertEquals "test" "$X"
 }

-testLeadingSeparatorExtractFieldWithCommentsBeforeSep() {
+testLeadingSeperatorExtractFieldWithCommentsBeforeSep() {
  cat >test.yml <<EOL
 # hi peeps
 # cool
@@ -144,7 +65,7 @@ EOL
 }


-testLeadingSeparatorExtractFieldMultiDoc() {
+testLeadingSeperatorExtractFieldMultiDoc() {
  cat >test.yml <<EOL
 ---
 a: test
@@ -161,13 +82,13 @@ EOM
  assertEquals "$expected" "$X"
 }

-testLeadingSeparatorExtractFieldMultiDocWithComments() {
+testLeadingSeperatorExtractFieldMultiDocWithComments() {
  cat >test.yml <<EOL
 # here
 ---
 # there
 a: test
-# wherever
+# whereever
 ---
 # you are
 a: test2
@@ -184,26 +105,26 @@ EOM
 }


-testLeadingSeparatorEvalSeq() {
+testLeadingSeperatorEvalSeq() {
  X=$(./yq e test.yml)
  expected=$(cat test.yml)
  assertEquals "$expected" "$X"
 }

-testLeadingSeparatorPipeIntoEvalAll() {
+testLeadingSeperatorPipeIntoEvalAll() {
  X=$(./yq ea - < test.yml)
  expected=$(cat test.yml)
  assertEquals "$expected" "$X"
 }


-testLeadingSeparatorEvalAll() {
+testLeadingSeperatorEvalAll() {
  X=$(./yq ea test.yml)
  expected=$(cat test.yml)
  assertEquals "$expected" "$X"
 }

-testLeadingSeparatorMultiDocEvalSimple() {
+testLeadingSeperatorMultiDocEvalSimple() {
  read -r -d '' expected << EOM
 ---
 a: test
@@ -217,7 +138,7 @@ EOM
  assertEquals "$expected" "$X"
 }

-testLeadingSeparatorMultiDocInOneFile() {
+testLeadingSeperatorMultiDocInOneFile() {
  cat >test.yml <<EOL
 ---
 # hi peeps
@@ -231,7 +152,7 @@ EOL
  assertEquals "$expected" "$X"
 }

-testLeadingSeparatorMultiDocInOneFileEvalAll() {
+testLeadingSeperatorMultiDocInOneFileEvalAll() {
  cat >test.yml <<EOL
 ---
 # hi peeps
@@ -245,7 +166,7 @@ EOL
  assertEquals "$expected" "$X"
 }

-testLeadingSeparatorMultiDocEvalComments() {
+testLeadingSeperatorMultiDocEvalComments() {
  cat >test.yml <<EOL
 # hi peeps
 # cool
@@ -273,7 +194,7 @@ EOM
  assertEquals "$expected" "$X"
 }

-testLeadingSeparatorMultiDocEvalCommentsTrailingSep() {
+testLeadingSeperatorMultiDocEvalCommentsTrailingSep() {
  cat >test.yml <<EOL
 # hi peeps
 # cool
@@ -305,7 +226,7 @@ EOM
  assertEquals "$expected" "$X"
 }

-testLeadingSeparatorMultiMultiDocEvalCommentsTrailingSep() {
+testLeadingSeperatorMultiMultiDocEvalCommentsTrailingSep() {
  cat >test.yml <<EOL
 # hi peeps
 # cool
@@ -345,7 +266,7 @@ EOM
  assertEquals "$expected" "$X"
 }

-testLeadingSeparatorMultiDocEvalCommentsLeadingSep() {
+testLeadingSeperatorMultiDocEvalCommentsLeadingSep() {
  cat >test.yml <<EOL
 ---
 # hi peeps
@@ -405,7 +326,7 @@ EOM
  assertEquals "$expected" "$X"
 }

-testLeadingSeparatorMultiDocEvalCommentsStripComments() {
+testLeadingSeperatorMultiDocEvalCommentsStripComments() {
  cat >test.yml <<EOL
 ---
 # hi peeps
@@ -428,7 +349,7 @@ EOM
  assertEquals "$expected" "$X"
 }

-testLeadingSeparatorMultiDocEvalCommentsLeadingSepNoDocFlag() {
+testLeadingSeperatorMultiDocEvalCommentsLeadingSepNoDocFlag() {
  cat >test.yml <<EOL
 ---
 # hi peeps
@@ -454,7 +375,7 @@ EOM
  assertEquals "$expected" "$X"
 }

-testLeadingSeparatorMultiDocEvalJsonFlag() {
+testLeadingSeperatorMultiDocEvalJsonFlag() {
  cat >test.yml <<EOL
 ---
 # hi peeps
@@ -483,7 +404,7 @@ EOM
  assertEquals "$expected" "$X"
 }

-testLeadingSeparatorMultiDocEvalAllJsonFlag() {
+testLeadingSeperatorMultiDocEvalAllJsonFlag() {
  cat >test.yml <<EOL
 ---
 # hi peeps
@@ -512,7 +433,7 @@ EOM
  assertEquals "$expected" "$X"
 }

-testLeadingSeparatorMultiDocEvalAll() {
+testLeadingSeperatorMultiDocEvalAll() {
  read -r -d '' expected << EOM
 ---
 a: test
@@ -9,7 +9,7 @@ testLoadFileNotExist() {
 testLoadFileExpNotExist() {
  result=$(./yq e -n 'load(.a)' 2>&1)
  assertEquals 1 $?
-  assertEquals "Error: filename expression returned nil" "$result"
+  assertEquals "Error: Filename expression returned nil" "$result"
 }

 testStrLoadFileNotExist() {
@@ -21,7 +21,7 @@ testStrLoadFileNotExist() {
 testStrLoadFileExpNotExist() {
  result=$(./yq e -n 'strload(.a)' 2>&1)
  assertEquals 1 $?
-  assertEquals "Error: filename expression returned nil" "$result"
+  assertEquals "Error: Filename expression returned nil" "$result"
 }

 source ./scripts/shunit2
@@ -1,286 +0,0 @@
-#!/bin/bash
-
-setUp() {
-    rm test*.yml || true
-}
-
-## Convenient bash shortcut to read records of NUL separated values
-## from stdin the safe way. See example usage in the next tests.
-read-0() {
-    local eof="" IFS=''
-    while [ "$1" ]; do
-        ## - The `-r` avoids bad surprise with '\n' and other interpreted
-        ##   sequences that can be read.
-        ## - The `-d ''` is the (strange?) way to refer to NUL delimiter.
-        ## - The `--` is how to avoid unpleasant surprises if your
-        ##   "$1" starts with "-" (minus) sign. This protection also
-        ##   will produce a readable error if you want to try to start
-        ##   your variable names with a "-".
-        read -r -d '' -- "$1" || eof=1
-        shift
-    done
-    [ -z "$eof" ] ## fail on EOF
-}
-
-## Convenient bash shortcut to be used with the next function `p-err`
-## to read NUL separated values the safe way AND catch any errors from
-## the process creating the stream of NUL separated data.  See example
-## usage in the tests.
-read-0-err() {
-    local ret="$1" eof="" idx=0 last=
-    read -r -- "${ret?}" <<<"0"
-    shift
-    while [ "$1" ]; do
-        last=$idx
-        read -r -d '' -- "$1" || {
-            ## Put this last value in ${!ret}
-            eof="$1"
-            read -r -- "$ret" <<<"${!eof}"
-            break
-        }
-        ((idx++))
-        shift
-    done
-    [ -z "$eof" ] || {
-        if [ "$last" != 0 ]; then
-            ## Uhoh, we have no idea if the errorlevel of the internal
-            ## command was properly delimited with a NUL char, and
-            ## anyway something went really wrong at least about the
-            ## number of fields separated by NUL char and the one
-            ## expected.
-            echo "Error: read-0-err couldn't fill all value $ret = '${!ret}', '$eof', '${!eof}'" >&2
-            read -r -- "$ret" <<<"not-enough-values"
-        else
-            if ! [[ "${!ret}" =~ ^[0-9]+$ && "${!ret}" -ge 0 && "${!ret}" -le 127 ]]; then
-                ## This could happen if you don't use `p-err` wrapper,
-                ## or used stdout in unexpected ways in your inner
-                ## command.
-                echo "Error: last value is not a number, did you finish with an errorlevel ?" >&2
-                read -r -- "$ret" <<<"last-value-not-a-number"
-            fi
-        fi
-        false
-    }
-}
-
-## Simply runs command given as argument and adds errorlevel in the
-## standard output. Is expected to be used in tandem with
-## `read-0-err`.
-p-err() {
-    local exp="$1"
-    "$@"
-    printf "%s" "$?"
-}
-
-wyq-r() {
-    local exp="$1"
-    ./yq e -0 -r=false "$1"
-    printf "%s" "$?"
-}
-
-testBasicUsageRaw() {
-  cat >test.yml <<EOL
-a: foo
-b: bar
-EOL
-
-  printf "foo\0bar\0" > expected.out
-
-  ## We need to compare binary content here. We have to filter the compared
-  ## content through a representation that gets rid of NUL chars but accurately
-  ## transcribe the content.
-  ## Also as it would be nice to have a pretty output in case the test fails,
-  ## we use here 'hd': a widely available shortcut to 'hexdump' that will
-  ## pretty-print any binary to it's hexadecimal representation.
-  ##
-  ## Note that the standard `assertEquals` compare its arguments
-  ## value, but they can't hold NUL characters (this comes from the
-  ## limitation of the C API of `exec*(..)` functions that requires
-  ## `const char *arv[]`). And these are NUL terminated strings.  As a
-  ## consequence, the NUL characters gets removed in bash arguments.
-  assertEquals "$(hd expected.out)" \
-               "$(./yq e -0 '.a, .b' test.yml | hd)"
-
-  rm expected.out
-}
-
-testBasicUsage() {
-  local a b
-  cat >test.yml <<EOL
-a: foo
-b: bar
-EOL
-
-  ## We provide 2 values, and ask to fill 2 variables.
-  read-0 a b < <(./yq e -0 '.a, .b' test.yml)
-  assertEquals "$?" "0"      ## Everything is fine
-  assertEquals "foo" "$a"    ## Values are correctly parsed
-  assertEquals "bar" "$b"
-
-  a=YYY ; b=XXX
-  ## Not enough values provided to fill `a` and `b`.
-  read-0 a b < <(./yq e -0 '.a' test.yml)
-  assertEquals "$?" "1"      ## An error was emitted
-  assertEquals "foo" "$a"    ## First value was correctly parsed
-  assertEquals "" "$b"       ## Second was still reset
-
-  ## Error from inner command are not catchable !. Use
-  ## `read-0-err`/`p-err` for that.
-  read-0 a < <(printf "\0"; ./yq e -0 'xxx' test.yml; )
-  assertEquals "$?" "0"
-
-}
-
-testBasicUsageJson() {
-  cat >test.yml <<EOL
-a:
-  x: foo
-b: bar
-EOL
-
-  read-0 a b < <(./yq e -0 -o=json '.a, .b' test.yml)
-
-  assertEquals '{
-  "x": "foo"
-}' "$a"
-  assertEquals '"bar"' "$b"
-
-}
-
-testFailWithValueContainingNUL() {
-  local a b c
-  ## Note that value of field 'a' actually contains a NUL char !
-  cat >test.yml <<EOL
-a: "foo\u0000bar"
-b: 1
-c: |
-  wiz
-  boom
-EOL
-
-  ## We are looking for trouble with asking to separated fields with NUL
-  ## char and requested value `.a` actually contains itself a NUL char !
-  read-0 a b c < <(./yq e -0 '.a, .b, .c' test.yml)
-  assertNotEquals "0" "$?"   ## read-0 failed to fill all values
-
-  ## But here, we can request for one value, even if `./yq` fails
-  read-0 b < <(./yq e -0 '.b, .a' test.yml)
-  assertEquals "0" "$?"   ## read-0 succeeds at feeding the first value
-  ## Note: to catch the failure of `yq`, see in the next tests the usage
-  ## of `read-0-err`.
-
-  ## using -r=false solves any NUL containing value issues, but keeps
-  ## all in YAML representation:
-  read-0 a b c < <(./yq e -0 -r=false '.a, .b, .c' test.yml)
-  assertEquals "0" "$?"    ## All goes well despite asking for `a` value
-
-  assertEquals '"foo\0bar"' "$a"   ## This is a YAML string representation
-  assertEquals '1' "$b"
-  assertEquals '|
-  wiz
-  boom' "$c"
-}
-
-testStandardLoop() {
-    local E a b res
-
-    ## Here everything is normal: 4 values, that will be paired
-    ## in key/values.
-    cat >test.yml <<EOL
- yay
- wiz
- hop
- pow
-EOL
-
-    res=""
-    while read-0-err E a b; do
-        res+="$a: $b;"
-    done < <(p-err ./yq -0 '.[]' test.yml)
-
-    assertEquals "0" "$E"                     ## errorlevel of internal command
-    assertEquals "yay: wiz;hop: pow;" "$res"  ## expected result
-}
-
-testStandardLoopWithoutEnoughValues() {
-    local E a b res
-
-    ## Here 5 values, there will be a missing value when reading
-    ## pairs of value.
-    cat >test.yml <<EOL
- yay
- wiz
- hop
- pow
- kwak
-EOL
-
-    res=""
-    ## The loop will succeed 2 times then fail
-    while read-0-err E a b; do
-        res+="$a: $b;"
-    done < <(p-err ./yq -0 '.[]' test.yml)
-
-    assertEquals "not-enough-values" "$E"     ## Not enough value error
-    assertEquals "yay: wiz;hop: pow;" "$res"  ## the 2 full key/value pairs
-
-}
-
-testStandardLoopWithInternalCmdError() {
-    local E a b res
-
-    ## Note the third value contains a NUL char !
-    cat >test.yml <<EOL
- yay
- wiz
- "foo\0bar"
- hop
- pow
-EOL
-
-    res=""
-    ## It should be only upon the second pass in the loop that
-    ## read-0-err will catch the fact that there is an error !
-    while read-0-err E a b; do
-        res+="$a: $b;"
-    done < <(p-err ./yq -0 '.[]' test.yml)
-    assertEquals "1" "$E"            ## Internal command errorlevel (from `./yq`)
-    assertEquals "yay: wiz;" "$res"  ## first 2 values were ok at least
-
-}
-
-testStandardLoopNotEnoughErrorEatsCmdError() {
-    local E a b res
-
-    ## Because of possible edge cases where the internal errorlevel
-    ## reported by `p-err` in the standard output might be mangled
-    ## with the unfinished record, `read-0-err E ...` will NOT report
-    ## the internal command error in the variable E and instead will
-    ## store the value 'not-enough-values'. In real world, anyway, you
-    ## will want to react the same if the internal command failed
-    ## and/or you didn't get as much values as expected while
-    ## reading. Keep in mind also that standard error is not
-    ## swallowed, so you can read reports from the inner command AND
-    ## from `read-0-err`.
-
-    ## Here, note that the fourth value contains a NUL char !
-    cat >test.yml <<EOL
- yay
- wiz
- hop
- "foo\0bar"
- pow
-EOL
-
-    res=""
-    ## It should be only upon the second loop that read-0-err will catch
-    ## the fact that there are not enough data to fill the requested variables
-    while read-0-err E a b; do
-        res+="$a: $b;"
-    done < <(p-err ./yq -0 '.[]' test.yml)
-    assertEquals "not-enough-values" "$E"          ## Not enough values error eats internal error !
-    assertEquals "yay: wiz;" "$res"  ## first 2 values were ok at least
-}
-
-
-source ./scripts/shunit2
@@ -48,64 +48,6 @@ EOM
  assertEquals "$expected" "$X"
 }

-testOutputYamlRawDefault() {
-  cat >test.yml <<EOL
-a: "cat"
-EOL
-
-  X=$(./yq e  '.a' test.yml)
-  assertEquals "cat" "$X"
-
-  X=$(./yq ea '.a' test.yml)
-  assertEquals "cat" "$X"
-}
-
-testOutputYamlRawOff() {
-  cat >test.yml <<EOL
-a: "cat"
-EOL
-
-  X=$(./yq e -r=false '.a' test.yml)
-  assertEquals "\"cat\"" "$X"
-
-  X=$(./yq ea -r=false '.a' test.yml)
-  assertEquals "\"cat\"" "$X"
-}
-
-testOutputYamlRawOnRoot() {
-  cat >test.yml <<EOL
-'a'
-EOL
-
-  X=$(./yq e -r '.' test.yml)
-  assertEquals "a" "$X"
-}
-
-testOutputJsonRaw() {
-  cat >test.yml <<EOL
-a: cat
-EOL
-
-  X=$(./yq e -r --output-format=json '.a' test.yml)
-  assertEquals "cat" "$X"
-
-  X=$(./yq ea -r --output-format=json '.a' test.yml)
-  assertEquals "cat" "$X"
-}
-
-testOutputJsonDefault() {
-  cat >test.yml <<EOL
-a: cat
-EOL
-
-  X=$(./yq e --output-format=json '.a' test.yml)
-  assertEquals "\"cat\"" "$X"
-
-  X=$(./yq ea --output-format=json '.a' test.yml)
-  assertEquals "\"cat\"" "$X"
-}
-
-
 testOutputJsonShort() {
  cat >test.yml <<EOL
 a: {b: ["cat"]}
@@ -130,11 +72,11 @@ EOM

 testOutputProperties() {
  cat >test.yml <<EOL
-a: {b: {c: ["cat cat"]}}
+a: {b: {c: ["cat"]}}
 EOL

  read -r -d '' expected << EOM
-a.b.c.0 = cat cat
+a.b.c.0 = cat
 EOM

  X=$(./yq e --output-format=props test.yml)
@@ -144,30 +86,13 @@ EOM
  assertEquals "$expected" "$X"
 }

-testOutputPropertiesDontUnwrap() {
-  cat >test.yml <<EOL
-a: {b: {c: ["cat cat"]}}
-EOL
-
-  read -r -d '' expected << EOM
-a.b.c.0 = "cat cat"
-EOM
-
-  X=$(./yq e -r=false --output-format=props test.yml)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea -r=false --output-format=props test.yml)
-  assertEquals "$expected" "$X"
-}
-
-
 testOutputPropertiesShort() {
  cat >test.yml <<EOL
-a: {b: {c: ["cat cat"]}}
+a: {b: {c: ["cat"]}}
 EOL

  read -r -d '' expected << EOM
-a.b.c.0 = cat cat
+a.b.c.0 = cat
 EOM

  X=$(./yq e -o=p test.yml)
@@ -198,27 +123,6 @@ EOM
  assertEquals "$expected" "$X"
 }

-testOutputCSVCustomSeparator() {
-  cat >test.yml <<EOL
- fruit: apple
-  yumLevel: 5
- fruit: banana
-  yumLevel: 4
-EOL
-
-  read -r -d '' expected << EOM
-fruit;yumLevel
-apple;5
-banana;4
-EOM
-
-  X=$(./yq -oc --csv-separator ";" test.yml)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea -o=csv --csv-separator ";" test.yml)
-  assertEquals "$expected" "$X"
-}
-
 testOutputTSV() {
  cat >test.yml <<EOL
 - fruit: apple
@@ -282,7 +186,7 @@ EOM

 testOutputXmComplex() {
  cat >test.yml <<EOL
-a: {b: {c: ["cat", "dog"], +@f: meow}}
+a: {b: {c: ["cat", "dog"], +f: meow}}
 EOL

  read -r -d '' expected << EOM
@@ -301,43 +205,4 @@ EOM
  assertEquals "$expected" "$X"
 }

-testLuaOutputPretty() {
-  cat >test.yml <<EOL
-animals:
-  cat: meow
-EOL
-
-  read -r -d '' expected << EOM
-return {
-	["animals"] = {
-		["cat"] = "meow";
-	};
-};
-EOM
-
-  X=$(./yq e --output-format=lua test.yml)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq e --output-format=lua --prettyPrint test.yml)
-  assertEquals "$expected" "$X"
-
-}
-
-testLuaOutputSubset() {
-  cat >test.yml <<EOL
-animals:
-  cat: meow
-EOL
-
-  read -r -d '' expected << EOM
-return {
-	["cat"] = "meow";
-};
-EOM
-
-  X=$(./yq e --output-format=lua '.animals' test.yml)
-  assertEquals "$expected" "$X"
-
-}
-
-source ./scripts/shunit2
+source ./scripts/shunit2
@@ -1,26 +0,0 @@
-#!/bin/bash
-
-setUp() {
-  rm test*.yq || true
-  cat >test.yq <<EOL
-#!./yq
-.a.b
-EOL
-chmod +x test.yq
-
-rm test*.yml || true
-  cat >test.yml <<EOL
-a: {b: apple}
-EOL
-}
-
-testCanExecYqFile() {
-  read -r -d '' expected << EOM
-apple
-EOM
-   X=$(./test.yq test.yml)
-  assertEquals "$expected" "$X"
-}
-
-source ./scripts/shunit2
-
@@ -12,6 +12,6 @@ outputs:
    description: "The complete result from the yq command being run"
 runs:
  using: 'docker'
-  image: 'docker://mikefarah/yq:4-githubaction'
+  image: 'github-action/Dockerfile'
  args:
    - ${{ inputs.cmd }}
@@ -1,15 +1,18 @@
 package cmd

-var unwrapScalarFlag = newUnwrapFlag()
-
-var unwrapScalar = false
+var leadingContentPreProcessing = true
+var unwrapScalar = true

 var writeInplace = false
 var outputToJSON = false
+var outputFormat = "yaml"
+var inputFormat = "yaml"

-var outputFormat = ""
-
-var inputFormat = ""
+var xmlAttributePrefix = "+"
+var xmlContentName = "+content"
+var xmlStrictMode = false
+var xmlKeepNamespace = true
+var xmlUseRawToken = true

 var exitStatus = false
 var forceColor = false
@@ -18,7 +21,6 @@ var colorsEnabled = false
 var indent = 2
 var noDocSeparators = false
 var nullInput = false
-var nulSepOutput = false
 var verbose = false
 var version = false
 var prettyPrint = false
@@ -12,14 +12,8 @@ func createEvaluateAllCommand() *cobra.Command {
 		Use:     "eval-all [expression] [yaml_file1]...",
 		Aliases: []string{"ea"},
 		Short:   "Loads _all_ yaml documents of _all_ yaml files and runs expression once",
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			if len(args) == 0 {
-				return nil, cobra.ShellCompDirectiveNoFileComp
-			}
-			return nil, cobra.ShellCompDirectiveDefault
-		},
 		Example: `
-# Merge f2.yml into f1.yml (in place)
+# Merge f2.yml into f1.yml (inplace)
 yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' f1.yml f2.yml
 ## the same command and expression using shortened names:
 yq ea -i 'select(fi == 0) * select(fi == 1)' f1.yml f2.yml
@@ -32,7 +26,7 @@ yq ea '. as $item ireduce ({}; . * $item )' file1.yml file2.yml ...
 ## use '-' as a filename to pipe from STDIN
 cat file2.yml | yq ea '.a.b' file1.yml - file3.yml
 `,
-		Long: `yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) 
+		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
 See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.

 ## Evaluate All ##
@@ -76,12 +70,12 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 		}()
 	}

-	format, err := yqlib.FormatFromString(outputFormat)
+	format, err := yqlib.OutputFormatFromString(outputFormat)
 	if err != nil {
 		return err
 	}

-	decoder, err := configureDecoder(true)
+	decoder, err := configureDecoder()
 	if err != nil {
 		return err
 	}
@@ -90,15 +84,9 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 	if err != nil {
 		return err
 	}
-	encoder, err := configureEncoder()
-	if err != nil {
-		return err
-	}
+	encoder := configureEncoder(format)

 	printer := yqlib.NewPrinter(encoder, printerWriter)
-	if nulSepOutput {
-		printer.SetNulSepOutput(true)
-	}

 	if frontMatter != "" {
 		frontMatterHandler := yqlib.NewFrontMatterHandler(args[0])
@@ -121,13 +109,13 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 	switch len(args) {
 	case 0:
 		if nullInput {
-			err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(expression), printer)
+			err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(expression), printer, "")
 		} else {
 			cmd.Println(cmd.UsageString())
 			return nil
 		}
 	default:
-		err = allAtOnceEvaluator.EvaluateFiles(processExpression(expression), args, printer, decoder)
+		err = allAtOnceEvaluator.EvaluateFiles(processExpression(expression), args, printer, leadingContentPreProcessing, decoder)
 	}

 	completedSuccessfully = err == nil
@@ -13,12 +13,6 @@ func createEvaluateSequenceCommand() *cobra.Command {
 		Use:     "eval [expression] [yaml_file1]...",
 		Aliases: []string{"e"},
 		Short:   "(default) Apply the expression to each document in each yaml file in sequence",
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			if len(args) == 0 {
-				return nil, cobra.ShellCompDirectiveNoFileComp
-			}
-			return nil, cobra.ShellCompDirectiveDefault
-		},
 		Example: `
 # Reads field under the given path for each file
 yq e '.a.b' f1.yml f2.yml 
@@ -34,10 +28,10 @@ cat file2.yml | yq e '.a.b' file1.yml - file3.yml
 ## Note that editing an empty file does not work.
 yq e -n '.a.b.c = "cat"' 

-# Update a file in place
+# Update a file inplace
 yq e '.a.b = "cool"' -i file.yaml 
 `,
-		Long: `yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) 
+		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
 See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.

 ## Evaluate Sequence ##
@@ -90,7 +84,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 		}()
 	}

-	format, err := yqlib.FormatFromString(outputFormat)
+	format, err := yqlib.OutputFormatFromString(outputFormat)
 	if err != nil {
 		return err
 	}
@@ -99,17 +93,11 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 	if err != nil {
 		return err
 	}
-	encoder, err := configureEncoder()
-	if err != nil {
-		return err
-	}
+	encoder := configureEncoder(format)

 	printer := yqlib.NewPrinter(encoder, printerWriter)
-	if nulSepOutput {
-		printer.SetNulSepOutput(true)
-	}

-	decoder, err := configureDecoder(false)
+	decoder, err := configureDecoder()
 	if err != nil {
 		return err
 	}
@@ -135,13 +123,13 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 	switch len(args) {
 	case 0:
 		if nullInput {
-			err = streamEvaluator.EvaluateNew(processExpression(expression), printer)
+			err = streamEvaluator.EvaluateNew(processExpression(expression), printer, "")
 		} else {
 			cmd.Println(cmd.UsageString())
 			return nil
 		}
 	default:
-		err = streamEvaluator.EvaluateFiles(processExpression(expression), args, printer, decoder)
+		err = streamEvaluator.EvaluateFiles(processExpression(expression), args, printer, leadingContentPreProcessing, decoder)
 	}
 	completedSuccessfully = err == nil

@@ -1,54 +1,21 @@
 package cmd

 import (
-	"fmt"
 	"os"
-	"strings"

-	"github.com/mikefarah/yq/v4/pkg/properties"
-	"github.com/mikefarah/yq/v4/pkg/xml"
 	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"github.com/spf13/cobra"
 	logging "gopkg.in/op/go-logging.v1"
 )

-type runeValue rune
-
-func newRuneVar(p *rune) *runeValue {
-	return (*runeValue)(p)
-}
-
-func (r *runeValue) String() string {
-	return string(*r)
-}
-
-func (r *runeValue) Set(rawVal string) error {
-	val := strings.ReplaceAll(rawVal, "\\n", "\n")
-	val = strings.ReplaceAll(val, "\\t", "\t")
-	val = strings.ReplaceAll(val, "\\r", "\r")
-	val = strings.ReplaceAll(val, "\\f", "\f")
-	val = strings.ReplaceAll(val, "\\v", "\v")
-	if len(val) != 1 {
-		return fmt.Errorf("[%v] is not a valid character. Must be length 1 was %v", val, len(val))
-	}
-	*r = runeValue(rune(val[0]))
-	return nil
-}
-
-func (r *runeValue) Type() string {
-	return "char"
-}
-
 func New() *cobra.Command {
 	var rootCmd = &cobra.Command{
 		Use:   "yq",
-		Short: "yq is a lightweight and portable command-line data file processor.",
-		Long: `yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) 
+		Short: "yq is a lightweight and portable command-line YAML processor.",
+		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
 See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 		Example: `
-# yq tries to auto-detect the file format based off the extension, and defaults to YAML if it's unknown (or piping through STDIN)
-# Use the '-p/--input-format' flag to specify a format type.
-cat file.xml | yq -p xml
+# yq defaults to 'eval' command if no command is specified. See "yq eval --help" for more examples.

 # read the "stuff" node from "myfile.yml"
 yq '.stuff' < myfile.yml
@@ -57,7 +24,7 @@ yq '.stuff' < myfile.yml
 yq -i '.stuff = "foo"' myfile.yml

 # print contents of sample.json as idiomatic YAML
-yq -P -oy sample.json
+yq -P sample.json
 `,

 		RunE: func(cmd *cobra.Command, args []string) error {
@@ -68,26 +35,26 @@ yq -P -oy sample.json
 			return evaluateSequence(cmd, args)

 		},
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
 			cmd.SetOut(cmd.OutOrStdout())
-			level := logging.WARNING
-			stringFormat := `[%{level}] %{color}%{time:15:04:05}%{color:reset} %{message}`

-			if verbose {
-				level = logging.DEBUG
-				stringFormat = `[%{level:5.5s}] %{color}%{time:15:04:05}%{color:bold} %{shortfile:-33s} %{shortfunc:-25s}%{color:reset} %{message}`
-			}
-
-			var format = logging.MustStringFormatter(stringFormat)
+			var format = logging.MustStringFormatter(
+				`%{color}%{time:15:04:05} %{shortfunc} [%{level:.4s}]%{color:reset} %{message}`,
+			)
 			var backend = logging.AddModuleLevel(
 				logging.NewBackendFormatter(logging.NewLogBackend(os.Stderr, "", 0), format))

-			backend.SetLevel(level, "")
+			if verbose {
+				backend.SetLevel(logging.DEBUG, "")
+			} else {
+				backend.SetLevel(logging.ERROR, "")
+			}

 			logging.SetBackend(backend)
 			yqlib.InitExpressionParser()
-
-			return nil
+			yqlib.XMLPreferences.AttributePrefix = xmlAttributePrefix
+			yqlib.XMLPreferences.ContentName = xmlContentName
+			yqlib.XMLPreferences.StrictMode = xmlStrictMode
 		},
 	}

@@ -99,110 +66,35 @@ yq -P -oy sample.json
 		panic(err)
 	}

-	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", fmt.Sprintf("[auto|a|%v] output format type.", yqlib.GetAvailableOutputFormatString()))
-	var outputCompletions = []string{"auto"}
-	for _, formats := range yqlib.GetAvailableOutputFormats() {
-		outputCompletions = append(outputCompletions, formats.FormalName)
-	}
+	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "yaml", "[yaml|y|json|j|props|p|xml|x] output format type.")
+	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "yaml", "[yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.")

-	if err = rootCmd.RegisterFlagCompletionFunc("output-format", cobra.FixedCompletions(outputCompletions, cobra.ShellCompDirectiveNoFileComp)); err != nil {
-		panic(err)
-	}
-	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", fmt.Sprintf("[auto|a|%v] parse format for input.", yqlib.GetAvailableInputFormatString()))
-
-	var inputCompletions = []string{"auto"}
-	for _, formats := range yqlib.GetAvailableInputFormats() {
-		inputCompletions = append(inputCompletions, formats.FormalName)
-	}
-
-	if err = rootCmd.RegisterFlagCompletionFunc("input-format", cobra.FixedCompletions(inputCompletions, cobra.ShellCompDirectiveNoFileComp)); err != nil {
-		panic(err)
-	}
-
-	rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", xml.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
-	if err = rootCmd.RegisterFlagCompletionFunc("xml-attribute-prefix", cobra.NoFileCompletions); err != nil {
-		panic(err)
-	}
-	rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.ContentName, "xml-content-name", xml.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")
-	if err = rootCmd.RegisterFlagCompletionFunc("xml-content-name", cobra.NoFileCompletions); err != nil {
-		panic(err)
-	}
-	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.StrictMode, "xml-strict-mode", xml.ConfiguredXMLPreferences.StrictMode, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
-	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.KeepNamespace, "xml-keep-namespace", xml.ConfiguredXMLPreferences.KeepNamespace, "enables keeping namespace after parsing attributes")
-	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.UseRawToken, "xml-raw-token", xml.ConfiguredXMLPreferences.UseRawToken, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")
-	rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.ProcInstPrefix, "xml-proc-inst-prefix", xml.ConfiguredXMLPreferences.ProcInstPrefix, "prefix for xml processing instructions (e.g. <?xml version=\"1\"?>)")
-	if err = rootCmd.RegisterFlagCompletionFunc("xml-proc-inst-prefix", cobra.NoFileCompletions); err != nil {
-		panic(err)
-	}
-	rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.DirectiveName, "xml-directive-name", xml.ConfiguredXMLPreferences.DirectiveName, "name for xml directives (e.g. <!DOCTYPE thing cat>)")
-	if err = rootCmd.RegisterFlagCompletionFunc("xml-directive-name", cobra.NoFileCompletions); err != nil {
-		panic(err)
-	}
-	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", xml.ConfiguredXMLPreferences.SkipProcInst, "skip over process instructions (e.g. <?xml version=\"1\"?>)")
-	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", xml.ConfiguredXMLPreferences.SkipDirectives, "skip over directives (e.g. <!DOCTYPE thing cat>)")
-
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredCsvPreferences.AutoParse, "csv-auto-parse", yqlib.ConfiguredCsvPreferences.AutoParse, "parse CSV YAML/JSON values")
-	rootCmd.PersistentFlags().Var(newRuneVar(&yqlib.ConfiguredCsvPreferences.Separator), "csv-separator", "CSV Separator character")
-
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredTsvPreferences.AutoParse, "tsv-auto-parse", yqlib.ConfiguredTsvPreferences.AutoParse, "parse TSV YAML/JSON values")
-
-	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocPrefix, "lua-prefix", yqlib.ConfiguredLuaPreferences.DocPrefix, "prefix")
-	if err = rootCmd.RegisterFlagCompletionFunc("lua-prefix", cobra.NoFileCompletions); err != nil {
-		panic(err)
-	}
-	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocSuffix, "lua-suffix", yqlib.ConfiguredLuaPreferences.DocSuffix, "suffix")
-	if err = rootCmd.RegisterFlagCompletionFunc("lua-suffix", cobra.NoFileCompletions); err != nil {
-		panic(err)
-	}
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.UnquotedKeys, "lua-unquoted", yqlib.ConfiguredLuaPreferences.UnquotedKeys, "output unquoted string keys (e.g. {foo=\"bar\"})")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.Globals, "lua-globals", yqlib.ConfiguredLuaPreferences.Globals, "output keys as top-level global variables")
-
-	rootCmd.PersistentFlags().StringVar(&properties.ConfiguredPropertiesPreferences.KeyValueSeparator, "properties-separator", properties.ConfiguredPropertiesPreferences.KeyValueSeparator, "separator to use between keys and values")
-	rootCmd.PersistentFlags().BoolVar(&properties.ConfiguredPropertiesPreferences.UseArrayBrackets, "properties-array-brackets", properties.ConfiguredPropertiesPreferences.UseArrayBrackets, "use [x] in array paths (e.g. for SpringBoot)")
-
-	rootCmd.PersistentFlags().BoolVar(&yqlib.StringInterpolationEnabled, "string-interpolation", yqlib.StringInterpolationEnabled, "Toggles strings interpolation of \\(exp)")
+	rootCmd.PersistentFlags().StringVar(&xmlAttributePrefix, "xml-attribute-prefix", "+", "prefix for xml attributes")
+	rootCmd.PersistentFlags().StringVar(&xmlContentName, "xml-content-name", "+content", "name for xml content (if no attribute name is present).")
+	rootCmd.PersistentFlags().BoolVar(&xmlStrictMode, "xml-strict-mode", false, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
+	rootCmd.PersistentFlags().BoolVar(&xmlKeepNamespace, "xml-keep-namespace", true, "enables keeping namespace after parsing attributes")
+	rootCmd.PersistentFlags().BoolVar(&xmlUseRawToken, "xml-raw-token", true, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")

 	rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.")
 	rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)")

 	rootCmd.PersistentFlags().IntVarP(&indent, "indent", "I", 2, "sets indent level for output")
-	if err = rootCmd.RegisterFlagCompletionFunc("indent", cobra.NoFileCompletions); err != nil {
-		panic(err)
-	}
 	rootCmd.Flags().BoolVarP(&version, "version", "V", false, "Print version information and quit")
-	rootCmd.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the file in place of first file given.")
-	rootCmd.PersistentFlags().VarP(unwrapScalarFlag, "unwrapScalar", "r", "unwrap scalar, print the value with no quotes, colors or comments. Defaults to true for yaml")
-	rootCmd.PersistentFlags().Lookup("unwrapScalar").NoOptDefVal = "true"
-	rootCmd.PersistentFlags().BoolVarP(&nulSepOutput, "nul-output", "0", false, "Use NUL char to separate values. If unwrap scalar is also set, fail if unwrapped scalar contains NUL char.")
-
+	rootCmd.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the file inplace of first file given.")
+	rootCmd.PersistentFlags().BoolVarP(&unwrapScalar, "unwrapScalar", "r", true, "unwrap scalar, print the value with no quotes, colors or comments")
 	rootCmd.PersistentFlags().BoolVarP(&prettyPrint, "prettyPrint", "P", false, "pretty print, shorthand for '... style = \"\"'")
 	rootCmd.PersistentFlags().BoolVarP(&exitStatus, "exit-status", "e", false, "set exit status if there are no matches or null or false is returned")

 	rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")
 	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
 	rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
-	if err = rootCmd.RegisterFlagCompletionFunc("front-matter", cobra.FixedCompletions([]string{"extract", "process"}, cobra.ShellCompDirectiveNoFileComp)); err != nil {
-		panic(err)
-	}
 	rootCmd.PersistentFlags().StringVarP(&forceExpression, "expression", "", "", "forcibly set the expression argument. Useful when yq argument detection thinks your expression is a file.")
-	if err = rootCmd.RegisterFlagCompletionFunc("expression", cobra.NoFileCompletions); err != nil {
-		panic(err)
-	}
-	rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredYamlPreferences.LeadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.")
+	rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.")

 	rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter.")
-	if err = rootCmd.RegisterFlagCompletionFunc("split-exp", cobra.NoFileCompletions); err != nil {
-		panic(err)
-	}
 	rootCmd.PersistentFlags().StringVarP(&splitFileExpFile, "split-exp-file", "", "", "Use a file to specify the split-exp expression.")
-	if err = rootCmd.MarkPersistentFlagFilename("split-exp-file"); err != nil {
-		panic(err)
-	}

 	rootCmd.PersistentFlags().StringVarP(&expressionFile, "from-file", "", "", "Load expression from specified file.")
-	if err = rootCmd.MarkPersistentFlagFilename("from-file"); err != nil {
-		panic(err)
-	}

 	rootCmd.AddCommand(
 		createEvaluateSequenceCommand(),
@@ -42,12 +42,12 @@ $ yq shell-completion fish > ~/.config/fish/completions/yq.fish
 `,
 	DisableFlagsInUseLine: true,
 	ValidArgs:             []string{"bash", "zsh", "fish", "powershell"},
-	Args:                  cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs),
+	Args:                  cobra.ExactValidArgs(1),
 	RunE: func(cmd *cobra.Command, args []string) error {
 		var err error = nil
 		switch args[0] {
 		case "bash":
-			err = cmd.Root().GenBashCompletionV2(os.Stdout, true)
+			err = cmd.Root().GenBashCompletion(os.Stdout)
 		case "zsh":
 			err = cmd.Root().GenZshCompletion(os.Stdout)
 		case "fish":
@@ -1,46 +0,0 @@
-package cmd
-
-import (
-	"strconv"
-
-	"github.com/spf13/pflag"
-)
-
-type boolFlag interface {
-	pflag.Value
-	IsExplicitlySet() bool
-	IsSet() bool
-}
-
-type unwrapScalarFlagStrc struct {
-	explicitlySet bool
-	value         bool
-}
-
-func newUnwrapFlag() boolFlag {
-	return &unwrapScalarFlagStrc{value: true}
-}
-
-func (f *unwrapScalarFlagStrc) IsExplicitlySet() bool {
-	return f.explicitlySet
-}
-
-func (f *unwrapScalarFlagStrc) IsSet() bool {
-	return f.value
-}
-
-func (f *unwrapScalarFlagStrc) String() string {
-	return strconv.FormatBool(f.value)
-}
-
-func (f *unwrapScalarFlagStrc) Set(value string) error {
-
-	v, err := strconv.ParseBool(value)
-	f.value = v
-	f.explicitlySet = true
-	return err
-}
-
-func (*unwrapScalarFlagStrc) Type() string {
-	return "bool"
-}
@@ -4,19 +4,12 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"strings"

-	"github.com/mikefarah/yq/v4/pkg/properties"
-	"github.com/mikefarah/yq/v4/pkg/xml"
 	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"github.com/spf13/cobra"
 	"gopkg.in/op/go-logging.v1"
 )

-func isAutomaticOutputFormat() bool {
-	return outputFormat == "" || outputFormat == "auto" || outputFormat == "a"
-}
-
 func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 	cmd.SilenceUsage = true

@@ -45,7 +38,7 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 	}

 	if writeInplace && (len(args) == 0 || args[0] == "-") {
-		return "", nil, fmt.Errorf("write in place flag only applicable when giving an expression and at least one file")
+		return "", nil, fmt.Errorf("write inplace flag only applicable when giving an expression and at least one file")
 	}

 	if frontMatter != "" && len(args) == 0 {
@@ -53,82 +46,38 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 	}

 	if writeInplace && splitFileExp != "" {
-		return "", nil, fmt.Errorf("write in place cannot be used with split file")
+		return "", nil, fmt.Errorf("write inplace cannot be used with split file")
 	}

 	if nullInput && len(args) > 0 {
 		return "", nil, fmt.Errorf("cannot pass files in when using null-input flag")
 	}

-	inputFilename := ""
-	if len(args) > 0 {
-		inputFilename = args[0]
-	}
-	if inputFormat == "" || inputFormat == "auto" || inputFormat == "a" {
-
-		inputFormat = yqlib.FormatStringFromFilename(inputFilename)
-
-		_, err := yqlib.FormatFromString(inputFormat)
-		if err != nil {
-			// unknown file type, default to yaml
-			yqlib.GetLogger().Debug("Unknown file format extension '%v', defaulting to yaml", inputFormat)
-			inputFormat = "yaml"
-			if isAutomaticOutputFormat() {
-				outputFormat = "yaml"
-			}
-		} else if isAutomaticOutputFormat() {
-			// automatic input worked, we can do it for output too unless specified
-			if inputFormat == "json" {
-				yqlib.GetLogger().Warning("JSON file output is now JSON by default (instead of yaml). Use '-oy' or '--output-format=yaml' for yaml output")
-			}
-			outputFormat = inputFormat
-		}
-	} else if isAutomaticOutputFormat() {
-		// backwards compatibility -
-		// before this was introduced, `yq -pcsv things.csv`
-		// would produce *yaml* output.
-		//
-		outputFormat = yqlib.FormatStringFromFilename(inputFilename)
-		if inputFilename != "-" {
-			yqlib.GetLogger().Warning("yq default output is now 'auto' (based on the filename extension). Normally yq would output '%v', but for backwards compatibility 'yaml' has been set. Please use -oy to specify yaml, or drop the -p flag.", outputFormat)
-		}
-		outputFormat = "yaml"
-	}
-
-	outputFormatType, err := yqlib.FormatFromString(outputFormat)
-
-	if err != nil {
-		return "", nil, err
-	}
-	yqlib.GetLogger().Debug("Using input format %v", inputFormat)
-	yqlib.GetLogger().Debug("Using output format %v", outputFormat)
-
-	if outputFormatType == yqlib.YamlFormat ||
-		outputFormatType == properties.PropertiesFormat {
-		unwrapScalar = true
-	}
-	if unwrapScalarFlag.IsExplicitlySet() {
-		unwrapScalar = unwrapScalarFlag.IsSet()
-	}
-
 	return expression, args, nil
 }

-func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
-	format, err := yqlib.FormatFromString(inputFormat)
+func configureDecoder() (yqlib.Decoder, error) {
+	yqlibInputFormat, err := yqlib.InputFormatFromString(inputFormat)
 	if err != nil {
 		return nil, err
 	}
-	yqlib.ConfiguredYamlPreferences.EvaluateTogether = evaluateTogether
-
-	yqlibDecoder := format.DecoderFactory()
-	if yqlibDecoder == nil {
-		return nil, fmt.Errorf("no support for %s input format", inputFormat)
+	switch yqlibInputFormat {
+	case yqlib.XMLInputFormat:
+		return yqlib.NewXMLDecoder(xmlAttributePrefix, xmlContentName, xmlStrictMode, xmlKeepNamespace, xmlUseRawToken), nil
+	case yqlib.PropertiesInputFormat:
+		return yqlib.NewPropertiesDecoder(), nil
+	case yqlib.JsonInputFormat:
+		return yqlib.NewJSONDecoder(), nil
+	case yqlib.CSVObjectInputFormat:
+		return yqlib.NewCSVObjectDecoder(','), nil
+	case yqlib.TSVObjectInputFormat:
+		return yqlib.NewCSVObjectDecoder('\t'), nil
 	}
-	return yqlibDecoder, nil
+
+	return yqlib.NewYamlDecoder(), nil
 }

-func configurePrinterWriter(format *yqlib.Format, out io.Writer) (yqlib.PrinterWriter, error) {
+func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yqlib.PrinterWriter, error) {

 	var printerWriter yqlib.PrinterWriter

@@ -145,30 +94,22 @@ func configurePrinterWriter(format *yqlib.Format, out io.Writer) (yqlib.PrinterW
 	return printerWriter, nil
 }

-func configureEncoder() (yqlib.Encoder, error) {
-	yqlibOutputFormat, err := yqlib.FormatFromString(outputFormat)
-	if err != nil {
-		return nil, err
+func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
+	switch format {
+	case yqlib.JSONOutputFormat:
+		return yqlib.NewJSONEncoder(indent, colorsEnabled)
+	case yqlib.PropsOutputFormat:
+		return yqlib.NewPropertiesEncoder(unwrapScalar)
+	case yqlib.CSVOutputFormat:
+		return yqlib.NewCsvEncoder(',')
+	case yqlib.TSVOutputFormat:
+		return yqlib.NewCsvEncoder('\t')
+	case yqlib.YamlOutputFormat:
+		return yqlib.NewYamlEncoder(indent, colorsEnabled, !noDocSeparators, unwrapScalar)
+	case yqlib.XMLOutputFormat:
+		return yqlib.NewXMLEncoder(indent, xmlAttributePrefix, xmlContentName)
 	}
-	xml.ConfiguredXMLPreferences.Indent = indent
-	yqlib.ConfiguredYamlPreferences.Indent = indent
-	yqlib.ConfiguredJSONPreferences.Indent = indent
-
-	yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
-	properties.ConfiguredPropertiesPreferences.UnwrapScalar = unwrapScalar
-	yqlib.ConfiguredJSONPreferences.UnwrapScalar = unwrapScalar
-
-	yqlib.ConfiguredYamlPreferences.ColorsEnabled = colorsEnabled
-	yqlib.ConfiguredJSONPreferences.ColorsEnabled = colorsEnabled
-
-	yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
-
-	encoder := yqlibOutputFormat.EncoderFactory()
-
-	if encoder == nil {
-		return nil, fmt.Errorf("no support for %s output format", outputFormat)
-	}
-	return encoder, err
+	panic("invalid encoder")
 }

 // this is a hack to enable backwards compatibility with githubactions (which pipe /dev/null into everything)
@@ -192,11 +133,8 @@ func maybeFile(str string) bool {
 }

 func processStdInArgs(args []string) []string {
-	stat, err := os.Stdin.Stat()
-	if err != nil {
-		yqlib.GetLogger().Debugf("error getting stdin: %v", err)
-	}
-	pipingStdin := stat != nil && (stat.Mode()&os.ModeCharDevice) == 0
+	stat, _ := os.Stdin.Stat()
+	pipingStdin := (stat.Mode() & os.ModeCharDevice) == 0

 	// if we've been given a file, don't automatically
 	// read from stdin.
@@ -220,25 +158,15 @@ func processStdInArgs(args []string) []string {

 func processArgs(originalArgs []string) (string, []string, error) {
 	expression := forceExpression
-	args := processStdInArgs(originalArgs)
-	maybeFirstArgIsAFile := len(args) > 0 && maybeFile(args[0])
-
-	if expressionFile == "" && maybeFirstArgIsAFile && strings.HasSuffix(args[0], ".yq") {
-		// lets check if an expression file was given
-		yqlib.GetLogger().Debug("Assuming arg %v is an expression file", args[0])
-		expressionFile = args[0]
-		args = args[1:]
-	}
-
 	if expressionFile != "" {
 		expressionBytes, err := os.ReadFile(expressionFile)
 		if err != nil {
 			return "", nil, err
 		}
-		//replace \r\n (windows) with good ol' unix file endings.
-		expression = strings.ReplaceAll(string(expressionBytes), "\r\n", "\n")
+		expression = string(expressionBytes)
 	}

+	args := processStdInArgs(originalArgs)
 	yqlib.GetLogger().Debugf("processed args: %v", args)
 	if expression == "" && len(args) > 0 && args[0] != "-" && !maybeFile(args[0]) {
 		yqlib.GetLogger().Debug("assuming expression is '%v'", args[0])
@@ -11,7 +11,7 @@ var (
 	GitDescribe string

 	// Version is main version number that is being run at the moment.
-	Version = "v4.42.1"
+	Version = "4.26.1"

 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release
@@ -1,14 +0,0 @@
---
-$schema: https://raw.githubusercontent.com/streetsidesoftware/cspell/main/cspell.schema.json
-version: '0.2'
-language: en-GB
-dictionaryDefinitions:
-  - name: project-words
-    path: './project-words.txt'
-    addWords: true
-dictionaries:
-  - project-words
-ignorePaths:
-  - 'vendor'
-  - 'bin'
-  - '/project-words.txt'
@@ -103,7 +103,7 @@ yq (4.16.2) focal; urgency=medium
  * Improved extract-checksum.sh
  * Bump github.com/spf13/cobra from 1.2.1 to 1.3.0 (#1039)
  * enable more linters (#1043)
-  * Bump Golang compiler #1037
+  * Bump golang compiler #1037

 -- Roberto Mier Escandon <rmescandon@gmail.com>  Tue, 21 Dec 2021 09:41:44 +0000

@@ -127,7 +127,7 @@ yq (4.9.6) focal; urgency=medium

  * Added darwin/arm64 build, thanks @alecthomas
  * Incremented docker alpine base version, thanks @da6d6i7-bronga
-  * Bug fix: multiline expression
+  * Bug fix: multine expression
  * Bug fix: special character

 -- Roberto Mier Escandon <rmescandon@gmail.com>  Tue, 29 Jun 2021 21:32:14 +0200
@@ -135,7 +135,7 @@ yq (4.9.6) focal; urgency=medium
 yq (3.3.2) focal; urgency=medium

  * Bug fix: existStatus bug (#459)
-  * Automatically makes an os temp directory if it does not exist (#461)
+  * Automatically makes a os temp directory if it does not exist (#461)

 -- Roberto Mier Escandon <rmescandon@gmail.com>  Fri, 07 Aug 2020 18:53:01 +0200

@@ -16,7 +16,7 @@ XSBC-Original-Maintainer: Roberto Mier Escandón <rmescandon@gmail.com>
 Package: yq
 Architecture: any
 Depends: ${shlibs:Depends}, ${misc:Depends}
-Description: lightweight and portable command-line data file processor
+Description: lightweight and portable command-line YAML processor
 .
 The aim of the project is to be the 
 [jq](https://github.com/stedolan/jq) or sed of yaml files.
@@ -1,2 +1,11 @@
- [cat, dog, frog, cow]
- [apple, banana, grape, mango]
+---
+- become: true
+  gather_facts: false
+  hosts: lalaland
+  name: "Apply smth"
+  roles:
+    - lala
+    - land
+  serial: 1
+- become: false
+  gather_facts: true
@@ -1,9 +0,0 @@
-return {
-	["country"] = "Australia"; -- this place
-	["cities"] = {
-		"Sydney",
-		"Melbourne",
-		"Brisbane",
-		"Perth",
-	};
-};
@@ -1,11 +1,12 @@
-# block comments come through
-person: # neither do comments on maps
-    name: Mike Wazowski # comments on values appear
-    pets: 
-    - cat # comments on array values appear
-    - dog # comments on array values appear
-    - things:
-        - frog
-    food: [pizza] # comments on arrays do not
-emptyArray: []
-emptyMap: []
+# --------------------------------------------------
+# It's a test with comment
+# --------------------------------------------------
+test:
+  - name: a
+  - name: b
+  - name: c
+groups:
+  - name: a
+  - name: b
+  - name: c
+  - name: c
@@ -1,5 +0,0 @@
-#! yq
-.[] |(
-    ( select(kind == "scalar") | key + "='" + . + "'"),
-    ( select(kind == "seq") | key + "=(" + (map("'" + . + "'") | join(",")) + ")")
-)
@@ -1 +1,14 @@
-<zoo><thing><frog>boing</frog></thing></zoo>
+<!-- before cat -->
+<cat>
+	<!-- in cat before -->
+	<x>3<!-- multi
+line comment
+for x --></x>
+	<y>
+		<!-- in y before -->
+		<d><!-- in d before -->4<!-- in d after --></d>
+		<!-- in y after -->
+	</y>
+	<!-- in_cat_after -->
+</cat>
+<!-- after cat -->
@@ -1,26 +0,0 @@
-
-
-# This is a TOML document
-
-title = "TOML Example"
-
-[owner]
-name = "Tom Preston-Werner"
-dob = 1979-05-27T07:32:00-08:00
-
-[database]
-enabled = true
-ports = [ 8000, 8001, 8002 ]
-data = [ ["delta", "phi"], [3.14] ]
-temp_targets = { cpu = 79.5, case = 72.0 }
-
-[servers]
-
-[servers.alpha]
-ip = "10.0.0.1"
-role = "frontend"
-
-[servers.beta]
-ip = "10.0.0.2"
-role = "backend"
-
@@ -1,2 +1,11 @@
- # things
-a: apple
+# Some doc
+
+a: true
+b:
+  c: 2
+  d: [3, 4, 5]
+  e:
+    - name: fred
+      value: 3
+    - name: sam
+      value: 4
@@ -1,3 +1,3 @@
 name,numberOfCats,likesApples,height
-,1,true,168.8
+Gary,1,true,168.8
 Samantha's Rabbit,2,false,-188.8
@@ -1,4 +0,0 @@
---
-# comment
-# about things
-a: cat
@@ -2,8 +2,7 @@ FROM mikefarah/yq:4

 COPY entrypoint.sh /entrypoint.sh

-# github action recommendation is to run as root.
-# https://docs.github.com/en/actions/creating-actions/dockerfile-support-for-github-actions#user
-USER root
+# this seems to be the default user in github actions
+USER 1001

 ENTRYPOINT ["/entrypoint.sh"]
@@ -2,11 +2,8 @@
 set -e
 echo "::debug::\$cmd: $1"
 RESULT=$(eval "$1")
+RESULT="${RESULT//'%'/'%25'}"
+RESULT="${RESULT//$'\n'/'%0A'}"
+RESULT="${RESULT//$'\r'/'%0D'}"
 echo "::debug::\$RESULT: $RESULT"
-# updating from 
-# https://github.com/orgs/community/discussions/26288#discussioncomment-3876281
-# https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-output-parameter
-delimiter=$(cat /proc/sys/kernel/random/uuid)
-echo "result<<${delimiter}" >> "${GITHUB_OUTPUT}"
-echo "${RESULT}" >> "${GITHUB_OUTPUT}"
-echo "${delimiter}" >> "${GITHUB_OUTPUT}"
+echo ::set-output name=result::"$RESULT"
@@ -1,33 +1,30 @@
 module github.com/mikefarah/yq/v4

 require (
-	github.com/a8m/envsubst v1.4.2
-	github.com/alecthomas/participle/v2 v2.1.1
-	github.com/alecthomas/repr v0.4.0
-	github.com/dimchansky/utfbom v1.1.1
-	github.com/elliotchance/orderedmap v1.5.1
-	github.com/fatih/color v1.16.0
-	github.com/goccy/go-json v0.10.2
-	github.com/goccy/go-yaml v1.11.3
-	github.com/jinzhu/copier v0.4.0
-	github.com/magiconair/properties v1.8.7
-	github.com/pelletier/go-toml/v2 v2.1.1
+	github.com/a8m/envsubst v1.3.0
+	github.com/alecthomas/participle/v2 v2.0.0-beta.4
+	github.com/alecthomas/repr v0.1.0
+	github.com/elliotchance/orderedmap v1.4.0
+	github.com/fatih/color v1.13.0
+	github.com/goccy/go-json v0.9.10
+	github.com/goccy/go-yaml v1.9.5
+	github.com/jinzhu/copier v0.3.5
+	github.com/magiconair/properties v1.8.6
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
-	github.com/spf13/cobra v1.8.0
-	github.com/spf13/pflag v1.0.5
-	github.com/yuin/gopher-lua v1.1.1
-	golang.org/x/net v0.22.0
-	golang.org/x/text v0.14.0
+	github.com/spf13/cobra v1.5.0
+	golang.org/x/net v0.0.0-20220708220712-1185a9018129
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.1
 )

 require (
-	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.20 // indirect
-	golang.org/x/sys v0.18.0 // indirect
-	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 )

-go 1.21
+go 1.18
@@ -1,85 +1,81 @@
-github.com/a8m/envsubst v1.4.2 h1:4yWIHXOLEJHQEFd4UjrWDrYeYlV7ncFWJOCBRLOZHQg=
-github.com/a8m/envsubst v1.4.2/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
-github.com/alecthomas/assert/v2 v2.3.0 h1:mAsH2wmvjsuvyBvAmCtm7zFsBlb8mIHx5ySLVdDZXL0=
-github.com/alecthomas/assert/v2 v2.3.0/go.mod h1:pXcQ2Asjp247dahGEmsZ6ru0UVwnkhktn7S0bBDLxvQ=
-github.com/alecthomas/participle/v2 v2.1.1 h1:hrjKESvSqGHzRb4yW1ciisFJ4p3MGYih6icjJvbsmV8=
-github.com/alecthomas/participle/v2 v2.1.1/go.mod h1:Y1+hAs8DHPmc3YUFzqllV+eSQ9ljPTk0ZkPMtEdAx2c=
-github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
-github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
-github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/a8m/envsubst v1.3.0 h1:GmXKmVssap0YtlU3E230W98RWtWCyIZzjtf1apWWyAg=
+github.com/a8m/envsubst v1.3.0/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
+github.com/alecthomas/assert/v2 v2.0.3 h1:WKqJODfOiQG0nEJKFKzDIG3E29CN2/4zR9XGJzKIkbg=
+github.com/alecthomas/participle/v2 v2.0.0-beta.4 h1:ublfGBm+x+p2j7KotHhrUMbKtejT7M0Gv1Mt1u3absw=
+github.com/alecthomas/participle/v2 v2.0.0-beta.4/go.mod h1:RC764t6n4L8D8ITAJv0qdokritYSNR3wV5cVwmIEaMM=
+github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE=
+github.com/alecthomas/repr v0.1.0/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
-github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
-github.com/elliotchance/orderedmap v1.5.1 h1:G1X4PYlljzimbdQ3RXmtIZiQ9d6aRQ3sH1nzjq5mECE=
-github.com/elliotchance/orderedmap v1.5.1/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
-github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
-github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
-github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
+github.com/elliotchance/orderedmap v1.4.0 h1:wZtfeEONCbx6in1CZyE6bELEt/vFayMvsxqI5SgsR+A=
+github.com/elliotchance/orderedmap v1.4.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
+github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
+github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
+github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
+github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
-github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
-github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
-github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
-github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
-github.com/goccy/go-yaml v1.11.3 h1:B3W9IdWbvrUu2OYQGwvU1nZtvMQJPBKgBUuweJjLj6I=
-github.com/goccy/go-yaml v1.11.3/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/goccy/go-json v0.9.10 h1:hCeNmprSNLB8B8vQKWl6DpuH0t60oEs+TAk9a7CScKc=
+github.com/goccy/go-json v0.9.10/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/goccy/go-yaml v1.9.5 h1:Eh/+3uk9kLxG4koCX6lRMAPS1OaMSAi+FJcya0INdB0=
+github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
-github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
-github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
-github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
-github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
-github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
-github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg=
+github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
-github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
-github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
-github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
-github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
-github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
-github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
+github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=
+github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
+github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
-github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
+github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
+github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
-github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
-golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
-golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
-golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20220708220712-1185a9018129 h1:vucSRfWwTsoXro7P+3Cjlr6flUMtzCwzlvkxEQtHHB0=
+golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e h1:NHvCuwuS43lGnYhten69ZWqi2QOj/CiDNcKbVqwVoew=
+golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0=
+golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW8s2qTSe3wGBtvo0MbVQG/c5k8RE=
 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -29,7 +29,7 @@ It pipes the current, lets call it 'root' context through the `lhs` expression o
 cat
 ```

-Side note: this node holds not only its value 'cat', but comments and metadata too, including path and parent information.
+Sidenote: this node holds not only its value 'cat', but comments and metadata too, including path and parent information.

 The `=` operator then pipes the 'root' context through the `rhs` expression of `.b` to return the node

@@ -37,7 +37,7 @@ The `=` operator then pipes the 'root' context through the `rhs` expression of `
 dog
 ```

-Both sides have now been evaluated, so now the operator copies across the value from the RHS (`.b`) to the LHS (`.a`), and it returns the now updated context:
+Both sides have now been evaluated, so now the operator copies across the value from the RHS (`.b`) to the the LHS (`.a`), and it returns the now updated context:

 ```yaml
 a: dog
@@ -1,117 +0,0 @@
-package properties
-
-import (
-	"bytes"
-	"fmt"
-	"io"
-	"strconv"
-	"strings"
-
-	"github.com/magiconair/properties"
-	"github.com/mikefarah/yq/v4/pkg/yqlib"
-)
-
-type propertiesDecoder struct {
-	reader   io.Reader
-	finished bool
-	d        yqlib.DataTreeNavigator
-}
-
-func NewPropertiesDecoder() yqlib.Decoder {
-	return &propertiesDecoder{d: yqlib.NewDataTreeNavigator(), finished: false}
-}
-
-func (dec *propertiesDecoder) Init(reader io.Reader) error {
-	dec.reader = reader
-	dec.finished = false
-	return nil
-}
-
-func parsePropKey(key string) []interface{} {
-	pathStrArray := strings.Split(key, ".")
-	path := make([]interface{}, len(pathStrArray))
-	for i, pathStr := range pathStrArray {
-		num, err := strconv.ParseInt(pathStr, 10, 32)
-		if err == nil {
-			path[i] = num
-		} else {
-			path[i] = pathStr
-		}
-	}
-	return path
-}
-
-func (dec *propertiesDecoder) processComment(c string) string {
-	if c == "" {
-		return ""
-	}
-	return "# " + c
-}
-
-func (dec *propertiesDecoder) applyPropertyComments(context yqlib.Context, path []interface{}, comments []string) error {
-	rhsCandidateNode := &yqlib.CandidateNode{
-		Tag:         "!!str",
-		Value:       fmt.Sprintf("%v", path[len(path)-1]),
-		HeadComment: dec.processComment(strings.Join(comments, "\n")),
-		Kind:        yqlib.ScalarNode,
-	}
-	rhsCandidateNode.Tag = rhsCandidateNode.GuessTagFromCustomType()
-	return dec.d.DeeplyAssignKey(context, path, rhsCandidateNode)
-}
-
-func (dec *propertiesDecoder) applyProperty(context yqlib.Context, properties *properties.Properties, key string) error {
-	value, _ := properties.Get(key)
-	path := parsePropKey(key)
-
-	propertyComments := properties.GetComments(key)
-	if len(propertyComments) > 0 {
-		err := dec.applyPropertyComments(context, path, propertyComments)
-		if err != nil {
-			return nil
-		}
-	}
-
-	rhsNode := yqlib.CreateStringScalarNode(value)
-	rhsNode.Tag = rhsNode.GuessTagFromCustomType()
-
-	return dec.d.DeeplyAssign(context, path, rhsNode)
-}
-
-func (dec *propertiesDecoder) Decode() (*yqlib.CandidateNode, error) {
-	if dec.finished {
-		return nil, io.EOF
-	}
-	buf := new(bytes.Buffer)
-
-	if _, err := buf.ReadFrom(dec.reader); err != nil {
-		return nil, err
-	}
-	if buf.Len() == 0 {
-		dec.finished = true
-		return nil, io.EOF
-	}
-	properties, err := properties.LoadString(buf.String())
-	if err != nil {
-		return nil, err
-	}
-	properties.DisableExpansion = true
-
-	rootMap := &yqlib.CandidateNode{
-		Kind: yqlib.MappingNode,
-		Tag:  "!!map",
-	}
-
-	context := yqlib.Context{}
-	context = context.SingleChildContext(rootMap)
-
-	for _, key := range properties.Keys() {
-		if err := dec.applyProperty(context, properties, key); err != nil {
-			return nil, err
-		}
-
-	}
-	dec.finished = true
-
-	return rootMap, nil
-
-}
@@ -1,208 +0,0 @@
-package properties
-
-import (
-	"bufio"
-	"bytes"
-	"strings"
-	"testing"
-
-	"github.com/mikefarah/yq/v4/test"
-)
-
-type keyValuePair struct {
-	key     string
-	value   string
-	comment string
-}
-
-func (kv *keyValuePair) String(unwrap bool, sep string) string {
-	builder := strings.Builder{}
-
-	if kv.comment != "" {
-		builder.WriteString(kv.comment)
-		builder.WriteString("\n")
-	}
-
-	builder.WriteString(kv.key)
-	builder.WriteString(sep)
-
-	if unwrap {
-		builder.WriteString(kv.value)
-	} else {
-		builder.WriteString("\"")
-		builder.WriteString(kv.value)
-		builder.WriteString("\"")
-	}
-
-	return builder.String()
-}
-
-type testProperties struct {
-	pairs []keyValuePair
-}
-
-func (tp *testProperties) String(unwrap bool, sep string) string {
-	kvs := []string{}
-
-	for _, kv := range tp.pairs {
-		kvs = append(kvs, kv.String(unwrap, sep))
-	}
-
-	return strings.Join(kvs, "\n")
-}
-
-func yamlToProps(sampleYaml string, unwrapScalar bool, separator string) string {
-	var output bytes.Buffer
-	writer := bufio.NewWriter(&output)
-
-	var propsEncoder = NewPropertiesEncoder(PropertiesPreferences{KeyValueSeparator: separator, UnwrapScalar: unwrapScalar})
-	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
-	if err != nil {
-		panic(err)
-	}
-	node := inputs.Front().Value.(*CandidateNode)
-	err = propsEncoder.Encode(writer, node)
-	if err != nil {
-		panic(err)
-	}
-	writer.Flush()
-
-	return strings.TrimSuffix(output.String(), "\n")
-}
-
-func doTest(t *testing.T, sampleYaml string, props testProperties, testUnwrapped, testWrapped bool) {
-	wraps := []bool{}
-	if testUnwrapped {
-		wraps = append(wraps, true)
-	}
-	if testWrapped {
-		wraps = append(wraps, false)
-	}
-
-	for _, unwrap := range wraps {
-		for _, sep := range []string{" = ", ";", "=", " "} {
-			var actualProps = yamlToProps(sampleYaml, unwrap, sep)
-			test.AssertResult(t, props.String(unwrap, sep), actualProps)
-		}
-	}
-}
-
-func TestPropertiesEncoderSimple(t *testing.T) {
-	var sampleYaml = `a: 'bob cool'`
-
-	doTest(
-		t, sampleYaml,
-		testProperties{
-			pairs: []keyValuePair{
-				{
-					key:   "a",
-					value: "bob cool",
-				},
-			},
-		},
-		true, true,
-	)
-}
-
-func TestPropertiesEncoderSimpleWithComments(t *testing.T) {
-	var sampleYaml = `a: 'bob cool' # line`
-
-	doTest(
-		t, sampleYaml,
-		testProperties{
-			pairs: []keyValuePair{
-				{
-					key:     "a",
-					value:   "bob cool",
-					comment: "# line",
-				},
-			},
-		},
-		true, true,
-	)
-}
-
-func TestPropertiesEncoderDeep(t *testing.T) {
-	var sampleYaml = `a: 
-  b: "bob cool"
-`
-
-	doTest(
-		t, sampleYaml,
-		testProperties{
-			pairs: []keyValuePair{
-				{
-					key:   "a.b",
-					value: "bob cool",
-				},
-			},
-		},
-		true, true,
-	)
-}
-
-func TestPropertiesEncoderDeepWithComments(t *testing.T) {
-	var sampleYaml = `a:  # a thing
-  b: "bob cool" # b thing
-`
-
-	doTest(
-		t, sampleYaml,
-		testProperties{
-			pairs: []keyValuePair{
-				{
-					key:     "a.b",
-					value:   "bob cool",
-					comment: "# b thing",
-				},
-			},
-		},
-		true, true,
-	)
-}
-
-func TestPropertiesEncoderArray_Unwrapped(t *testing.T) {
-	var sampleYaml = `a: 
-  b: [{c: dog}, {c: cat}]
-`
-
-	doTest(
-		t, sampleYaml,
-		testProperties{
-			pairs: []keyValuePair{
-				{
-					key:   "a.b.0.c",
-					value: "dog",
-				},
-				{
-					key:   "a.b.1.c",
-					value: "cat",
-				},
-			},
-		},
-		true, false,
-	)
-}
-
-func TestPropertiesEncoderArray_Wrapped(t *testing.T) {
-	var sampleYaml = `a: 
-  b: [{c: dog named jim}, {c: cat named jim}]
-`
-
-	doTest(
-		t, sampleYaml,
-		testProperties{
-			pairs: []keyValuePair{
-				{
-					key:   "a.b.0.c",
-					value: "dog named jim",
-				},
-				{
-					key:   "a.b.1.c",
-					value: "cat named jim",
-				},
-			},
-		},
-		false, true,
-	)
-}
@@ -1,45 +0,0 @@
-package properties
-
-import "github.com/mikefarah/yq/v4/pkg/yqlib"
-
-type PropertiesPreferences struct {
-	UnwrapScalar      bool
-	KeyValueSeparator string
-	UseArrayBrackets  bool
-}
-
-func NewDefaultPropertiesPreferences() PropertiesPreferences {
-	return PropertiesPreferences{
-		UnwrapScalar:      true,
-		KeyValueSeparator: " = ",
-		UseArrayBrackets:  false,
-	}
-}
-
-func (p *PropertiesPreferences) Copy() PropertiesPreferences {
-	return PropertiesPreferences{
-		UnwrapScalar:      p.UnwrapScalar,
-		KeyValueSeparator: p.KeyValueSeparator,
-		UseArrayBrackets:  p.UseArrayBrackets,
-	}
-}
-
-var PropertiesFormat = &yqlib.Format{"props", []string{"p", "properties"}, "properties",
-	func() yqlib.Encoder { return NewPropertiesEncoder(ConfiguredPropertiesPreferences) },
-	func() yqlib.Decoder { return NewPropertiesDecoder() },
-	nil,
-}
-
-var propertyYqRules = []*yqlib.ParticipleYqRule{
-	{"PropertiesDecode", `from_?props|@propsd`, yqlib.CreateDecodeOpYqAction(PropertiesFormat), 0},
-	{"PropsEncode", `to_?props|@props`, yqlib.CreateEncodeOpYqAction(PropertiesFormat, 2), 0},
-	{"LoadProperties", `load_?props`, yqlib.CreateLoadOpYqAction(NewPropertiesDecoder()), 0},
-}
-
-func RegisterPropertiesFormat() {
-	yqlib.RegisterFormat(PropertiesFormat)
-	yqlib.RegisterRules(propertyYqRules)
-
-}
-
-var ConfiguredPropertiesPreferences = NewDefaultPropertiesPreferences()
@@ -1,408 +0,0 @@
-//go:build !yq_noxml
-
-package xml
-
-import (
-	"encoding/xml"
-	"errors"
-	"fmt"
-	"io"
-	"regexp"
-	"strings"
-	"unicode"
-
-	"github.com/mikefarah/yq/v4/pkg/yqlib"
-	"golang.org/x/net/html/charset"
-)
-
-type xmlDecoder struct {
-	reader       io.Reader
-	readAnything bool
-	finished     bool
-	prefs        XmlPreferences
-}
-
-func NewXMLDecoder(prefs XmlPreferences) yqlib.Decoder {
-	return &xmlDecoder{
-		finished: false,
-		prefs:    prefs,
-	}
-}
-
-func (dec *xmlDecoder) Init(reader io.Reader) error {
-	dec.reader = reader
-	dec.readAnything = false
-	dec.finished = false
-	return nil
-}
-
-func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yqlib.CandidateNode, error) {
-	yamlNode := &yqlib.CandidateNode{Kind: yqlib.SequenceNode, Tag: "!!seq"}
-	for _, child := range nodes {
-		yamlChild, err := dec.convertToYamlNode(child)
-		if err != nil {
-			return nil, err
-		}
-		yamlNode.AddChild(yamlChild)
-	}
-
-	return yamlNode, nil
-}
-
-var decoderCommentPrefix = regexp.MustCompile(`(^|\n)([[:alpha:]])`)
-
-func (dec *xmlDecoder) processComment(c string) string {
-	if c == "" {
-		return ""
-	}
-	//need to replace "cat " with "# cat"
-	// "\ncat\n" with "\n cat\n"
-	// ensure non-empty comments starting with newline have a space in front
-
-	replacement := decoderCommentPrefix.ReplaceAllString(c, "$1 $2")
-	replacement = "#" + strings.ReplaceAll(strings.TrimRight(replacement, " "), "\n", "\n#")
-	return replacement
-}
-
-func (dec *xmlDecoder) createMap(n *xmlNode) (*yqlib.CandidateNode, error) {
-	yqlib.GetLogger().Debug("createMap: headC: %v, lineC: %v, footC: %v", n.HeadComment, n.LineComment, n.FootComment)
-	yamlNode := &yqlib.CandidateNode{Kind: yqlib.MappingNode, Tag: "!!map"}
-
-	if len(n.Data) > 0 {
-		yqlib.GetLogger().Debugf("creating content node for map: %v", dec.prefs.ContentName)
-		label := dec.prefs.ContentName
-		labelNode := yqlib.CreateScalarNode(label, label)
-		labelNode.HeadComment = dec.processComment(n.HeadComment)
-		labelNode.LineComment = dec.processComment(n.LineComment)
-		labelNode.FootComment = dec.processComment(n.FootComment)
-		yamlNode.AddKeyValueChild(labelNode, dec.createValueNodeFromData(n.Data))
-	}
-
-	for i, keyValuePair := range n.Children {
-		label := keyValuePair.K
-		children := keyValuePair.V
-		labelNode := yqlib.CreateScalarNode(label, label)
-		var valueNode *yqlib.CandidateNode
-		var err error
-
-		if i == 0 {
-			yqlib.GetLogger().Debugf("head comment here")
-			labelNode.HeadComment = dec.processComment(n.HeadComment)
-
-		}
-		yqlib.GetLogger().Debugf("label=%v, i=%v, keyValuePair.FootComment: %v", label, i, keyValuePair.FootComment)
-		labelNode.FootComment = dec.processComment(keyValuePair.FootComment)
-
-		yqlib.GetLogger().Debug("len of children in %v is %v", label, len(children))
-		if len(children) > 1 {
-			valueNode, err = dec.createSequence(children)
-			if err != nil {
-				return nil, err
-			}
-		} else {
-			// comment hack for maps of scalars
-			// if the value is a scalar, the head comment of the scalar needs to go on the key?
-			// add tests for <z/> as well as multiple <ds> of inputXmlWithComments > yaml
-			if len(children[0].Children) == 0 && children[0].HeadComment != "" {
-				if len(children[0].Data) > 0 {
-
-					yqlib.GetLogger().Debug("scalar comment hack, currentlabel [%v]", labelNode.HeadComment)
-					labelNode.HeadComment = joinComments([]string{labelNode.HeadComment, strings.TrimSpace(children[0].HeadComment)}, "\n")
-					children[0].HeadComment = ""
-				} else {
-					// child is null, put the headComment as a linecomment for reasons
-					children[0].LineComment = children[0].HeadComment
-					children[0].HeadComment = ""
-				}
-			}
-			valueNode, err = dec.convertToYamlNode(children[0])
-			if err != nil {
-				return nil, err
-			}
-		}
-		yamlNode.AddKeyValueChild(labelNode, valueNode)
-	}
-
-	return yamlNode, nil
-}
-
-func (dec *xmlDecoder) createValueNodeFromData(values []string) *yqlib.CandidateNode {
-	switch len(values) {
-	case 0:
-		return yqlib.CreateScalarNode(nil, "")
-	case 1:
-		return yqlib.CreateScalarNode(values[0], values[0])
-	default:
-		content := make([]*yqlib.CandidateNode, 0)
-		for _, value := range values {
-			content = append(content, yqlib.CreateScalarNode(value, value))
-		}
-		return &yqlib.CandidateNode{
-			Kind:    yqlib.SequenceNode,
-			Tag:     "!!seq",
-			Content: content,
-		}
-	}
-}
-
-func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yqlib.CandidateNode, error) {
-	if len(n.Children) > 0 {
-		return dec.createMap(n)
-	}
-
-	scalar := dec.createValueNodeFromData(n.Data)
-
-	yqlib.GetLogger().Debug("scalar (%v), headC: %v, lineC: %v, footC: %v", scalar.Tag, n.HeadComment, n.LineComment, n.FootComment)
-	scalar.HeadComment = dec.processComment(n.HeadComment)
-	scalar.LineComment = dec.processComment(n.LineComment)
-	if scalar.Tag == "!!seq" {
-		scalar.Content[0].HeadComment = scalar.LineComment
-		scalar.LineComment = ""
-	}
-
-	scalar.FootComment = dec.processComment(n.FootComment)
-
-	return scalar, nil
-}
-
-func (dec *xmlDecoder) Decode() (*yqlib.CandidateNode, error) {
-	if dec.finished {
-		return nil, io.EOF
-	}
-	root := &xmlNode{}
-	// cant use xj - it doesn't keep map order.
-	err := dec.decodeXML(root)
-
-	if err != nil {
-		return nil, err
-	}
-	firstNode, err := dec.convertToYamlNode(root)
-
-	if err != nil {
-		return nil, err
-	} else if firstNode.Tag == "!!null" {
-		dec.finished = true
-		if dec.readAnything {
-			return nil, io.EOF
-		}
-	}
-	dec.readAnything = true
-	dec.finished = true
-
-	return firstNode, nil
-}
-
-type xmlNode struct {
-	Children    []*xmlChildrenKv
-	HeadComment string
-	FootComment string
-	LineComment string
-	Data        []string
-}
-
-type xmlChildrenKv struct {
-	K           string
-	V           []*xmlNode
-	FootComment string
-}
-
-// AddChild appends a node to the list of children
-func (n *xmlNode) AddChild(s string, c *xmlNode) {
-
-	if n.Children == nil {
-		n.Children = make([]*xmlChildrenKv, 0)
-	}
-	yqlib.GetLogger().Debug("looking for %s", s)
-	// see if we can find an existing entry to add to
-	for _, childEntry := range n.Children {
-		if childEntry.K == s {
-			yqlib.GetLogger().Debug("found it, appending an entry%s", s)
-			childEntry.V = append(childEntry.V, c)
-			yqlib.GetLogger().Debug("yay len of children in %v is %v", s, len(childEntry.V))
-			return
-		}
-	}
-	yqlib.GetLogger().Debug("not there, making a new one %s", s)
-	n.Children = append(n.Children, &xmlChildrenKv{K: s, V: []*xmlNode{c}})
-}
-
-type element struct {
-	parent *element
-	n      *xmlNode
-	label  string
-	state  string
-}
-
-// this code is heavily based on https://github.com/basgys/goxml2json
-// main changes are to decode into a structure that preserves the original order
-// of the map keys.
-func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
-	xmlDec := xml.NewDecoder(dec.reader)
-	xmlDec.Strict = dec.prefs.StrictMode
-	// That will convert the charset if the provided XML is non-UTF-8
-	xmlDec.CharsetReader = charset.NewReaderLabel
-
-	started := false
-
-	// Create first element from the root node
-	elem := &element{
-		parent: nil,
-		n:      root,
-	}
-
-	getToken := func() (xml.Token, error) {
-		if dec.prefs.UseRawToken {
-			return xmlDec.RawToken()
-		}
-		return xmlDec.Token()
-	}
-
-	for {
-		t, e := getToken()
-		if e != nil && !errors.Is(e, io.EOF) {
-			return e
-		}
-		if t == nil {
-			break
-		}
-
-		switch se := t.(type) {
-		case xml.StartElement:
-			yqlib.GetLogger().Debug("start element %v", se.Name.Local)
-			elem.state = "started"
-			// Build new a new current element and link it to its parent
-			elem = &element{
-				parent: elem,
-				n:      &xmlNode{},
-				label:  se.Name.Local,
-			}
-
-			// Extract attributes as children
-			for _, a := range se.Attr {
-				if dec.prefs.KeepNamespace {
-					if a.Name.Space != "" {
-						a.Name.Local = a.Name.Space + ":" + a.Name.Local
-					}
-				}
-				elem.n.AddChild(dec.prefs.AttributePrefix+a.Name.Local, &xmlNode{Data: []string{a.Value}})
-			}
-		case xml.CharData:
-
-			// Extract XML data (if any)
-			newBit := trimNonGraphic(string(se))
-			if !started && len(newBit) > 0 {
-				return fmt.Errorf("invalid XML: Encountered chardata [%v] outside of XML node", newBit)
-			}
-
-			if len(newBit) > 0 {
-				elem.n.Data = append(elem.n.Data, newBit)
-				elem.state = "chardata"
-				yqlib.GetLogger().Debug("chardata [%v] for %v", elem.n.Data, elem.label)
-			}
-		case xml.EndElement:
-			if elem == nil {
-				yqlib.GetLogger().Debug("no element, probably bad xml")
-				continue
-			}
-			yqlib.GetLogger().Debug("end element %v", elem.label)
-			elem.state = "finished"
-			// And add it to its parent list
-			if elem.parent != nil {
-				elem.parent.n.AddChild(elem.label, elem.n)
-			}
-
-			// Then change the current element to its parent
-			elem = elem.parent
-		case xml.Comment:
-
-			commentStr := string(xml.CharData(se))
-			if elem.state == "started" {
-				applyFootComment(elem, commentStr)
-
-			} else if elem.state == "chardata" {
-				yqlib.GetLogger().Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
-				elem.n.LineComment = joinComments([]string{elem.n.LineComment, commentStr}, " ")
-			} else {
-				yqlib.GetLogger().Debug("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
-				elem.n.HeadComment = joinComments([]string{elem.n.HeadComment, commentStr}, " ")
-			}
-
-		case xml.ProcInst:
-			if !dec.prefs.SkipProcInst {
-				elem.n.AddChild(dec.prefs.ProcInstPrefix+se.Target, &xmlNode{Data: []string{string(se.Inst)}})
-			}
-		case xml.Directive:
-			if !dec.prefs.SkipDirectives {
-				elem.n.AddChild(dec.prefs.DirectiveName, &xmlNode{Data: []string{string(se)}})
-			}
-		}
-		started = true
-	}
-
-	return nil
-}
-
-func applyFootComment(elem *element, commentStr string) {
-
-	// first lets try to put the comment on the last child
-	if len(elem.n.Children) > 0 {
-		lastChildIndex := len(elem.n.Children) - 1
-		childKv := elem.n.Children[lastChildIndex]
-		yqlib.GetLogger().Debug("got a foot comment, putting on last child for %v: [%v]", childKv.K, commentStr)
-		// if it's an array of scalars, put the foot comment on the scalar itself
-		if len(childKv.V) > 0 && len(childKv.V[0].Children) == 0 {
-			nodeToUpdate := childKv.V[len(childKv.V)-1]
-			nodeToUpdate.FootComment = joinComments([]string{nodeToUpdate.FootComment, commentStr}, " ")
-		} else {
-			childKv.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ")
-		}
-	} else {
-		yqlib.GetLogger().Debug("got a foot comment for %v: [%v]", elem.label, commentStr)
-		elem.n.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ")
-	}
-}
-
-func joinComments(rawStrings []string, joinStr string) string {
-	stringsToJoin := make([]string, 0)
-	for _, str := range rawStrings {
-		if str != "" {
-			stringsToJoin = append(stringsToJoin, str)
-		}
-	}
-	return strings.Join(stringsToJoin, joinStr)
-}
-
-// trimNonGraphic returns a slice of the string s, with all leading and trailing
-// non graphic characters and spaces removed.
-//
-// Graphic characters include letters, marks, numbers, punctuation, symbols,
-// and spaces, from categories L, M, N, P, S, Zs.
-// Spacing characters are set by category Z and property Pattern_White_Space.
-func trimNonGraphic(s string) string {
-	if s == "" {
-		return s
-	}
-
-	var first *int
-	var last int
-	for i, r := range []rune(s) {
-		if !unicode.IsGraphic(r) || unicode.IsSpace(r) {
-			continue
-		}
-
-		if first == nil {
-			f := i // copy i
-			first = &f
-			last = i
-		} else {
-			last = i
-		}
-	}
-
-	// If first is nil, it means there are no graphic characters
-	if first == nil {
-		return ""
-	}
-
-	return string([]rune(s)[*first : last+1])
-}
@@ -1,337 +0,0 @@
-//go:build !yq_noxml
-
-package xml
-
-import (
-	"encoding/xml"
-	"fmt"
-	"io"
-	"regexp"
-	"strings"
-
-	"github.com/mikefarah/yq/v4/pkg/yqlib"
-)
-
-type xmlEncoder struct {
-	indentString   string
-	writer         io.Writer
-	prefs          XmlPreferences
-	leadingContent string
-}
-
-func NewXMLEncoder(prefs XmlPreferences) yqlib.Encoder {
-	var indentString = ""
-
-	for index := 0; index < prefs.Indent; index++ {
-		indentString = indentString + " "
-	}
-	return &xmlEncoder{indentString, nil, prefs, ""}
-}
-
-func (e *xmlEncoder) CanHandleAliases() bool {
-	return false
-}
-
-func (e *xmlEncoder) PrintDocumentSeparator(_ io.Writer) error {
-	return nil
-}
-
-func (e *xmlEncoder) PrintLeadingContent(_ io.Writer, content string) error {
-	e.leadingContent = content
-	return nil
-}
-
-func (e *xmlEncoder) Encode(writer io.Writer, node *yqlib.CandidateNode) error {
-	encoder := xml.NewEncoder(writer)
-	// hack so we can manually add newlines to procInst and directives
-	e.writer = writer
-	encoder.Indent("", e.indentString)
-	var newLine xml.CharData = []byte("\n")
-
-	if node.Tag == "!!map" {
-		// make sure <?xml .. ?> processing instructions are encoded first
-		for i := 0; i < len(node.Content); i += 2 {
-			key := node.Content[i]
-			value := node.Content[i+1]
-
-			if key.Value == (e.prefs.ProcInstPrefix + "xml") {
-				name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1)
-				procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)}
-				if err := encoder.EncodeToken(procInst); err != nil {
-					return err
-				}
-				if _, err := e.writer.Write([]byte("\n")); err != nil {
-					yqlib.GetLogger().Warning("Unable to write newline, skipping: %w", err)
-				}
-			}
-		}
-	}
-
-	if e.leadingContent != "" {
-
-		// remove first and last newlines if present
-		err := e.encodeComment(encoder, e.leadingContent)
-		if err != nil {
-			return err
-		}
-		err = encoder.EncodeToken(newLine)
-		if err != nil {
-			return err
-		}
-	}
-
-	switch node.Kind {
-	case yqlib.MappingNode:
-		err := e.encodeTopLevelMap(encoder, node)
-		if err != nil {
-			return err
-		}
-	case yqlib.ScalarNode:
-		var charData xml.CharData = []byte(node.Value)
-		err := encoder.EncodeToken(charData)
-		if err != nil {
-			return err
-		}
-		return encoder.Flush()
-	default:
-		return fmt.Errorf("cannot encode %v to XML - only maps can be encoded", node.Tag)
-	}
-
-	return encoder.EncodeToken(newLine)
-
-}
-
-func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yqlib.CandidateNode) error {
-	err := e.encodeComment(encoder, node.CleanHeadAndLineComment())
-	if err != nil {
-		return err
-	}
-	for i := 0; i < len(node.Content); i += 2 {
-		key := node.Content[i]
-		value := node.Content[i+1]
-
-		start := xml.StartElement{Name: xml.Name{Local: key.Value}}
-		yqlib.GetLogger().Debugf("comments of key %v", key.Value)
-		err := e.encodeComment(encoder, key.CleanHeadAndLineComment())
-		if err != nil {
-			return err
-		}
-		if key.CleanHeadAndLineComment() != "" {
-			var newLine xml.CharData = []byte("\n")
-			err = encoder.EncodeToken(newLine)
-			if err != nil {
-				return err
-			}
-		}
-
-		if key.Value == (e.prefs.ProcInstPrefix + "xml") { //nolint
-			// dont double process these.
-		} else if strings.HasPrefix(key.Value, e.prefs.ProcInstPrefix) {
-			name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1)
-			procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)}
-			if err := encoder.EncodeToken(procInst); err != nil {
-				return err
-			}
-			if _, err := e.writer.Write([]byte("\n")); err != nil {
-				yqlib.GetLogger().Warning("Unable to write newline, skipping: %w", err)
-			}
-		} else if key.Value == e.prefs.DirectiveName {
-			var directive xml.Directive = []byte(value.Value)
-			if err := encoder.EncodeToken(directive); err != nil {
-				return err
-			}
-			if _, err := e.writer.Write([]byte("\n")); err != nil {
-				yqlib.GetLogger().Warning("Unable to write newline, skipping: %w", err)
-			}
-		} else {
-
-			yqlib.GetLogger().Debugf("recursing")
-
-			err = e.doEncode(encoder, value, start)
-			if err != nil {
-				return err
-			}
-		}
-		err = e.encodeComment(encoder, key.CleanFootComment())
-		if err != nil {
-			return err
-		}
-	}
-	return e.encodeComment(encoder, node.CleanFootComment())
-}
-
-func (e *xmlEncoder) encodeStart(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
-	err := encoder.EncodeToken(start)
-	if err != nil {
-		return err
-	}
-	return e.encodeComment(encoder, node.CleanHeadComment())
-}
-
-func (e *xmlEncoder) encodeEnd(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
-	err := encoder.EncodeToken(start.End())
-	if err != nil {
-		return err
-	}
-	return e.encodeComment(encoder, node.CleanFootComment())
-}
-
-func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
-	switch node.Kind {
-	case yqlib.MappingNode:
-		return e.encodeMap(encoder, node, start)
-	case yqlib.SequenceNode:
-		return e.encodeArray(encoder, node, start)
-	case yqlib.ScalarNode:
-		err := e.encodeStart(encoder, node, start)
-		if err != nil {
-			return err
-		}
-
-		var charData xml.CharData = []byte(node.Value)
-		err = encoder.EncodeToken(charData)
-		if err != nil {
-			return err
-		}
-
-		if err = e.encodeComment(encoder, node.CleanLineComment()); err != nil {
-			return err
-		}
-
-		return e.encodeEnd(encoder, node, start)
-	}
-	return fmt.Errorf("unsupported type %v", node.Tag)
-}
-
-var xmlEncodeMultilineCommentRegex = regexp.MustCompile(`(^|\n) *# ?(.*)`)
-var xmlEncodeSingleLineCommentRegex = regexp.MustCompile(`^\s*#(.*)\n?`)
-var chompRegexp = regexp.MustCompile(`\n$`)
-
-func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) error {
-	if commentStr != "" {
-		yqlib.GetLogger().Debugf("got comment [%v]", commentStr)
-		// multi line string
-		if len(commentStr) > 2 && strings.Contains(commentStr[1:len(commentStr)-1], "\n") {
-			commentStr = chompRegexp.ReplaceAllString(commentStr, "")
-			yqlib.GetLogger().Debugf("chompRegexp [%v]", commentStr)
-			commentStr = xmlEncodeMultilineCommentRegex.ReplaceAllString(commentStr, "$1$2")
-			yqlib.GetLogger().Debugf("processed multiline [%v]", commentStr)
-			// if the first line is non blank, add a space
-			if commentStr[0] != '\n' && commentStr[0] != ' ' {
-				commentStr = " " + commentStr
-			}
-
-		} else {
-			commentStr = xmlEncodeSingleLineCommentRegex.ReplaceAllString(commentStr, "$1")
-		}
-
-		if !strings.HasSuffix(commentStr, " ") && !strings.HasSuffix(commentStr, "\n") {
-			commentStr = commentStr + " "
-			yqlib.GetLogger().Debugf("added suffix [%v]", commentStr)
-		}
-		yqlib.GetLogger().Debugf("encoding comment [%v]", commentStr)
-
-		var comment xml.Comment = []byte(commentStr)
-		err := encoder.EncodeToken(comment)
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
-
-	if err := e.encodeComment(encoder, node.CleanHeadAndLineComment()); err != nil {
-		return err
-	}
-
-	for i := 0; i < len(node.Content); i++ {
-		value := node.Content[i]
-		if err := e.doEncode(encoder, value, start.Copy()); err != nil {
-			return err
-		}
-	}
-	return e.encodeComment(encoder, node.CleanFootComment())
-}
-
-func (e *xmlEncoder) isAttribute(name string) bool {
-	return strings.HasPrefix(name, e.prefs.AttributePrefix) &&
-		name != e.prefs.ContentName &&
-		name != e.prefs.DirectiveName &&
-		!strings.HasPrefix(name, e.prefs.ProcInstPrefix)
-}
-
-func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
-	yqlib.GetLogger().Debug("its a map")
-
-	//first find all the attributes and put them on the start token
-	for i := 0; i < len(node.Content); i += 2 {
-		key := node.Content[i]
-		value := node.Content[i+1]
-
-		if e.isAttribute(key.Value) {
-			if value.Kind == yqlib.ScalarNode {
-				attributeName := strings.Replace(key.Value, e.prefs.AttributePrefix, "", 1)
-				start.Attr = append(start.Attr, xml.Attr{Name: xml.Name{Local: attributeName}, Value: value.Value})
-			} else {
-				return fmt.Errorf("cannot use %v as attribute, only scalars are supported", value.Tag)
-			}
-		}
-	}
-
-	err := e.encodeStart(encoder, node, start)
-	if err != nil {
-		return err
-	}
-
-	//now we encode non attribute tokens
-	for i := 0; i < len(node.Content); i += 2 {
-		key := node.Content[i]
-		value := node.Content[i+1]
-
-		err := e.encodeComment(encoder, key.CleanHeadAndLineComment())
-		if err != nil {
-			return err
-		}
-		if strings.HasPrefix(key.Value, e.prefs.ProcInstPrefix) {
-			name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1)
-			procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)}
-			if err := encoder.EncodeToken(procInst); err != nil {
-				return err
-			}
-		} else if key.Value == e.prefs.DirectiveName {
-			var directive xml.Directive = []byte(value.Value)
-			if err := encoder.EncodeToken(directive); err != nil {
-				return err
-			}
-		} else if key.Value == e.prefs.ContentName {
-			// directly encode the contents
-			err = e.encodeComment(encoder, value.CleanHeadAndLineComment())
-			if err != nil {
-				return err
-			}
-			var charData xml.CharData = []byte(value.Value)
-			err = encoder.EncodeToken(charData)
-			if err != nil {
-				return err
-			}
-			err = e.encodeComment(encoder, value.CleanFootComment())
-			if err != nil {
-				return err
-			}
-		} else if !e.isAttribute(key.Value) {
-			start := xml.StartElement{Name: xml.Name{Local: key.Value}}
-			err := e.doEncode(encoder, value, start)
-			if err != nil {
-				return err
-			}
-		}
-		err = e.encodeComment(encoder, key.CleanFootComment())
-		if err != nil {
-			return err
-		}
-	}
-
-	return e.encodeEnd(encoder, node, start)
-}
@@ -1,72 +0,0 @@
-package xml
-
-import "github.com/mikefarah/yq/v4/pkg/yqlib"
-
-type XmlPreferences struct {
-	Indent          int
-	AttributePrefix string
-	ContentName     string
-	StrictMode      bool
-	KeepNamespace   bool
-	UseRawToken     bool
-	ProcInstPrefix  string
-	DirectiveName   string
-	SkipProcInst    bool
-	SkipDirectives  bool
-}
-
-func NewDefaultXmlPreferences() XmlPreferences {
-	return XmlPreferences{
-		Indent:          2,
-		AttributePrefix: "+@",
-		ContentName:     "+content",
-		StrictMode:      false,
-		KeepNamespace:   true,
-		UseRawToken:     true,
-		ProcInstPrefix:  "+p_",
-		DirectiveName:   "+directive",
-		SkipProcInst:    false,
-		SkipDirectives:  false,
-	}
-}
-
-func (p *XmlPreferences) Copy() XmlPreferences {
-	return XmlPreferences{
-		Indent:          p.Indent,
-		AttributePrefix: p.AttributePrefix,
-		ContentName:     p.ContentName,
-		StrictMode:      p.StrictMode,
-		KeepNamespace:   p.KeepNamespace,
-		UseRawToken:     p.UseRawToken,
-		ProcInstPrefix:  p.ProcInstPrefix,
-		DirectiveName:   p.DirectiveName,
-		SkipProcInst:    p.SkipProcInst,
-		SkipDirectives:  p.SkipDirectives,
-	}
-}
-
-var ConfiguredXMLPreferences = NewDefaultXmlPreferences()
-
-var XMLFormat = &yqlib.Format{"xml", []string{"x"}, "xml",
-	func() yqlib.Encoder { return NewXMLEncoder(ConfiguredXMLPreferences) },
-	func() yqlib.Decoder { return NewXMLDecoder(ConfiguredXMLPreferences) },
-	func(indent int) yqlib.Encoder {
-		prefs := ConfiguredXMLPreferences.Copy()
-		prefs.Indent = indent
-		return NewXMLEncoder(prefs)
-	},
-}
-
-var xmlYqRules = []*yqlib.ParticipleYqRule{
-	{"XMLEncodeWithIndent", `to_?xml\([0-9]+\)`, yqlib.CreateEncodeYqActionParsingIndent(XMLFormat), 0},
-	{"XmlDecode", `from_?xml|@xmld`, yqlib.CreateDecodeOpYqAction(XMLFormat), 0},
-	{"XMLEncode", `to_?xml`, yqlib.CreateEncodeOpYqAction(XMLFormat, 2), 0},
-	{"XMLEncodeNoIndent", `@xml`, yqlib.CreateEncodeOpYqAction(XMLFormat, 0), 0},
-	{"LoadXML", `load_?xml|xml_?load`, yqlib.CreateLoadOpYqAction(NewXMLDecoder(ConfiguredXMLPreferences)), 0},
-}
-
-func RegisterXmlFormat() {
-	yqlib.RegisterFormat(XMLFormat)
-	yqlib.RegisterRules(xmlYqRules)
-
-}
@@ -1,827 +0,0 @@
-//go:build !yq_noxml
-
-package xml
-
-import (
-	"bufio"
-	"fmt"
-	"testing"
-
-	"github.com/mikefarah/yq/v4/pkg/yqlib"
-	"github.com/mikefarah/yq/v4/test"
-)
-
-const yamlInputWithProcInstAndHeadComment = `# cats
-+p_xml: version="1.0"
-this: is some xml`
-
-const expectedXmlProcInstAndHeadComment = `<?xml version="1.0"?>
-<!-- cats -->
-<this>is some xml</this>
-`
-
-const xmlProcInstAndHeadCommentBlock = `<?xml version="1.0"?>
-<!--
-cats
-->
-<this>is some xml</this>
-`
-
-const expectedYamlProcInstAndHeadCommentBlock = `#
-# cats
-#
-+p_xml: version="1.0"
-this: is some xml
-`
-
-const inputXMLWithComments = `
-<!-- before cat -->
-<cat>
-	<!-- in cat before -->
-	<x>3<!-- multi
-line comment 
-for x --></x>
-	<!-- before y -->
-	<y>
-		<!-- in y before -->
-		<d><!-- in d before -->z<!-- in d after --></d>
-		
-		<!-- in y after -->
-	</y>
-	<!-- in_cat_after -->
-</cat>
-<!-- after cat -->
-`
-const inputXMLWithCommentsWithSubChild = `
-<!-- before cat -->
-<cat>
-	<!-- in cat before -->
-	<x>3<!-- multi
-line comment 
-for x --></x>
-	<!-- before y -->
-	<y>
-		<!-- in y before -->
-		<d><!-- in d before --><z sweet="cool"/><!-- in d after --></d>
-		
-		<!-- in y after -->
-	</y>
-	<!-- in_cat_after -->
-</cat>
-<!-- after cat -->
-`
-
-const expectedDecodeYamlWithSubChild = `# before cat
-cat:
-    # in cat before
-    x: "3" # multi
-    # line comment 
-    # for x
-    # before y
-
-    y:
-        # in y before
-        d:
-            # in d before
-            z:
-                +@sweet: cool
-            # in d after
-        # in y after
-    # in_cat_after
-# after cat
-`
-
-const inputXMLWithCommentsWithArray = `
-<!-- before cat -->
-<cat>
-	<!-- in cat before -->
-	<x>3<!-- multi
-line comment 
-for x --></x>
-	<!-- before y -->
-	<y>
-		<!-- in y before -->
-		<d><!-- in d before --><z sweet="cool"/><!-- in d after --></d>
-        <d><!-- in d2 before --><z sweet="cool2"/><!-- in d2 after --></d>
-		
-		<!-- in y after -->
-	</y>
-	<!-- in_cat_after -->
-</cat>
-<!-- after cat -->
-`
-
-const expectedDecodeYamlWithArray = `# before cat
-cat:
-    # in cat before
-    x: "3" # multi
-    # line comment 
-    # for x
-    # before y
-
-    y:
-        # in y before
-        d:
-            - # in d before
-              z:
-                +@sweet: cool
-              # in d after
-            - # in d2 before
-              z:
-                +@sweet: cool2
-              # in d2 after
-        # in y after
-    # in_cat_after
-# after cat
-`
-
-const expectedDecodeYamlWithComments = `# before cat
-cat:
-    # in cat before
-    x: "3" # multi
-    # line comment 
-    # for x
-    # before y
-
-    y:
-        # in y before
-        # in d before
-        d: z # in d after
-        # in y after
-    # in_cat_after
-# after cat
-`
-
-const expectedRoundtripXMLWithComments = `<!-- before cat -->
-<cat><!-- in cat before -->
-  <x>3<!-- multi
-line comment 
-for x --></x><!-- before y -->
-  <y><!-- in y before
-in d before -->
-    <d>z<!-- in d after --></d><!-- in y after -->
-  </y><!-- in_cat_after -->
-</cat><!-- after cat -->
-`
-
-const yamlWithComments = `#
-# header comment
-# above_cat
-#
-cat: # inline_cat
-  # above_array
-  array: # inline_array
-    - val1 # inline_val1
-    # above_val2
-    - val2 # inline_val2
-# below_cat
-`
-
-const expectedXMLWithComments = `<!--
-header comment
-above_cat
-->
-<!-- inline_cat -->
-<cat><!-- above_array inline_array -->
-  <array>val1<!-- inline_val1 --></array>
-  <array><!-- above_val2 -->val2<!-- inline_val2 --></array>
-</cat><!-- below_cat -->
-`
-
-const inputXMLWithNamespacedAttr = `<?xml version="1.0"?>
-<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url"></map>
-`
-
-const expectedYAMLWithNamespacedAttr = `+p_xml: version="1.0"
-map:
-  +@xmlns: some-namespace
-  +@xmlns:xsi: some-instance
-  +@xsi:schemaLocation: some-url
-`
-
-const expectedYAMLWithRawNamespacedAttr = `+p_xml: version="1.0"
-map:
-  +@xmlns: some-namespace
-  +@xmlns:xsi: some-instance
-  +@xsi:schemaLocation: some-url
-`
-
-const expectedYAMLWithoutRawNamespacedAttr = `+p_xml: version="1.0"
-map:
-  +@xmlns: some-namespace
-  +@xmlns:xsi: some-instance
-  +@some-instance:schemaLocation: some-url
-`
-
-const xmlWithCustomDtd = `
-<?xml version="1.0"?>
-<!DOCTYPE root [
-<!ENTITY writer "Blah.">
-<!ENTITY copyright "Blah">
-]>
-<root>
-    <item>&writer;&copyright;</item>
-</root>`
-
-const expectedDtd = `<?xml version="1.0"?>
-<!DOCTYPE root [
-<!ENTITY writer "Blah.">
-<!ENTITY copyright "Blah">
-]>
-<root>
-  <item>&amp;writer;&amp;copyright;</item>
-</root>
-`
-
-const expectedSkippedDtd = `<?xml version="1.0"?>
-<root>
-  <item>&amp;writer;&amp;copyright;</item>
-</root>
-`
-
-const xmlWithProcInstAndDirectives = `<?xml version="1.0"?>
-<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
-<apple>
-  <?coolioo version="1.0"?>
-  <!CATYPE meow purr puss >
-  <b>things</b>
-</apple>
-`
-
-const yamlWithProcInstAndDirectives = `+p_xml: version="1.0"
-+directive: 'DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" '
-apple:
-  +p_coolioo: version="1.0"
-  +directive: 'CATYPE meow purr puss '
-  b: things
-`
-
-const expectedXmlWithProcInstAndDirectives = `<?xml version="1.0"?>
-<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
-<apple><?coolioo version="1.0"?><!CATYPE meow purr puss >
-  <b>things</b>
-</apple>
-`
-
-var xmlScenarios = []yqlib.FormatScenario{
-	{
-		skipDoc:     true,
-		description: "bad xml",
-		input:       `<?xml version="1.0" encoding="UTF-8"?></Child></Root>`,
-		expected:    "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\n",
-	},
-	{
-		skipDoc:  true,
-		input:    "  <root>value<!-- comment--> </root>",
-		expected: "root: value # comment\n",
-	},
-	{
-		skipDoc:       true,
-		input:         "value<root>value</root>",
-		expectedError: "bad file 'sample.yml': invalid XML: Encountered chardata [value] outside of XML node",
-		scenarioType:  "decode-error",
-	},
-	{
-		skipDoc:  true,
-		input:    "<root><!-- comment-->value</root>",
-		expected: "# comment\nroot: value\n",
-	},
-	{
-		skipDoc:  true,
-		input:    "<root> <!-- comment--></root>",
-		expected: "root: # comment\n",
-	},
-	{
-		skipDoc:  true,
-		input:    "<root>value<!-- comment-->anotherValue </root>",
-		expected: "root:\n    # comment\n    - value\n    - anotherValue\n",
-	},
-	{
-		skipDoc:  true,
-		input:    "<root><cats><cat>quick</cat><cat>soft</cat><!-- kitty_comment--><cat>squishy</cat></cats></root>",
-		expected: "root:\n    cats:\n        cat:\n            - quick\n            - soft\n            # kitty_comment\n\n            - squishy\n",
-	},
-	{
-		description:  "ProcInst with head comment",
-		skipDoc:      true,
-		input:        yamlInputWithProcInstAndHeadComment,
-		expected:     expectedXmlProcInstAndHeadComment,
-		scenarioType: "encode",
-	},
-	{
-		description:  "Scalar roundtrip",
-		skipDoc:      true,
-		input:        "<mike>cat</mike>",
-		expression:   ".mike",
-		expected:     "cat",
-		scenarioType: "roundtrip",
-	},
-	{
-		description:  "ProcInst with head comment round trip",
-		skipDoc:      true,
-		input:        expectedXmlProcInstAndHeadComment,
-		expected:     expectedXmlProcInstAndHeadComment,
-		scenarioType: "roundtrip",
-	},
-	{
-		description:  "ProcInst with block head comment to yaml",
-		skipDoc:      true,
-		input:        xmlProcInstAndHeadCommentBlock,
-		expected:     expectedYamlProcInstAndHeadCommentBlock,
-		scenarioType: "decode",
-	},
-	{
-		description:  "ProcInst with block head comment from yaml",
-		skipDoc:      true,
-		input:        expectedYamlProcInstAndHeadCommentBlock,
-		expected:     xmlProcInstAndHeadCommentBlock,
-		scenarioType: "encode",
-	},
-	{
-		description:  "ProcInst with head comment round trip block",
-		skipDoc:      true,
-		input:        xmlProcInstAndHeadCommentBlock,
-		expected:     xmlProcInstAndHeadCommentBlock,
-		scenarioType: "roundtrip",
-	},
-	{
-		description:    "Parse xml: simple",
-		subdescription: "Notice how all the values are strings, see the next example on how you can fix that.",
-		input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat>\n  <says>meow</says>\n  <legs>4</legs>\n  <cute>true</cute>\n</cat>",
-		expected:       "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n    says: meow\n    legs: \"4\"\n    cute: \"true\"\n",
-	},
-	{
-		description:    "Parse xml: number",
-		subdescription: "All values are assumed to be strings when parsing XML, but you can use the `from_yaml` operator on all the strings values to autoparse into the correct type.",
-		input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat>\n  <says>meow</says>\n  <legs>4</legs>\n  <cute>true</cute>\n</cat>",
-		expression:     " (.. | select(tag == \"!!str\")) |= from_yaml",
-		expected:       "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n    says: meow\n    legs: 4\n    cute: true\n",
-	},
-	{
-		description:    "Parse xml: array",
-		subdescription: "Consecutive nodes with identical xml names are assumed to be arrays.",
-		input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<animal>cat</animal>\n<animal>goat</animal>",
-		expected:       "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\nanimal:\n    - cat\n    - goat\n",
-	},
-	{
-		description:    "Parse xml: force as an array",
-		subdescription: "In XML, if your array has a single item, then yq doesn't know its an array. This is how you can consistently force it to be an array. This handles the 3 scenarios of having nothing in the array, having a single item and having multiple.",
-		input:          "<zoo><animal>cat</animal></zoo>",
-		expression:     ".zoo.animal |= ([] + .)",
-		expected:       "zoo:\n    animal:\n        - cat\n",
-	},
-	{
-		description: "Parse xml: force all as an array",
-		input:       "<zoo><thing><frog>boing</frog></thing></zoo>",
-		expression:  ".. |= [] + .",
-		expected:    "- zoo:\n    - thing:\n        - frog:\n            - boing\n",
-	},
-	{
-		description:    "Parse xml: attributes",
-		subdescription: "Attributes are converted to fields, with the default attribute prefix '+'. Use '--xml-attribute-prefix` to set your own.",
-		input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">\n  <legs>7</legs>\n</cat>",
-		expected:       "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n    +@legs: \"4\"\n    legs: \"7\"\n",
-	},
-	{
-		description:    "Parse xml: attributes with content",
-		subdescription: "Content is added as a field, using the default content name of `+content`. Use `--xml-content-name` to set your own.",
-		input:          "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<cat legs=\"4\">meow</cat>",
-		expected:       "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\ncat:\n    +content: meow\n    +@legs: \"4\"\n",
-	},
-	{
-		description:    "Parse xml: content split between comments/children",
-		subdescription: "Multiple content texts are collected into a sequence.",
-		input:          "<root>  value  <!-- comment-->anotherValue <a>frog</a> cool!</root>",
-		expected:       "root:\n    +content: # comment\n        - value\n        - anotherValue\n        - cool!\n    a: frog\n",
-	},
-	{
-		description:    "Parse xml: custom dtd",
-		subdescription: "DTD entities are processed as directives.",
-		input:          xmlWithCustomDtd,
-		expected:       expectedDtd,
-		scenarioType:   "roundtrip",
-	},
-	{
-		description:  "Roundtrip with name spaced attributes",
-		skipDoc:      true,
-		input:        inputXMLWithNamespacedAttr,
-		expected:     inputXMLWithNamespacedAttr,
-		scenarioType: "roundtrip",
-	},
-	{
-		description:    "Parse xml: skip custom dtd",
-		subdescription: "DTDs are directives, skip over directives to skip DTDs.",
-		input:          xmlWithCustomDtd,
-		expected:       expectedSkippedDtd,
-		scenarioType:   "roundtrip-skip-directives",
-	},
-	{
-		description:    "Parse xml: with comments",
-		subdescription: "A best attempt is made to preserve comments.",
-		input:          inputXMLWithComments,
-		expected:       expectedDecodeYamlWithComments,
-		scenarioType:   "decode",
-	},
-	{
-		description:  "Empty doc",
-		skipDoc:      true,
-		input:        "",
-		expected:     "\n",
-		scenarioType: "decode",
-	},
-	{
-		description:  "Empty single node",
-		skipDoc:      true,
-		input:        "<a/>",
-		expected:     "a:\n",
-		scenarioType: "decode",
-	},
-	{
-		description:  "Empty close node",
-		skipDoc:      true,
-		input:        "<a></a>",
-		expected:     "a:\n",
-		scenarioType: "decode",
-	},
-	{
-		description:  "Nested empty",
-		skipDoc:      true,
-		input:        "<a><b/></a>",
-		expected:     "a:\n    b:\n",
-		scenarioType: "decode",
-	},
-	{
-		description:  "Parse xml: with comments subchild",
-		skipDoc:      true,
-		input:        inputXMLWithCommentsWithSubChild,
-		expected:     expectedDecodeYamlWithSubChild,
-		scenarioType: "decode",
-	},
-	{
-		description:  "Parse xml: with comments array",
-		skipDoc:      true,
-		input:        inputXMLWithCommentsWithArray,
-		expected:     expectedDecodeYamlWithArray,
-		scenarioType: "decode",
-	},
-	{
-		description:    "Parse xml: keep attribute namespace",
-		subdescription: fmt.Sprintf(`Defaults to %v`, ConfiguredXMLPreferences.KeepNamespace),
-		skipDoc:        false,
-		input:          inputXMLWithNamespacedAttr,
-		expected:       expectedYAMLWithNamespacedAttr,
-		scenarioType:   "decode-keep-ns",
-	},
-	{
-		description:  "Parse xml: keep raw attribute namespace",
-		skipDoc:      true,
-		input:        inputXMLWithNamespacedAttr,
-		expected:     expectedYAMLWithRawNamespacedAttr,
-		scenarioType: "decode-raw-token",
-	},
-	{
-		description:    "Parse xml: keep raw attribute namespace",
-		subdescription: fmt.Sprintf(`Defaults to %v`, ConfiguredXMLPreferences.UseRawToken),
-		skipDoc:        false,
-		input:          inputXMLWithNamespacedAttr,
-		expected:       expectedYAMLWithoutRawNamespacedAttr,
-		scenarioType:   "decode-raw-token-off",
-	},
-	{
-		description:  "Encode xml: simple",
-		input:        "cat: purrs",
-		expected:     "<cat>purrs</cat>\n",
-		scenarioType: "encode",
-	},
-	{
-		description:  "includes map tags",
-		skipDoc:      true,
-		input:        "<cat>purrs</cat>\n",
-		expression:   `tag`,
-		expected:     "!!map\n",
-		scenarioType: "decode",
-	},
-	{
-		description:  "includes array tags",
-		skipDoc:      true,
-		input:        "<cat>purrs</cat><cat>purrs</cat>\n",
-		expression:   `.cat | tag`,
-		expected:     "!!seq\n",
-		scenarioType: "decode",
-	},
-	{
-		description:  "Encode xml: array",
-		input:        "pets:\n  cat:\n    - purrs\n    - meows",
-		expected:     "<pets>\n  <cat>purrs</cat>\n  <cat>meows</cat>\n</pets>\n",
-		scenarioType: "encode",
-	},
-	{
-		description:    "Encode xml: attributes",
-		subdescription: "Fields with the matching xml-attribute-prefix are assumed to be attributes.",
-		input:          "cat:\n  +@name: tiger\n  meows: true\n",
-		expected:       "<cat name=\"tiger\">\n  <meows>true</meows>\n</cat>\n",
-		scenarioType:   "encode",
-	},
-	{
-		description:  "double prefix",
-		skipDoc:      true,
-		input:        "cat:\n  +@+@name: tiger\n  meows: true\n",
-		expected:     "<cat +@name=\"tiger\">\n  <meows>true</meows>\n</cat>\n",
-		scenarioType: "encode",
-	},
-	{
-		description:   "arrays cannot be encoded",
-		skipDoc:       true,
-		input:         "[cat, dog, fish]",
-		expectedError: "cannot encode !!seq to XML - only maps can be encoded",
-		scenarioType:  "encode-error",
-	},
-	{
-		description:   "arrays cannot be encoded - 2",
-		skipDoc:       true,
-		input:         "[cat, dog]",
-		expectedError: "cannot encode !!seq to XML - only maps can be encoded",
-		scenarioType:  "encode-error",
-	},
-	{
-		description:    "Encode xml: attributes with content",
-		subdescription: "Fields with the matching xml-content-name is assumed to be content.",
-		input:          "cat:\n  +@name: tiger\n  +content: cool\n",
-		expected:       "<cat name=\"tiger\">cool</cat>\n",
-		scenarioType:   "encode",
-	},
-	{
-		description:  "round trip multiline 1",
-		skipDoc:      true,
-		input:        "<x><!-- cats --></x>\n",
-		expected:     "<x><!-- cats --></x>\n",
-		scenarioType: "roundtrip",
-	},
-	{
-		description:  "round trip multiline 2",
-		skipDoc:      true,
-		input:        "<x><!--\n cats\n --></x>\n",
-		expected:     "<x><!--\ncats\n--></x>\n",
-		scenarioType: "roundtrip",
-	},
-	{
-		description:  "round trip multiline 3",
-		skipDoc:      true,
-		input:        "<x><!--\n\tcats\n --></x>\n",
-		expected:     "<x><!--\n\tcats\n--></x>\n",
-		scenarioType: "roundtrip",
-	},
-	{
-		description:  "round trip multiline 4",
-		skipDoc:      true,
-		input:        "<x><!--\n\tcats\n\tdogs\n--></x>\n",
-		expected:     "<x><!--\n\tcats\n\tdogs\n--></x>\n",
-		scenarioType: "roundtrip",
-	},
-	{
-		description:  "round trip multiline 5",
-		skipDoc:      true, // pity spaces aren't kept atm.
-		input:        "<x><!--\ncats\ndogs\n--></x>\n",
-		expected:     "<x><!--\ncats\ndogs\n--></x>\n",
-		scenarioType: "roundtrip",
-	},
-	{
-		description:    "Encode xml: comments",
-		subdescription: "A best attempt is made to copy comments to xml.",
-		input:          yamlWithComments,
-		expected:       expectedXMLWithComments,
-		scenarioType:   "encode",
-	},
-	{
-		description:    "Encode: doctype and xml declaration",
-		subdescription: "Use the special xml names to add/modify proc instructions and directives.",
-		input:          yamlWithProcInstAndDirectives,
-		expected:       expectedXmlWithProcInstAndDirectives,
-		scenarioType:   "encode",
-	},
-	{
-		description:    "Round trip: with comments",
-		subdescription: "A best effort is made, but comment positions and white space are not preserved perfectly.",
-		input:          inputXMLWithComments,
-		expected:       expectedRoundtripXMLWithComments,
-		scenarioType:   "roundtrip",
-	},
-	{
-		description:    "Roundtrip: with doctype and declaration",
-		subdescription: "yq parses XML proc instructions and directives into nodes.\nUnfortunately the underlying XML parser loses whitespace information.",
-		input:          xmlWithProcInstAndDirectives,
-		expected:       expectedXmlWithProcInstAndDirectives,
-		scenarioType:   "roundtrip",
-	},
-}
-
-func testXMLScenario(t *testing.T, s yqlib.FormatScenario) {
-	switch s.scenarioType {
-	case "", "decode":
-		yamlPrefs := yqlib.ConfiguredYamlPreferences.Copy()
-		yamlPrefs.Indent = 4
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(yamlPrefs)), s.description)
-	case "encode":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(ConfiguredXMLPreferences)), s.description)
-	case "roundtrip":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(ConfiguredXMLPreferences)), s.description)
-	case "decode-keep-ns":
-		prefs := NewDefaultXmlPreferences()
-		prefs.KeepNamespace = true
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
-	case "decode-raw-token":
-		prefs := NewDefaultXmlPreferences()
-		prefs.UseRawToken = true
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
-	case "decode-raw-token-off":
-		prefs := NewDefaultXmlPreferences()
-		prefs.UseRawToken = false
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
-	case "roundtrip-skip-directives":
-		prefs := NewDefaultXmlPreferences()
-		prefs.SkipDirectives = true
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs)), s.description)
-	case "decode-error":
-		result, err := processFormatScenario(s, NewXMLDecoder(NewDefaultXmlPreferences()), NewYamlEncoder(ConfiguredYamlPreferences))
-		if err == nil {
-			t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result)
-		} else {
-			test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description)
-		}
-	case "encode-error":
-		result, err := processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(NewDefaultXmlPreferences()))
-		if err == nil {
-			t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result)
-		} else {
-			test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description)
-		}
-
-	default:
-		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
-	}
-}
-
-func documentXMLScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(FormatScenario)
-
-	if s.skipDoc {
-		return
-	}
-	switch s.scenarioType {
-	case "", "decode":
-		documentXMLDecodeScenario(w, s)
-	case "encode":
-		documentXMLEncodeScenario(w, s)
-	case "roundtrip":
-		documentXMLRoundTripScenario(w, s)
-	case "decode-keep-ns":
-		documentXMLDecodeKeepNsScenario(w, s)
-	case "decode-raw-token-off":
-		documentXMLDecodeKeepNsRawTokenScenario(w, s)
-	case "roundtrip-skip-directives":
-		documentXMLSkipDirectivesScenario(w, s)
-
-	default:
-		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
-	}
-}
-
-func documentXMLDecodeScenario(w *bufio.Writer, s FormatScenario) {
-	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
-
-	if s.subdescription != "" {
-		writeOrPanic(w, s.subdescription)
-		writeOrPanic(w, "\n\n")
-	}
-
-	writeOrPanic(w, "Given a sample.xml file of:\n")
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input))
-
-	writeOrPanic(w, "then\n")
-	expression := s.expression
-	if expression == "" {
-		expression = "."
-	}
-	writeOrPanic(w, fmt.Sprintf("```bash\nyq -oy '%v' sample.xml\n```\n", expression))
-	writeOrPanic(w, "will output\n")
-
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(ConfiguredYamlPreferences))))
-}
-
-func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s FormatScenario) {
-	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
-
-	if s.subdescription != "" {
-		writeOrPanic(w, s.subdescription)
-		writeOrPanic(w, "\n\n")
-	}
-
-	writeOrPanic(w, "Given a sample.xml file of:\n")
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input))
-
-	writeOrPanic(w, "then\n")
-	writeOrPanic(w, "```bash\nyq --xml-keep-namespace=false '.' sample.xml\n```\n")
-	writeOrPanic(w, "will output\n")
-	prefs := NewDefaultXmlPreferences()
-	prefs.KeepNamespace = false
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs))))
-
-	prefsWithout := NewDefaultXmlPreferences()
-	prefs.KeepNamespace = true
-	writeOrPanic(w, "instead of\n")
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(prefsWithout))))
-}
-
-func documentXMLDecodeKeepNsRawTokenScenario(w *bufio.Writer, s FormatScenario) {
-	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
-
-	if s.subdescription != "" {
-		writeOrPanic(w, s.subdescription)
-		writeOrPanic(w, "\n\n")
-	}
-
-	writeOrPanic(w, "Given a sample.xml file of:\n")
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input))
-
-	writeOrPanic(w, "then\n")
-	writeOrPanic(w, "```bash\nyq --xml-raw-token=false '.' sample.xml\n```\n")
-	writeOrPanic(w, "will output\n")
-
-	prefs := NewDefaultXmlPreferences()
-	prefs.UseRawToken = false
-
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs))))
-
-	prefsWithout := NewDefaultXmlPreferences()
-	prefsWithout.UseRawToken = true
-
-	writeOrPanic(w, "instead of\n")
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefsWithout), NewXMLEncoder(prefsWithout))))
-}
-
-func documentXMLEncodeScenario(w *bufio.Writer, s FormatScenario) {
-	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
-
-	if s.subdescription != "" {
-		writeOrPanic(w, s.subdescription)
-		writeOrPanic(w, "\n\n")
-	}
-
-	writeOrPanic(w, "Given a sample.yml file of:\n")
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input))
-
-	writeOrPanic(w, "then\n")
-	writeOrPanic(w, "```bash\nyq -o=xml sample.yml\n```\n")
-	writeOrPanic(w, "will output\n")
-
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewXMLEncoder(ConfiguredXMLPreferences))))
-}
-
-func documentXMLRoundTripScenario(w *bufio.Writer, s FormatScenario) {
-	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
-
-	if s.subdescription != "" {
-		writeOrPanic(w, s.subdescription)
-		writeOrPanic(w, "\n\n")
-	}
-
-	writeOrPanic(w, "Given a sample.xml file of:\n")
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input))
-
-	writeOrPanic(w, "then\n")
-	writeOrPanic(w, "```bash\nyq '.' sample.xml\n```\n")
-	writeOrPanic(w, "will output\n")
-
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(ConfiguredXMLPreferences))))
-}
-
-func documentXMLSkipDirectivesScenario(w *bufio.Writer, s FormatScenario) {
-	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
-
-	if s.subdescription != "" {
-		writeOrPanic(w, s.subdescription)
-		writeOrPanic(w, "\n\n")
-	}
-
-	writeOrPanic(w, "Given a sample.xml file of:\n")
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input))
-
-	writeOrPanic(w, "then\n")
-	writeOrPanic(w, "```bash\nyq --xml-skip-directives '.' sample.xml\n```\n")
-	writeOrPanic(w, "will output\n")
-	prefs := NewDefaultXmlPreferences()
-	prefs.SkipDirectives = true
-
-	writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(prefs), NewXMLEncoder(prefs))))
-}
-
-func TestXMLScenarios(t *testing.T) {
-	for _, tt := range xmlScenarios {
-		testXMLScenario(t, tt)
-	}
-	genericScenarios := make([]interface{}, len(xmlScenarios))
-	for i, s := range xmlScenarios {
-		genericScenarios[i] = s
-	}
-	documentScenarios(t, "usage", "xml", genericScenarios, documentXMLScenario)
-}
@@ -2,14 +2,16 @@ package yqlib

 import (
 	"container/list"
+
+	yaml "gopkg.in/yaml.v3"
 )

 // A yaml expression evaluator that runs the expression once against all files/nodes in memory.
 type Evaluator interface {
-	EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error
+	EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error

 	// EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes
-	EvaluateNodes(expression string, nodes ...*CandidateNode) (*list.List, error)
+	EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error)

 	// EvaluateCandidateNodes takes an expression and list of candidate nodes, returning a list of matching candidate nodes
 	EvaluateCandidateNodes(expression string, inputCandidateNodes *list.List) (*list.List, error)
@@ -24,10 +26,10 @@ func NewAllAtOnceEvaluator() Evaluator {
 	return &allAtOnceEvaluator{treeNavigator: NewDataTreeNavigator()}
 }

-func (e *allAtOnceEvaluator) EvaluateNodes(expression string, nodes ...*CandidateNode) (*list.List, error) {
+func (e *allAtOnceEvaluator) EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error) {
 	inputCandidates := list.New()
 	for _, node := range nodes {
-		inputCandidates.PushBack(node)
+		inputCandidates.PushBack(&CandidateNode{Node: node})
 	}
 	return e.EvaluateCandidateNodes(expression, inputCandidates)
 }
@@ -44,16 +46,21 @@ func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCand
 	return context.MatchingNodes, nil
 }

-func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error {
+func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error {
 	fileIndex := 0
+	firstFileLeadingContent := ""

 	var allDocuments = list.New()
 	for _, filename := range filenames {
-		reader, err := readStream(filename)
+		reader, leadingContent, err := readStream(filename, fileIndex == 0 && leadingContentPreProcessing)
 		if err != nil {
 			return err
 		}

+		if fileIndex == 0 {
+			firstFileLeadingContent = leadingContent
+		}
+
 		fileDocuments, err := readDocuments(reader, filename, fileIndex, decoder)
 		if err != nil {
 			return err
@@ -63,8 +70,16 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
 	}

 	if allDocuments.Len() == 0 {
-		candidateNode := CreateScalarNode(nil, "")
+		candidateNode := &CandidateNode{
+			Document:       0,
+			Filename:       "",
+			Node:           &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
+			FileIndex:      0,
+			LeadingContent: firstFileLeadingContent,
+		}
 		allDocuments.PushBack(candidateNode)
+	} else {
+		allDocuments.Front().Value.(*CandidateNode).LeadingContent = firstFileLeadingContent
 	}

 	matches, err := e.EvaluateCandidateNodes(expression, allDocuments)
@@ -1,8 +1,6 @@
 package yqlib

 import (
-	"bufio"
-	"strings"
 	"testing"

 	"github.com/mikefarah/yq/v4/test"
@@ -20,7 +18,7 @@ var evaluateNodesScenario = []expressionScenario{
 		document:   `a: hello`,
 		expression: `.`,
 		expected: []string{
-			"D0, P[], (!!map)::a: hello\n",
+			"D0, P[], (doc)::a: hello\n",
 		},
 	},
 	{
@@ -34,23 +32,9 @@ var evaluateNodesScenario = []expressionScenario{

 func TestAllAtOnceEvaluateNodes(t *testing.T) {
 	var evaluator = NewAllAtOnceEvaluator()
-	// logging.SetLevel(logging.DEBUG, "")
 	for _, tt := range evaluateNodesScenario {
-		decoder := NewYamlDecoder(ConfiguredYamlPreferences)
-		reader := bufio.NewReader(strings.NewReader(tt.document))
-		err := decoder.Init(reader)
-		if err != nil {
-			t.Error(err)
-			return
-		}
-		candidateNode, errorReading := decoder.Decode()
-
-		if errorReading != nil {
-			t.Error(errorReading)
-			return
-		}
-
-		list, _ := evaluator.EvaluateNodes(tt.expression, candidateNode)
+		node := test.ParseData(tt.document)
+		list, _ := evaluator.EvaluateNodes(tt.expression, &node)
 		test.AssertResultComplex(t, tt.expected, resultsToString(t, list))
 	}
 }
@@ -3,189 +3,51 @@ package yqlib
 import (
 	"container/list"
 	"fmt"
-	"strconv"
 	"strings"
+
+	"github.com/jinzhu/copier"
+	yaml "gopkg.in/yaml.v3"
 )

-type Kind uint32
-
-const (
-	SequenceNode Kind = 1 << iota
-	MappingNode
-	ScalarNode
-	AliasNode
-)
-
-type Style uint32
-
-const (
-	TaggedStyle Style = 1 << iota
-	DoubleQuotedStyle
-	SingleQuotedStyle
-	LiteralStyle
-	FoldedStyle
-	FlowStyle
-)
-
-func CreateStringScalarNode(stringValue string) *CandidateNode {
-	var node = &CandidateNode{Kind: ScalarNode}
-	node.Value = stringValue
-	node.Tag = "!!str"
-	return node
-}
-
-func CreateScalarNode(value interface{}, stringValue string) *CandidateNode {
-	var node = &CandidateNode{Kind: ScalarNode}
-	node.Value = stringValue
-
-	switch value.(type) {
-	case float32, float64:
-		node.Tag = "!!float"
-	case int, int64, int32:
-		node.Tag = "!!int"
-	case bool:
-		node.Tag = "!!bool"
-	case string:
-		node.Tag = "!!str"
-	case nil:
-		node.Tag = "!!null"
-	}
-	return node
-}
-
 type CandidateNode struct {
-	Kind  Kind
-	Style Style
-
-	Tag     string
-	Value   string
-	Anchor  string
-	Alias   *CandidateNode
-	Content []*CandidateNode
-
-	HeadComment string
-	LineComment string
-	FootComment string
-
+	Node   *yaml.Node     // the actual node
 	Parent *CandidateNode // parent node
-	Key    *CandidateNode // node key, if this is a value from a map (or index in an array)
+	Key    *yaml.Node     // node key, if this is a value from a map (or index in an array)

-	LeadingContent string
+	LeadingContent  string
+	TrailingContent string

-	document uint // the document index of this node
-	filename string
-
-	Line   int
-	Column int
-
-	fileIndex int
+	Path      []interface{} /// the path we took to get to this node
+	Document  uint          // the document index of this node
+	Filename  string
+	FileIndex int
 	// when performing op against all nodes given, this will treat all the nodes as one
 	// (e.g. top level cross document merge). This property does not propagate to child nodes.
 	EvaluateTogether bool
 	IsMapKey         bool
 }

-func (n *CandidateNode) CreateChild() *CandidateNode {
-	return &CandidateNode{
-		Parent: n,
-	}
-}
-
-func (n *CandidateNode) SetDocument(idx uint) {
-	n.document = idx
-}
-
-func (n *CandidateNode) GetDocument() uint {
-	// defer to parent
-	if n.Parent != nil {
-		return n.Parent.GetDocument()
-	}
-	return n.document
-}
-
-func (n *CandidateNode) SetFilename(name string) {
-	n.filename = name
-}
-
-func (n *CandidateNode) GetFilename() string {
-	if n.Parent != nil {
-		return n.Parent.GetFilename()
-	}
-	return n.filename
-}
-
-func (n *CandidateNode) SetFileIndex(idx int) {
-	n.fileIndex = idx
-}
-
-func (n *CandidateNode) GetFileIndex() int {
-	if n.Parent != nil {
-		return n.Parent.GetFileIndex()
-	}
-	return n.fileIndex
-}
-
 func (n *CandidateNode) GetKey() string {
 	keyPrefix := ""
 	if n.IsMapKey {
-		keyPrefix = fmt.Sprintf("key-%v-", n.Value)
+		keyPrefix = "key-"
 	}
-	key := ""
-	if n.Key != nil {
-		key = n.Key.Value
-	}
-	return fmt.Sprintf("%v%v - %v", keyPrefix, n.GetDocument(), key)
+	return fmt.Sprintf("%v%v - %v", keyPrefix, n.Document, n.Path)
 }

-func (n *CandidateNode) getParsedKey() interface{} {
-	if n.IsMapKey {
-		return n.Value
-	}
-	if n.Key == nil {
-		return nil
-	}
-	if n.Key.Tag == "!!str" {
-		return n.Key.Value
-	}
-	index, err := parseInt(n.Key.Value)
-	if err != nil {
-		return n.Key.Value
-	}
-	return index
-
-}
-
-func (n *CandidateNode) GetPath() []interface{} {
-	key := n.getParsedKey()
-	if n.Parent != nil && key != nil {
-		return append(n.Parent.GetPath(), key)
-	}
-
-	if key != nil {
-		return []interface{}{key}
-	}
-	return make([]interface{}, 0)
+func (n *CandidateNode) GetNiceTag() string {
+	return unwrapDoc(n.Node).Tag
 }

 func (n *CandidateNode) GetNicePath() string {
-	var sb strings.Builder
-	path := n.GetPath()
-	for i, element := range path {
-		elementStr := fmt.Sprintf("%v", element)
-		switch element.(type) {
-		case int:
-			sb.WriteString("[" + elementStr + "]")
-		default:
-			if i == 0 {
-				sb.WriteString(elementStr)
-			} else if strings.ContainsRune(elementStr, '.') {
-				sb.WriteString("[" + elementStr + "]")
-			} else {
-				sb.WriteString("." + elementStr)
-			}
+	if n.Path != nil && len(n.Path) >= 0 {
+		pathStr := make([]string, len(n.Path))
+		for i, v := range n.Path {
+			pathStr[i] = fmt.Sprintf("%v", v)
 		}
+		return strings.Join(pathStr, ".")
 	}
-	return sb.String()
+	return ""
 }

 func (n *CandidateNode) AsList() *list.List {
@@ -194,268 +56,134 @@ func (n *CandidateNode) AsList() *list.List {
 	return elMap
 }

-func (n *CandidateNode) SetParent(parent *CandidateNode) {
-	n.Parent = parent
-}
-
-func (n *CandidateNode) AddKeyValueChild(rawKey *CandidateNode, rawValue *CandidateNode) (*CandidateNode, *CandidateNode) {
-	key := rawKey.Copy()
-	key.SetParent(n)
-	key.IsMapKey = true
-
-	value := rawValue.Copy()
-	value.SetParent(n)
-	value.IsMapKey = false // force this, incase we are creating a value from a key
-	value.Key = key
-
-	n.Content = append(n.Content, key, value)
-	return key, value
-}
-
-func (n *CandidateNode) AddChild(rawChild *CandidateNode) {
-	value := rawChild.Copy()
-	value.SetParent(n)
-	if value.Key != nil {
-		value.Key.SetParent(n)
-	} else {
-		index := len(n.Content)
-		keyNode := CreateScalarNode(index, fmt.Sprintf("%v", index))
-		keyNode.SetParent(n)
-		value.Key = keyNode
+func (n *CandidateNode) CreateChildInMap(key *yaml.Node, node *yaml.Node) *CandidateNode {
+	var value interface{}
+	if key != nil {
+		value = key.Value
 	}
-	n.Content = append(n.Content, value)
-}
-
-func (n *CandidateNode) AddChildren(children []*CandidateNode) {
-	if n.Kind == MappingNode {
-		for i := 0; i < len(children); i += 2 {
-			key := children[i]
-			value := children[i+1]
-			n.AddKeyValueChild(key, value)
-		}
-
-	} else {
-		for _, rawChild := range children {
-			n.AddChild(rawChild)
-		}
+	return &CandidateNode{
+		Node:      node,
+		Path:      n.createChildPath(value),
+		Parent:    n,
+		Key:       key,
+		Document:  n.Document,
+		Filename:  n.Filename,
+		FileIndex: n.FileIndex,
 	}
 }

-func (n *CandidateNode) GetValueRep() (interface{}, error) {
-	log.Debugf("GetValueRep for %v value: %v", n.GetNicePath(), n.Value)
-	realTag := n.GuessTagFromCustomType()
-
-	switch realTag {
-	case "!!int":
-		_, val, err := parseInt64(n.Value)
-		return val, err
-	case "!!float":
-		// need to test this
-		return strconv.ParseFloat(n.Value, 64)
-	case "!!bool":
-		return isTruthyNode(n), nil
-	case "!!null":
-		return nil, nil
+func (n *CandidateNode) CreateChildInArray(index int, node *yaml.Node) *CandidateNode {
+	return &CandidateNode{
+		Node:      node,
+		Path:      n.createChildPath(index),
+		Parent:    n,
+		Key:       &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%v", index), Tag: "!!int"},
+		Document:  n.Document,
+		Filename:  n.Filename,
+		FileIndex: n.FileIndex,
 	}
-
-	return n.Value, nil
 }

-func (n *CandidateNode) GuessTagFromCustomType() string {
-	if strings.HasPrefix(n.Tag, "!!") {
-		return n.Tag
-	} else if n.Value == "" {
-		log.Debug("GuessTagFromCustomType: node has no value to guess the type with")
-		return n.Tag
+func (n *CandidateNode) CreateReplacement(node *yaml.Node) *CandidateNode {
+	return &CandidateNode{
+		Node:      node,
+		Path:      n.createChildPath(nil),
+		Parent:    n.Parent,
+		Key:       n.Key,
+		IsMapKey:  n.IsMapKey,
+		Document:  n.Document,
+		Filename:  n.Filename,
+		FileIndex: n.FileIndex,
 	}
-	dataBucket, errorReading := parseSnippet(n.Value)
-
-	if errorReading != nil {
-		log.Debug("GuessTagFromCustomType: could not guess underlying tag type %v", errorReading)
-		return n.Tag
-	}
-	guessedTag := dataBucket.Tag
-	log.Info("im guessing the tag %v is a %v", n.Tag, guessedTag)
-	return guessedTag
 }

-func (n *CandidateNode) CreateReplacement(kind Kind, tag string, value string) *CandidateNode {
-	node := &CandidateNode{
-		Kind:  kind,
-		Tag:   tag,
-		Value: value,
-	}
-	return n.CopyAsReplacement(node)
-}
-
-func (n *CandidateNode) CopyAsReplacement(replacement *CandidateNode) *CandidateNode {
-	newCopy := replacement.Copy()
-	newCopy.Parent = n.Parent
-
-	if n.IsMapKey {
-		newCopy.Key = n
-	} else {
-		newCopy.Key = n.Key
-	}
-
-	return newCopy
-}
-
-func (n *CandidateNode) CreateReplacementWithComments(kind Kind, tag string, style Style) *CandidateNode {
-	replacement := n.CreateReplacement(kind, tag, "")
+func (n *CandidateNode) CreateReplacementWithDocWrappers(node *yaml.Node) *CandidateNode {
+	replacement := n.CreateReplacement(node)
 	replacement.LeadingContent = n.LeadingContent
-	replacement.HeadComment = n.HeadComment
-	replacement.LineComment = n.LineComment
-	replacement.FootComment = n.FootComment
-	replacement.Style = style
+	replacement.TrailingContent = n.TrailingContent
 	return replacement
 }

-func (n *CandidateNode) Copy() *CandidateNode {
-	return n.doCopy(true)
+func (n *CandidateNode) createChildPath(path interface{}) []interface{} {
+	if path == nil {
+		newPath := make([]interface{}, len(n.Path))
+		copy(newPath, n.Path)
+		return newPath
+	}
+
+	//don't use append as they may actually modify the path of the orignal node!
+	newPath := make([]interface{}, len(n.Path)+1)
+	copy(newPath, n.Path)
+	newPath[len(n.Path)] = path
+	return newPath
 }

-func (n *CandidateNode) CopyWithoutContent() *CandidateNode {
-	return n.doCopy(false)
-}
-
-func (n *CandidateNode) doCopy(cloneContent bool) *CandidateNode {
-	var content []*CandidateNode
-
-	var copyKey *CandidateNode
-	if n.Key != nil {
-		copyKey = n.Key.Copy()
+func (n *CandidateNode) Copy() (*CandidateNode, error) {
+	clone := &CandidateNode{}
+	err := copier.Copy(clone, n)
+	if err != nil {
+		return nil, err
 	}
-
-	clone := &CandidateNode{
-		Kind:  n.Kind,
-		Style: n.Style,
-
-		Tag:    n.Tag,
-		Value:  n.Value,
-		Anchor: n.Anchor,
-
-		// ok not to clone this,
-		// as its a reference to somewhere else.
-		Alias:   n.Alias,
-		Content: content,
-
-		HeadComment: n.HeadComment,
-		LineComment: n.LineComment,
-		FootComment: n.FootComment,
-
-		Parent: n.Parent,
-		Key:    copyKey,
-
-		LeadingContent: n.LeadingContent,
-
-		document:  n.document,
-		filename:  n.filename,
-		fileIndex: n.fileIndex,
-
-		Line:   n.Line,
-		Column: n.Column,
-
-		EvaluateTogether: n.EvaluateTogether,
-		IsMapKey:         n.IsMapKey,
-	}
-
-	if cloneContent {
-		clone.AddChildren(n.Content)
-	}
-
-	return clone
+	clone.Node = deepClone(n.Node)
+	return clone, nil
 }

 // updates this candidate from the given candidate node
 func (n *CandidateNode) UpdateFrom(other *CandidateNode, prefs assignPreferences) {
-	if n == other {
-		log.Debugf("UpdateFrom, no need to update from myself.")
-		return
-	}
+
 	// if this is an empty map or empty array, use the style of other node.
-	if (n.Kind != ScalarNode && len(n.Content) == 0) ||
+	if (n.Node.Kind != yaml.ScalarNode && len(n.Node.Content) == 0) ||
 		// if the tag has changed (e.g. from str to bool)
-		(n.GuessTagFromCustomType() != other.GuessTagFromCustomType()) {
-		n.Style = other.Style
+		(guessTagFromCustomType(n.Node) != guessTagFromCustomType(other.Node)) {
+		n.Node.Style = other.Node.Style
 	}

-	n.Content = make([]*CandidateNode, 0)
-	n.Kind = other.Kind
-	n.AddChildren(other.Content)
-
-	n.Value = other.Value
+	n.Node.Content = deepCloneContent(other.Node.Content)
+	n.Node.Kind = other.Node.Kind
+	n.Node.Value = other.Node.Value

 	n.UpdateAttributesFrom(other, prefs)

 }

 func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignPreferences) {
-	log.Debug("UpdateAttributesFrom: n: %v other: %v", NodeToString(n), NodeToString(other))
-	if n.Kind != other.Kind {
+	log.Debug("UpdateAttributesFrom: n: %v other: %v", n.GetKey(), other.GetKey())
+	if n.Node.Kind != other.Node.Kind {
 		// clear out the contents when switching to a different type
 		// e.g. map to array
-		n.Content = make([]*CandidateNode, 0)
-		n.Value = ""
+		n.Node.Content = make([]*yaml.Node, 0)
+		n.Node.Value = ""
 	}
-	n.Kind = other.Kind
+	n.Node.Kind = other.Node.Kind

 	// don't clobber custom tags...
-	if prefs.ClobberCustomTags || strings.HasPrefix(n.Tag, "!!") || n.Tag == "" {
-		n.Tag = other.Tag
+	if strings.HasPrefix(n.Node.Tag, "!!") || n.Node.Tag == "" {
+		n.Node.Tag = other.Node.Tag
 	}

-	n.Alias = other.Alias
+	n.Node.Alias = other.Node.Alias

 	if !prefs.DontOverWriteAnchor {
-		n.Anchor = other.Anchor
+		n.Node.Anchor = other.Node.Anchor
 	}

 	// merge will pickup the style of the new thing
 	// when autocreating nodes

-	if n.Style == 0 {
-		n.Style = other.Style
+	if n.Node.Style == 0 {
+		n.Node.Style = other.Node.Style
 	}

-	if other.FootComment != "" {
-		n.FootComment = other.FootComment
+	if other.Node.FootComment != "" {
+		n.Node.FootComment = other.Node.FootComment
 	}
-	if other.HeadComment != "" {
-		n.HeadComment = other.HeadComment
+	if other.TrailingContent != "" {
+		n.TrailingContent = other.TrailingContent
 	}
-	if other.LineComment != "" {
-		n.LineComment = other.LineComment
+	if other.Node.HeadComment != "" {
+		n.Node.HeadComment = other.Node.HeadComment
 	}
-}
-
-func (n *CandidateNode) CleanHeadAndLineComment() string {
-	return n.CleanHeadComment() + n.CleanLineComment()
-}
-
-func (n *CandidateNode) CleanHeadComment() string {
-	return strings.Replace(n.HeadComment, "#", "", 1)
-}
-
-func (n *CandidateNode) CleanLineComment() string {
-	return strings.Replace(n.LineComment, "#", "", 1)
-}
-
-func (n *CandidateNode) CleanFootComment() string {
-	return strings.Replace(n.FootComment, "#", "", 1)
-}
-
-func (n *CandidateNode) ConvertKeysToStrings() {
-
-	if n.Kind == MappingNode {
-		for index, child := range n.Content {
-			if index%2 == 0 { // its a map key
-				child.Tag = "!!str"
-			}
-		}
-	}
-
-	for _, child := range n.Content {
-		child.ConvertKeysToStrings()
+	if other.Node.LineComment != "" {
+		n.Node.LineComment = other.Node.LineComment
 	}
 }
@@ -1,180 +0,0 @@
-package yqlib
-
-import (
-	"fmt"
-	"strings"
-
-	yaml "github.com/goccy/go-yaml"
-	"github.com/goccy/go-yaml/ast"
-	goccyToken "github.com/goccy/go-yaml/token"
-)
-
-func (o *CandidateNode) goccyDecodeIntoChild(childNode ast.Node, cm yaml.CommentMap) (*CandidateNode, error) {
-	newChild := o.CreateChild()
-
-	err := newChild.UnmarshalGoccyYAML(childNode, cm)
-	return newChild, err
-}
-
-func (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap) error {
-	log.Debugf("UnmarshalYAML %v", node)
-	log.Debugf("UnmarshalYAML %v", node.Type().String())
-	log.Debugf("UnmarshalYAML Node Value: %v", node.String())
-	log.Debugf("UnmarshalYAML Node GetComment: %v", node.GetComment())
-
-	if node.GetComment() != nil {
-		commentMapComments := cm[node.GetPath()]
-		for _, comment := range node.GetComment().Comments {
-			// need to use the comment map to find the position :/
-			log.Debugf("%v has a comment of [%v]", node.GetPath(), comment.Token.Value)
-			for _, commentMapComment := range commentMapComments {
-				commentMapValue := strings.Join(commentMapComment.Texts, "\n")
-				if commentMapValue == comment.Token.Value {
-					log.Debug("found a matching entry in comment map")
-					// we found the comment in the comment map,
-					// now we can process the position
-					switch commentMapComment.Position {
-					case yaml.CommentHeadPosition:
-						o.HeadComment = comment.String()
-						log.Debug("its a head comment %v", comment.String())
-					case yaml.CommentLinePosition:
-						o.LineComment = comment.String()
-						log.Debug("its a line comment %v", comment.String())
-					case yaml.CommentFootPosition:
-						o.FootComment = comment.String()
-						log.Debug("its a foot comment %v", comment.String())
-					}
-				}
-			}
-
-		}
-	}
-
-	o.Value = node.String()
-	switch node.Type() {
-	case ast.IntegerType:
-		o.Kind = ScalarNode
-		o.Tag = "!!int"
-	case ast.FloatType:
-		o.Kind = ScalarNode
-		o.Tag = "!!float"
-	case ast.BoolType:
-		o.Kind = ScalarNode
-		o.Tag = "!!bool"
-	case ast.NullType:
-		o.Kind = ScalarNode
-		o.Tag = "!!null"
-		o.Value = node.GetToken().Value
-	case ast.StringType:
-		o.Kind = ScalarNode
-		o.Tag = "!!str"
-		switch node.GetToken().Type {
-		case goccyToken.SingleQuoteType:
-			o.Style = SingleQuotedStyle
-		case goccyToken.DoubleQuoteType:
-			o.Style = DoubleQuotedStyle
-		}
-		o.Value = node.(*ast.StringNode).Value
-		log.Debugf("string value %v", node.(*ast.StringNode).Value)
-	case ast.LiteralType:
-		o.Kind = ScalarNode
-		o.Tag = "!!str"
-		o.Style = LiteralStyle
-		astLiteral := node.(*ast.LiteralNode)
-		log.Debugf("astLiteral.Start.Type %v", astLiteral.Start.Type)
-		if astLiteral.Start.Type == goccyToken.FoldedType {
-			log.Debugf("folded Type %v", astLiteral.Start.Type)
-			o.Style = FoldedStyle
-		}
-		log.Debug("start value: %v ", node.(*ast.LiteralNode).Start.Value)
-		log.Debug("start value: %v ", node.(*ast.LiteralNode).Start.Type)
-		// TODO: here I could put the original value with line breaks
-		// to solve the multiline > problem
-		o.Value = astLiteral.Value.Value
-	case ast.TagType:
-		if err := o.UnmarshalGoccyYAML(node.(*ast.TagNode).Value, cm); err != nil {
-			return err
-		}
-		o.Tag = node.(*ast.TagNode).Start.Value
-	case ast.MappingType:
-		log.Debugf("UnmarshalYAML -  a mapping node")
-		o.Kind = MappingNode
-		o.Tag = "!!map"
-
-		mappingNode := node.(*ast.MappingNode)
-		if mappingNode.IsFlowStyle {
-			o.Style = FlowStyle
-		}
-		for _, mappingValueNode := range mappingNode.Values {
-			err := o.goccyProcessMappingValueNode(mappingValueNode, cm)
-			if err != nil {
-				return ast.ErrInvalidAnchorName
-			}
-		}
-		if mappingNode.FootComment != nil {
-			log.Debugf("mapping node has a foot comment of: %v", mappingNode.FootComment)
-			o.FootComment = mappingNode.FootComment.String()
-		}
-	case ast.MappingValueType:
-		log.Debugf("UnmarshalYAML -  a mapping node")
-		o.Kind = MappingNode
-		o.Tag = "!!map"
-		mappingValueNode := node.(*ast.MappingValueNode)
-		err := o.goccyProcessMappingValueNode(mappingValueNode, cm)
-		if err != nil {
-			return ast.ErrInvalidAnchorName
-		}
-	case ast.SequenceType:
-		log.Debugf("UnmarshalYAML -  a sequence node")
-		o.Kind = SequenceNode
-		o.Tag = "!!seq"
-		sequenceNode := node.(*ast.SequenceNode)
-		if sequenceNode.IsFlowStyle {
-			o.Style = FlowStyle
-		}
-		astSeq := sequenceNode.Values
-		o.Content = make([]*CandidateNode, len(astSeq))
-		for i := 0; i < len(astSeq); i++ {
-			keyNode := o.CreateChild()
-			keyNode.IsMapKey = true
-			keyNode.Tag = "!!int"
-			keyNode.Kind = ScalarNode
-			keyNode.Value = fmt.Sprintf("%v", i)
-
-			valueNode, err := o.goccyDecodeIntoChild(astSeq[i], cm)
-			if err != nil {
-				return err
-			}
-
-			valueNode.Key = keyNode
-			o.Content[i] = valueNode
-		}
-
-	default:
-		log.Debugf("UnmarshalYAML -  node idea of the type!!")
-	}
-	log.Debugf("KIND: %v", o.Kind)
-	return nil
-}
-
-func (o *CandidateNode) goccyProcessMappingValueNode(mappingEntry *ast.MappingValueNode, cm yaml.CommentMap) error {
-	log.Debug("UnmarshalYAML MAP KEY entry %v", mappingEntry.Key)
-	keyNode, err := o.goccyDecodeIntoChild(mappingEntry.Key, cm)
-	if err != nil {
-		return err
-	}
-	keyNode.IsMapKey = true
-
-	log.Debug("UnmarshalYAML MAP VALUE entry %v", mappingEntry.Value)
-	valueNode, err := o.goccyDecodeIntoChild(mappingEntry.Value, cm)
-	if err != nil {
-		return err
-	}
-
-	if mappingEntry.FootComment != nil {
-		valueNode.FootComment = mappingEntry.FootComment.String()
-	}
-	o.AddKeyValueChild(keyNode, valueNode)
-
-	return nil
-}
@@ -1,162 +0,0 @@
-package yqlib
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/mikefarah/yq/v4/test"
-)
-
-type valueRepScenario struct {
-	input    string
-	tag      string
-	expected interface{}
-}
-
-var valueRepScenarios = []valueRepScenario{
-	{
-		input:    `"cat"`,
-		expected: `"cat"`,
-	},
-	{
-		input:    `3`,
-		expected: int64(3),
-	},
-	{
-		input:    `3.1`,
-		expected: float64(3.1),
-	},
-	{
-		input:    `true`,
-		expected: true,
-	},
-	{
-		input:    `y`,
-		tag:      "!!bool",
-		expected: true,
-	},
-	{
-		tag:      "!!null",
-		expected: nil,
-	},
-}
-
-func TestCandidateNodeGetValueRepScenarios(t *testing.T) {
-	for _, tt := range valueRepScenarios {
-		node := CandidateNode{Value: tt.input, Tag: tt.tag}
-		actual, err := node.GetValueRep()
-		if err != nil {
-			t.Error(err)
-			return
-		}
-		test.AssertResult(t, tt.expected, actual)
-	}
-}
-
-func TestCandidateNodeChildWhenParentUpdated(t *testing.T) {
-	parent := CandidateNode{}
-	child := parent.CreateChild()
-	parent.SetDocument(1)
-	parent.SetFileIndex(2)
-	parent.SetFilename("meow")
-	test.AssertResultWithContext(t, "meow", child.GetFilename(), "filename")
-	test.AssertResultWithContext(t, 2, child.GetFileIndex(), "file index")
-	test.AssertResultWithContext(t, uint(1), child.GetDocument(), "document index")
-}
-
-type CreateScalarNodeScenario struct {
-	value       interface{}
-	stringValue string
-	expectedTag string
-}
-
-var createScalarScenarios = []CreateScalarNodeScenario{
-	{
-		value:       "mike",
-		stringValue: "mike",
-		expectedTag: "!!str",
-	},
-	{
-		value:       3,
-		stringValue: "3",
-		expectedTag: "!!int",
-	},
-	{
-		value:       3.1,
-		stringValue: "3.1",
-		expectedTag: "!!float",
-	},
-	{
-		value:       true,
-		stringValue: "true",
-		expectedTag: "!!bool",
-	},
-	{
-		value:       nil,
-		stringValue: "~",
-		expectedTag: "!!null",
-	},
-}
-
-func TestCreateScalarNodeScenarios(t *testing.T) {
-	for _, tt := range createScalarScenarios {
-		actual := CreateScalarNode(tt.value, tt.stringValue)
-		test.AssertResultWithContext(t, tt.stringValue, actual.Value, fmt.Sprintf("Value for: Value: [%v], String: %v", tt.value, tt.stringValue))
-		test.AssertResultWithContext(t, tt.expectedTag, actual.Tag, fmt.Sprintf("Value for: Value: [%v], String: %v", tt.value, tt.stringValue))
-	}
-}
-
-func TestGetKeyForMapValue(t *testing.T) {
-	key := CreateStringScalarNode("yourKey")
-	n := CandidateNode{Key: key, Value: "meow", document: 3}
-	test.AssertResult(t, "3 - yourKey", n.GetKey())
-}
-
-func TestGetKeyForMapKey(t *testing.T) {
-	key := CreateStringScalarNode("yourKey")
-	key.IsMapKey = true
-	key.document = 3
-	test.AssertResult(t, "key-yourKey-3 - ", key.GetKey())
-}
-
-func TestGetKeyForValue(t *testing.T) {
-	n := CandidateNode{Value: "meow", document: 3}
-	test.AssertResult(t, "3 - ", n.GetKey())
-}
-
-func TestGetParsedKeyForMapKey(t *testing.T) {
-	key := CreateStringScalarNode("yourKey")
-	key.IsMapKey = true
-	key.document = 3
-	test.AssertResult(t, "yourKey", key.getParsedKey())
-}
-
-func TestGetParsedKeyForLooseValue(t *testing.T) {
-	n := CandidateNode{Value: "meow", document: 3}
-	test.AssertResult(t, nil, n.getParsedKey())
-}
-
-func TestGetParsedKeyForMapValue(t *testing.T) {
-	key := CreateStringScalarNode("yourKey")
-	n := CandidateNode{Key: key, Value: "meow", document: 3}
-	test.AssertResult(t, "yourKey", n.getParsedKey())
-}
-
-func TestGetParsedKeyForArrayValue(t *testing.T) {
-	key := CreateScalarNode(4, "4")
-	n := CandidateNode{Key: key, Value: "meow", document: 3}
-	test.AssertResult(t, 4, n.getParsedKey())
-}
-
-func TestCandidateNodeAddKeyValueChild(t *testing.T) {
-	key := CandidateNode{Value: "cool", IsMapKey: true}
-	node := CandidateNode{}
-
-	// if we use a key in a new node as a value, it should no longer be marked as a key
-
-	_, keyIsValueNow := node.AddKeyValueChild(&CandidateNode{Value: "newKey"}, &key)
-
-	test.AssertResult(t, keyIsValueNow.IsMapKey, false)
-	test.AssertResult(t, key.IsMapKey, true)
-
-}
@@ -1,218 +0,0 @@
-package yqlib
-
-import (
-	"fmt"
-
-	yaml "gopkg.in/yaml.v3"
-)
-
-func MapYamlStyle(original yaml.Style) Style {
-	switch original {
-	case yaml.TaggedStyle:
-		return TaggedStyle
-	case yaml.DoubleQuotedStyle:
-		return DoubleQuotedStyle
-	case yaml.SingleQuotedStyle:
-		return SingleQuotedStyle
-	case yaml.LiteralStyle:
-		return LiteralStyle
-	case yaml.FoldedStyle:
-		return FoldedStyle
-	case yaml.FlowStyle:
-		return FlowStyle
-	case 0:
-		return 0
-	}
-	return Style(original)
-}
-
-func MapToYamlStyle(original Style) yaml.Style {
-	switch original {
-	case TaggedStyle:
-		return yaml.TaggedStyle
-	case DoubleQuotedStyle:
-		return yaml.DoubleQuotedStyle
-	case SingleQuotedStyle:
-		return yaml.SingleQuotedStyle
-	case LiteralStyle:
-		return yaml.LiteralStyle
-	case FoldedStyle:
-		return yaml.FoldedStyle
-	case FlowStyle:
-		return yaml.FlowStyle
-	case 0:
-		return 0
-	}
-	return yaml.Style(original)
-}
-
-func (o *CandidateNode) copyFromYamlNode(node *yaml.Node, anchorMap map[string]*CandidateNode) {
-	o.Style = MapYamlStyle(node.Style)
-
-	o.Tag = node.Tag
-	o.Value = node.Value
-	o.Anchor = node.Anchor
-
-	if o.Anchor != "" {
-		anchorMap[o.Anchor] = o
-		log.Debug("set anchor %v to %v", o.Anchor, NodeToString(o))
-	}
-
-	// its a single alias
-	if node.Alias != nil && node.Alias.Anchor != "" {
-		o.Alias = anchorMap[node.Alias.Anchor]
-		log.Debug("set alias to %v", NodeToString(anchorMap[node.Alias.Anchor]))
-	}
-	o.HeadComment = node.HeadComment
-	o.LineComment = node.LineComment
-	o.FootComment = node.FootComment
-
-	o.Line = node.Line
-	o.Column = node.Column
-}
-
-func (o *CandidateNode) copyToYamlNode(node *yaml.Node) {
-	node.Style = MapToYamlStyle(o.Style)
-
-	node.Tag = o.Tag
-	node.Value = o.Value
-	node.Anchor = o.Anchor
-
-	// node.Alias = TODO - find Alias in our own structure
-	// might need to be a post process thing
-
-	node.HeadComment = o.HeadComment
-
-	node.LineComment = o.LineComment
-	node.FootComment = o.FootComment
-
-	node.Line = o.Line
-	node.Column = o.Column
-}
-
-func (o *CandidateNode) decodeIntoChild(childNode *yaml.Node, anchorMap map[string]*CandidateNode) (*CandidateNode, error) {
-	newChild := o.CreateChild()
-
-	// null yaml.Nodes to not end up calling UnmarshalYAML
-	// so we call it explicitly
-	if childNode.Tag == "!!null" {
-		newChild.Kind = ScalarNode
-		newChild.copyFromYamlNode(childNode, anchorMap)
-		return newChild, nil
-	}
-
-	err := newChild.UnmarshalYAML(childNode, anchorMap)
-	return newChild, err
-}
-
-func (o *CandidateNode) UnmarshalYAML(node *yaml.Node, anchorMap map[string]*CandidateNode) error {
-	log.Debugf("UnmarshalYAML %v", node.Tag)
-	switch node.Kind {
-	case yaml.AliasNode:
-		log.Debug("UnmarshalYAML - alias from yaml: %v", o.Tag)
-		o.Kind = AliasNode
-		o.copyFromYamlNode(node, anchorMap)
-		return nil
-	case yaml.ScalarNode:
-		log.Debugf("UnmarshalYAML -  a scalar")
-		o.Kind = ScalarNode
-		o.copyFromYamlNode(node, anchorMap)
-		return nil
-	case yaml.MappingNode:
-		log.Debugf("UnmarshalYAML -  a mapping node")
-		o.Kind = MappingNode
-		o.copyFromYamlNode(node, anchorMap)
-		o.Content = make([]*CandidateNode, len(node.Content))
-		for i := 0; i < len(node.Content); i += 2 {
-
-			keyNode, err := o.decodeIntoChild(node.Content[i], anchorMap)
-			if err != nil {
-				return err
-			}
-
-			keyNode.IsMapKey = true
-
-			valueNode, err := o.decodeIntoChild(node.Content[i+1], anchorMap)
-			if err != nil {
-				return err
-			}
-
-			valueNode.Key = keyNode
-
-			o.Content[i] = keyNode
-			o.Content[i+1] = valueNode
-		}
-		log.Debugf("UnmarshalYAML -  finished mapping node")
-		return nil
-	case yaml.SequenceNode:
-		log.Debugf("UnmarshalYAML -  a sequence: %v", len(node.Content))
-		o.Kind = SequenceNode
-
-		o.copyFromYamlNode(node, anchorMap)
-		log.Debugf("node Style: %v", node.Style)
-		log.Debugf("o Style: %v", o.Style)
-		o.Content = make([]*CandidateNode, len(node.Content))
-		for i := 0; i < len(node.Content); i++ {
-			keyNode := o.CreateChild()
-			keyNode.IsMapKey = true
-			keyNode.Tag = "!!int"
-			keyNode.Kind = ScalarNode
-			keyNode.Value = fmt.Sprintf("%v", i)
-
-			valueNode, err := o.decodeIntoChild(node.Content[i], anchorMap)
-			if err != nil {
-				return err
-			}
-
-			valueNode.Key = keyNode
-			o.Content[i] = valueNode
-		}
-		return nil
-	case 0:
-		// not sure when this happens
-		o.copyFromYamlNode(node, anchorMap)
-		log.Debugf("UnmarshalYAML -  err.. %v", NodeToString(o))
-		return nil
-	default:
-		return fmt.Errorf("orderedMap: invalid yaml node")
-	}
-}
-
-func (o *CandidateNode) MarshalYAML() (*yaml.Node, error) {
-	log.Debug("MarshalYAML to yaml: %v", o.Tag)
-	switch o.Kind {
-	case AliasNode:
-		log.Debug("MarshalYAML - alias to yaml: %v", o.Tag)
-		target := &yaml.Node{Kind: yaml.AliasNode}
-		o.copyToYamlNode(target)
-		return target, nil
-	case ScalarNode:
-		log.Debug("MarshalYAML - scalar: %v", o.Value)
-		target := &yaml.Node{Kind: yaml.ScalarNode}
-		o.copyToYamlNode(target)
-		return target, nil
-	case MappingNode, SequenceNode:
-		targetKind := yaml.MappingNode
-		if o.Kind == SequenceNode {
-			targetKind = yaml.SequenceNode
-		}
-		target := &yaml.Node{Kind: targetKind}
-		o.copyToYamlNode(target)
-		log.Debugf("original style: %v", o.Style)
-		log.Debugf("original: %v, tag: %v, style: %v, kind: %v", NodeToString(o), target.Tag, target.Style, target.Kind == yaml.SequenceNode)
-		target.Content = make([]*yaml.Node, len(o.Content))
-		for i := 0; i < len(o.Content); i++ {
-
-			child, err := o.Content[i].MarshalYAML()
-
-			if err != nil {
-				return nil, err
-			}
-			target.Content[i] = child
-		}
-		return target, nil
-	}
-	target := &yaml.Node{}
-	o.copyToYamlNode(target)
-	return target, nil
-}
@@ -1,177 +0,0 @@
-package yqlib
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"io"
-
-	"github.com/goccy/go-json"
-)
-
-func (o *CandidateNode) setScalarFromJson(value interface{}) error {
-	o.Kind = ScalarNode
-	switch rawData := value.(type) {
-	case nil:
-		o.Tag = "!!null"
-		o.Value = "null"
-	case float64, float32:
-		o.Value = fmt.Sprintf("%v", value)
-		o.Tag = "!!float"
-		// json decoder returns ints as float.
-		if value == float64(int64(rawData.(float64))) {
-			// aha it's an int disguised as a float
-			o.Tag = "!!int"
-			o.Value = fmt.Sprintf("%v", int64(value.(float64)))
-		}
-	case int, int64, int32:
-		o.Value = fmt.Sprintf("%v", value)
-		o.Tag = "!!int"
-	case bool:
-		o.Value = fmt.Sprintf("%v", value)
-		o.Tag = "!!bool"
-	case string:
-		o.Value = rawData
-		o.Tag = "!!str"
-	default:
-		return fmt.Errorf("unrecognised type :( %v", rawData)
-	}
-	return nil
-}
-
-func (o *CandidateNode) UnmarshalJSON(data []byte) error {
-	log.Debug("UnmarshalJSON")
-	switch data[0] {
-	case '{':
-		log.Debug("UnmarshalJSON -  its a map!")
-		// its a map
-		o.Kind = MappingNode
-		o.Tag = "!!map"
-
-		dec := json.NewDecoder(bytes.NewReader(data))
-		_, err := dec.Token() // open object
-		if err != nil {
-			return err
-		}
-
-		// cycle through k/v
-		var tok json.Token
-		for tok, err = dec.Token(); err == nil; tok, err = dec.Token() {
-			// we can expect two types: string or Delim. Delim automatically means
-			// that it is the closing bracket of the object, whereas string means
-			// that there is another key.
-			if _, ok := tok.(json.Delim); ok {
-				break
-			}
-
-			childKey := o.CreateChild()
-			childKey.IsMapKey = true
-			childKey.Value = tok.(string)
-			childKey.Kind = ScalarNode
-			childKey.Tag = "!!str"
-
-			childValue := o.CreateChild()
-			childValue.Key = childKey
-
-			if err := dec.Decode(childValue); err != nil {
-				return err
-			}
-			o.Content = append(o.Content, childKey, childValue)
-		}
-		// unexpected error
-		if err != nil && !errors.Is(err, io.EOF) {
-			return err
-		}
-		return nil
-	case '[':
-		o.Kind = SequenceNode
-		o.Tag = "!!seq"
-		log.Debug("UnmarshalJSON -  its an array!")
-		var children []*CandidateNode
-		if err := json.Unmarshal(data, &children); err != nil {
-			return err
-		}
-		// now we put the children into the content, and set a key value for them
-		for i, child := range children {
-
-			if child == nil {
-				// need to represent it as a null scalar
-				child = CreateScalarNode(nil, "null")
-			}
-			childKey := o.CreateChild()
-			childKey.Kind = ScalarNode
-			childKey.Tag = "!!int"
-			childKey.Value = fmt.Sprintf("%v", i)
-			childKey.IsMapKey = true
-
-			child.Parent = o
-			child.Key = childKey
-			o.Content = append(o.Content, child)
-		}
-		return nil
-	}
-	log.Debug("UnmarshalJSON -  its a scalar!")
-	// otherwise, must be a scalar
-	var scalar interface{}
-	err := json.Unmarshal(data, &scalar)
-
-	if err != nil {
-		return err
-	}
-	log.Debug("UnmarshalJSON -  scalar is %v", scalar)
-
-	return o.setScalarFromJson(scalar)
-
-}
-
-func (o *CandidateNode) MarshalJSON() ([]byte, error) {
-	log.Debugf("MarshalJSON %v", NodeToString(o))
-	buf := new(bytes.Buffer)
-	enc := json.NewEncoder(buf)
-	enc.SetIndent("", " ")
-	enc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
-
-	switch o.Kind {
-	case AliasNode:
-		log.Debugf("MarshalJSON AliasNode")
-		err := enc.Encode(o.Alias)
-		return buf.Bytes(), err
-	case ScalarNode:
-		log.Debugf("MarshalJSON ScalarNode")
-		value, err := o.GetValueRep()
-		if err != nil {
-			return buf.Bytes(), err
-		}
-		err = enc.Encode(value)
-		return buf.Bytes(), err
-	case MappingNode:
-		log.Debugf("MarshalJSON MappingNode")
-		buf.WriteByte('{')
-		for i := 0; i < len(o.Content); i += 2 {
-			if err := enc.Encode(o.Content[i].Value); err != nil {
-				return nil, err
-			}
-			buf.WriteByte(':')
-			if err := enc.Encode(o.Content[i+1]); err != nil {
-				return nil, err
-			}
-			if i != len(o.Content)-2 {
-				buf.WriteByte(',')
-			}
-		}
-		buf.WriteByte('}')
-		return buf.Bytes(), nil
-	case SequenceNode:
-		log.Debugf("MarshalJSON SequenceNode, %v, len: %v", o.Content, len(o.Content))
-		var err error
-		if len(o.Content) == 0 {
-			buf.WriteString("[]")
-		} else {
-			err = enc.Encode(o.Content)
-		}
-		return buf.Bytes(), err
-	default:
-		err := enc.Encode(nil)
-		return buf.Bytes(), err
-	}
-}
@@ -1,25 +0,0 @@
-//go:build linux
-
-package yqlib
-
-import (
-	"io/fs"
-	"os"
-	"syscall"
-)
-
-func changeOwner(info fs.FileInfo, file *os.File) error {
-	if stat, ok := info.Sys().(*syscall.Stat_t); ok {
-		uid := int(stat.Uid)
-		gid := int(stat.Gid)
-
-		err := os.Chown(file.Name(), uid, gid)
-		if err != nil {
-			// this happens with snap confinement
-			// not really a big issue as users can chown
-			// the file themselves if required.
-			log.Info("Skipping chown: %v", err)
-		}
-	}
-	return nil
-}
@@ -1,12 +0,0 @@
-//go:build !linux
-
-package yqlib
-
-import (
-	"io/fs"
-	"os"
-)
-
-func changeOwner(info fs.FileInfo, file *os.File) error {
-	return nil
-}
@@ -3,6 +3,9 @@ package yqlib
 import (
 	"fmt"
 	"strconv"
+	"time"
+
+	yaml "gopkg.in/yaml.v3"
 )

 type compareTypePref struct {
@@ -11,14 +14,14 @@ type compareTypePref struct {
 }

 func compareOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	log.Debugf("compareOperator")
+	log.Debugf("-- compareOperator")
 	prefs := expressionNode.Operation.Preferences.(compareTypePref)
-	return crossFunction(d, context, expressionNode, compare(prefs), true)
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, compare(prefs), true)
 }

 func compare(prefs compareTypePref) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-		log.Debugf("compare cross function")
+		log.Debugf("-- compare cross function")
 		if lhs == nil && rhs == nil {
 			owner := &CandidateNode{}
 			return createBooleanCandidate(owner, prefs.OrEqual), nil
@@ -30,30 +33,33 @@ func compare(prefs compareTypePref) func(d *dataTreeNavigator, context Context,
 			return createBooleanCandidate(lhs, false), nil
 		}

-		switch lhs.Kind {
-		case MappingNode:
+		lhs.Node = unwrapDoc(lhs.Node)
+		rhs.Node = unwrapDoc(rhs.Node)
+
+		switch lhs.Node.Kind {
+		case yaml.MappingNode:
 			return nil, fmt.Errorf("maps not yet supported for comparison")
-		case SequenceNode:
+		case yaml.SequenceNode:
 			return nil, fmt.Errorf("arrays not yet supported for comparison")
 		default:
-			if rhs.Kind != ScalarNode {
-				return nil, fmt.Errorf("%v (%v) cannot be subtracted from %v", rhs.Tag, rhs.GetNicePath(), lhs.Tag)
+			if rhs.Node.Kind != yaml.ScalarNode {
+				return nil, fmt.Errorf("%v (%v) cannot be subtracted from %v", rhs.Node.Tag, rhs.Path, lhs.Node.Tag)
 			}
-			target := lhs.CopyWithoutContent()
-			boolV, err := compareScalars(context, prefs, lhs, rhs)
+			target := lhs.CreateReplacement(&yaml.Node{})
+			boolV, err := compareScalars(context, prefs, lhs.Node, rhs.Node)

 			return createBooleanCandidate(target, boolV), err
 		}
 	}
 }

-func compareDateTime(layout string, prefs compareTypePref, lhs *CandidateNode, rhs *CandidateNode) (bool, error) {
-	lhsTime, err := parseDateTime(layout, lhs.Value)
+func compareDateTime(layout string, prefs compareTypePref, lhs *yaml.Node, rhs *yaml.Node) (bool, error) {
+	lhsTime, err := time.Parse(layout, lhs.Value)
 	if err != nil {
 		return false, err
 	}

-	rhsTime, err := parseDateTime(layout, rhs.Value)
+	rhsTime, err := time.Parse(layout, rhs.Value)
 	if err != nil {
 		return false, err
 	}
@@ -68,14 +74,14 @@ func compareDateTime(layout string, prefs compareTypePref, lhs *CandidateNode, r

 }

-func compareScalars(context Context, prefs compareTypePref, lhs *CandidateNode, rhs *CandidateNode) (bool, error) {
-	lhsTag := lhs.GuessTagFromCustomType()
-	rhsTag := rhs.GuessTagFromCustomType()
+func compareScalars(context Context, prefs compareTypePref, lhs *yaml.Node, rhs *yaml.Node) (bool, error) {
+	lhsTag := guessTagFromCustomType(lhs)
+	rhsTag := guessTagFromCustomType(rhs)

 	isDateTime := lhs.Tag == "!!timestamp"
 	// if the lhs is a string, it might be a timestamp in a custom format.
-	if lhsTag == "!!str" {
-		_, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value)
+	if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
+		_, err := time.Parse(context.GetDateTimeLayout(), lhs.Value)
 		isDateTime = err == nil
 	}
 	if isDateTime {
@@ -121,10 +127,6 @@ func compareScalars(context Context, prefs compareTypePref, lhs *CandidateNode,
 			return lhs.Value > rhs.Value, nil
 		}
 		return lhs.Value < rhs.Value, nil
-	} else if lhsTag == "!!null" && rhsTag == "!!null" && prefs.OrEqual {
-		return true, nil
-	} else if lhsTag == "!!null" || rhsTag == "!!null" {
-		return false, nil
 	}

 	return false, fmt.Errorf("%v not yet supported for comparison", lhs.Tag)
@@ -14,20 +14,6 @@ var compareOperatorScenarios = []expressionScenario{
 			"D0, P[a], (!!bool)::true\n",
 		},
 	},
-	{
-		skipDoc:    true,
-		expression: "(.k | length) >= 0",
-		expected: []string{
-			"D0, P[k], (!!bool)::true\n",
-		},
-	},
-	{
-		skipDoc:    true,
-		expression: `"2022-01-30T15:53:09Z" > "2020-01-30T15:53:09Z"`,
-		expected: []string{
-			"D0, P[], (!!bool)::true\n",
-		},
-	},
 	{
 		skipDoc:    true,
 		document:   "a: 5\nb: 4",
@@ -299,28 +285,28 @@ var compareOperatorScenarios = []expressionScenario{
 		description: "Both sides are null: > is false",
 		expression:  ".a > .b",
 		expected: []string{
-			"D0, P[a], (!!bool)::false\n",
+			"D0, P[], (!!bool)::false\n",
 		},
 	},
 	{
 		skipDoc:    true,
 		expression: ".a < .b",
 		expected: []string{
-			"D0, P[a], (!!bool)::false\n",
+			"D0, P[], (!!bool)::false\n",
 		},
 	},
 	{
 		description: "Both sides are null: >= is true",
 		expression:  ".a >= .b",
 		expected: []string{
-			"D0, P[a], (!!bool)::true\n",
+			"D0, P[], (!!bool)::true\n",
 		},
 	},
 	{
 		skipDoc:    true,
 		expression: ".a <= .b",
 		expected: []string{
-			"D0, P[a], (!!bool)::true\n",
+			"D0, P[], (!!bool)::true\n",
 		},
 	},

@@ -364,7 +350,7 @@ var compareOperatorScenarios = []expressionScenario{
 		document:   `a: 5`,
 		expression: ".b <= .a",
 		expected: []string{
-			"D0, P[b], (!!bool)::false\n",
+			"D0, P[a], (!!bool)::false\n",
 		},
 	},
 	{
@@ -372,7 +358,7 @@ var compareOperatorScenarios = []expressionScenario{
 		document:   `a: 5`,
 		expression: ".b < .a",
 		expected: []string{
-			"D0, P[b], (!!bool)::false\n",
+			"D0, P[a], (!!bool)::false\n",
 		},
 	},
 }
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"time"

+	"github.com/jinzhu/copier"
 	logging "gopkg.in/op/go-logging.v1"
 )

@@ -57,19 +58,13 @@ func (n *Context) SetVariable(name string, value *list.List) {
 func (n *Context) ChildContext(results *list.List) Context {
 	clone := Context{DontAutoCreate: n.DontAutoCreate, datetimeLayout: n.datetimeLayout}
 	clone.Variables = make(map[string]*list.List)
-	for variableKey, originalValueList := range n.Variables {
-
-		variableCopyList := list.New()
-		for el := originalValueList.Front(); el != nil; el = el.Next() {
-			// note that we dont make a copy of the candidate node
-			// this is so the 'ref' operator can work correctly.
-			clonedNode := el.Value.(*CandidateNode)
-			variableCopyList.PushBack(clonedNode)
+	if len(n.Variables) > 0 {
+		err := copier.Copy(&clone.Variables, n.Variables)
+		if err != nil {
+			log.Error("Error cloning context :(")
+			panic(err)
 		}
-
-		clone.Variables[variableKey] = variableCopyList
 	}
-
 	clone.MatchingNodes = results
 	return clone
 }
@@ -83,18 +78,35 @@ func (n *Context) ToString() string {
 }

 func (n *Context) DeepClone() Context {
-
-	clonedContent := list.New()
+	clone := Context{}
+	err := copier.Copy(&clone, n)
+	// copier doesn't do lists properly for some reason
+	clone.MatchingNodes = list.New()
 	for el := n.MatchingNodes.Front(); el != nil; el = el.Next() {
-		clonedNode := el.Value.(*CandidateNode).Copy()
-		clonedContent.PushBack(clonedNode)
+		clonedNode, err := el.Value.(*CandidateNode).Copy()
+		if err != nil {
+			log.Error("Error cloning context :(")
+			panic(err)
+		}
+		clone.MatchingNodes.PushBack(clonedNode)
 	}

-	return n.ChildContext(clonedContent)
+	if err != nil {
+		log.Error("Error cloning context :(")
+		panic(err)
+	}
+	return clone
 }

 func (n *Context) Clone() Context {
-	return n.ChildContext(n.MatchingNodes)
+	clone := Context{}
+	err := copier.Copy(&clone, n)
+
+	if err != nil {
+		log.Error("Error cloning context :(")
+		panic(err)
+	}
+	return clone
 }

 func (n *Context) ReadOnlyClone() Context {
@@ -1,51 +0,0 @@
-package yqlib
-
-import (
-	"container/list"
-	"testing"
-
-	"github.com/mikefarah/yq/v4/test"
-)
-
-func TestChildContext(t *testing.T) {
-
-	expectedOriginal := make(map[string]*list.List)
-	expectedOriginal["dog"] = list.New()
-	expectedOriginal["dog"].PushBack(&CandidateNode{Value: "woof"})
-
-	originalVariables := make(map[string]*list.List)
-	originalVariables["dog"] = list.New()
-	originalVariables["dog"].PushBack(&CandidateNode{Value: "woof"})
-
-	original := Context{
-		DontAutoCreate: true,
-		datetimeLayout: "cat",
-		Variables:      originalVariables,
-	}
-
-	newResults := list.New()
-	newResults.PushBack(&CandidateNode{Value: "bar"})
-
-	clone := original.ChildContext(newResults)
-	test.AssertResultComplex(t, originalVariables, clone.Variables)
-
-	clone.Variables["dog"].PushBack("bark")
-	// ensure this is a separate copy
-	test.AssertResultComplex(t, 1, originalVariables["dog"].Len())
-
-}
-
-func TestChildContextNoVariables(t *testing.T) {
-
-	original := Context{
-		DontAutoCreate: true,
-		datetimeLayout: "cat",
-	}
-
-	newResults := list.New()
-	newResults.PushBack(&CandidateNode{Value: "bar"})
-
-	clone := original.ChildContext(newResults)
-	test.AssertResultComplex(t, make(map[string]*list.List), clone.Variables)
-
-}
@@ -1,23 +0,0 @@
-package yqlib
-
-type CsvPreferences struct {
-	Separator rune
-	AutoParse bool
-}
-
-func NewDefaultCsvPreferences() CsvPreferences {
-	return CsvPreferences{
-		Separator: ',',
-		AutoParse: true,
-	}
-}
-
-func NewDefaultTsvPreferences() CsvPreferences {
-	return CsvPreferences{
-		Separator: '\t',
-		AutoParse: true,
-	}
-}
-
-var ConfiguredCsvPreferences = NewDefaultCsvPreferences()
-var ConfiguredTsvPreferences = NewDefaultTsvPreferences()
@@ -13,13 +13,6 @@ Gary,1,true,168.8
 Samantha's Rabbit,2,false,-188.8
 `

-const csvSimpleWithObject = `name,numberOfCats,likesApples,height,facts
-Gary,1,true,168.8,cool: true
-Samantha's Rabbit,2,false,-188.8,tall: indeed
-`
-const csvMissing = `name,numberOfCats,likesApples,height
-,null,,168.8
-`
 const expectedUpdatedSimpleCsv = `name,numberOfCats,likesApples,height
 Gary,3,true,168.8
 Samantha's Rabbit,2,false,-188.8
@@ -44,31 +37,6 @@ const expectedYamlFromCSV = `- name: Gary
  likesApples: false
  height: -188.8
 `
-const expectedYamlFromCSVWithObject = `- name: Gary
-  numberOfCats: 1
-  likesApples: true
-  height: 168.8
-  facts:
-    cool: true
- name: Samantha's Rabbit
-  numberOfCats: 2
-  likesApples: false
-  height: -188.8
-  facts:
-    tall: indeed
-`
-
-const expectedYamlFromCSVNoParsing = `- name: Gary
-  numberOfCats: 1
-  likesApples: true
-  height: 168.8
-  facts: 'cool: true'
- name: Samantha's Rabbit
-  numberOfCats: 2
-  likesApples: false
-  height: -188.8
-  facts: 'tall: indeed'
-`

 const expectedYamlFromCSVMissingData = `- name: Gary
  numberOfCats: 1
@@ -94,7 +62,7 @@ const tsvTestExpectedSimpleCsv = `i	like	csv
 because	excel	is	cool
 `

-var csvScenarios = []FormatScenario{
+var csvScenarios = []formatScenario{
 	{
 		description:  "Encode CSV simple",
 		input:        csvTestSimpleYaml,
@@ -142,57 +110,12 @@ var csvScenarios = []FormatScenario{
 		expected:       csvSimpleMissingData,
 		scenarioType:   "encode-csv",
 	},
-	{
-		description:  "decode csv missing",
-		skipDoc:      true,
-		input:        csvMissing,
-		expected:     csvMissing,
-		scenarioType: "roundtrip-csv",
-	},
-	{
-		description:  "decode csv key",
-		skipDoc:      true,
-		input:        csvSimple,
-		expression:   ".[0].name | key",
-		expected:     "name\n",
-		scenarioType: "decode-csv",
-	},
-	{
-		description:  "decode csv parent",
-		skipDoc:      true,
-		input:        csvSimple,
-		expression:   ".[0].name | parent | .height",
-		expected:     "168.8\n",
-		scenarioType: "decode-csv",
-	},
 	{
 		description:    "Parse CSV into an array of objects",
-		subdescription: "First row is assumed to be the header row. By default, entries with YAML/JSON formatting will be parsed!",
-		input:          csvSimpleWithObject,
-		expected:       expectedYamlFromCSVWithObject,
-		scenarioType:   "decode-csv",
-	},
-	{
-		description:  "Decode CSV line breaks",
-		skipDoc:      true,
-		input:        "heading1\n\"some data\nwith a line break\"\n",
-		expected:     "- heading1: |-\n    some data\n    with a line break\n",
-		scenarioType: "decode-csv",
-	},
-	{
-		description:    "Parse CSV into an array of objects, no auto-parsing",
-		subdescription: "First row is assumed to be the header row. Entries with YAML/JSON will be left as strings.",
-		input:          csvSimpleWithObject,
-		expected:       expectedYamlFromCSVNoParsing,
-		scenarioType:   "decode-csv-no-auto",
-	},
-	{
-		description:  "Scalar roundtrip",
-		skipDoc:      true,
-		input:        "mike\ncat",
-		expression:   ".[0].mike",
-		expected:     "cat\n",
-		scenarioType: "roundtrip-csv",
+		subdescription: "First row is assumed to be the header row.",
+		input:          csvSimple,
+		expected:       expectedYamlFromCSV,
+		scenarioType:   "decode-csv-object",
 	},
 	{
 		description:    "Parse TSV into an array of objects",
@@ -210,26 +133,24 @@ var csvScenarios = []FormatScenario{
 	},
 }

-func testCSVScenario(t *testing.T, s FormatScenario) {
+func testCSVScenario(t *testing.T, s formatScenario) {
 	switch s.scenarioType {
 	case "encode-csv":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(ConfiguredCsvPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder(',')), s.description)
 	case "encode-tsv":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(ConfiguredTsvPreferences)), s.description)
-	case "decode-csv":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredCsvPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
-	case "decode-csv-no-auto":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: ',', AutoParse: false}), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder('\t')), s.description)
+	case "decode-csv-object":
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, true, true)), s.description)
 	case "decode-tsv-object":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredTsvPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, true, true)), s.description)
 	case "roundtrip-csv":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(ConfiguredCsvPreferences), NewCsvEncoder(ConfiguredCsvPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewCsvEncoder(',')), s.description)
 	default:
 		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
 	}
 }

-func documentCSVDecodeObjectScenario(w *bufio.Writer, s FormatScenario, formatType string) {
+func documentCSVDecodeObjectScenario(t *testing.T, w *bufio.Writer, s formatScenario, formatType string) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))

 	if s.subdescription != "" {
@@ -250,36 +171,11 @@ func documentCSVDecodeObjectScenario(w *bufio.Writer, s FormatScenario, formatTy
 	}

 	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n",
-		mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: true}), NewYamlEncoder(ConfiguredYamlPreferences))),
+		processFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, true, true))),
 	)
 }

-func documentCSVDecodeObjectNoAutoScenario(w *bufio.Writer, s FormatScenario, formatType string) {
-	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
-
-	if s.subdescription != "" {
-		writeOrPanic(w, s.subdescription)
-		writeOrPanic(w, "\n\n")
-	}
-
-	writeOrPanic(w, fmt.Sprintf("Given a sample.%v file of:\n", formatType))
-	writeOrPanic(w, fmt.Sprintf("```%v\n%v\n```\n", formatType, s.input))
-
-	writeOrPanic(w, "then\n")
-	writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=%v --csv-auto-parse=f sample.%v\n```\n", formatType, formatType))
-	writeOrPanic(w, "will output\n")
-
-	separator := ','
-	if formatType == "tsv" {
-		separator = '\t'
-	}
-
-	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n",
-		mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: false}), NewYamlEncoder(ConfiguredYamlPreferences))),
-	)
-}
-
-func documentCSVEncodeScenario(w *bufio.Writer, s FormatScenario, formatType string) {
+func documentCSVEncodeScenario(w *bufio.Writer, s formatScenario, formatType string) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))

 	if s.subdescription != "" {
@@ -305,14 +201,13 @@ func documentCSVEncodeScenario(w *bufio.Writer, s FormatScenario, formatType str
 	if formatType == "tsv" {
 		separator = '\t'
 	}
-	csvPrefs := NewDefaultCsvPreferences()
-	csvPrefs.Separator = separator
+
 	writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
-		mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(csvPrefs))),
+		processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder(separator))),
 	)
 }

-func documentCSVRoundTripScenario(w *bufio.Writer, s FormatScenario, formatType string) {
+func documentCSVRoundTripScenario(w *bufio.Writer, s formatScenario, formatType string) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))

 	if s.subdescription != "" {
@@ -339,16 +234,13 @@ func documentCSVRoundTripScenario(w *bufio.Writer, s FormatScenario, formatType
 		separator = '\t'
 	}

-	csvPrefs := NewDefaultCsvPreferences()
-	csvPrefs.Separator = separator
-
 	writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
-		mustProcessFormatScenario(s, NewCSVObjectDecoder(CsvPreferences{Separator: separator, AutoParse: true}), NewCsvEncoder(csvPrefs))),
+		processFormatScenario(s, NewCSVObjectDecoder(separator), NewCsvEncoder(separator))),
 	)
 }

-func documentCSVScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(FormatScenario)
+func documentCSVScenario(t *testing.T, w *bufio.Writer, i interface{}) {
+	s := i.(formatScenario)
 	if s.skipDoc {
 		return
 	}
@@ -357,12 +249,10 @@ func documentCSVScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
 		documentCSVEncodeScenario(w, s, "csv")
 	case "encode-tsv":
 		documentCSVEncodeScenario(w, s, "tsv")
-	case "decode-csv":
-		documentCSVDecodeObjectScenario(w, s, "csv")
-	case "decode-csv-no-auto":
-		documentCSVDecodeObjectNoAutoScenario(w, s, "csv")
+	case "decode-csv-object":
+		documentCSVDecodeObjectScenario(t, w, s, "csv")
 	case "decode-tsv-object":
-		documentCSVDecodeObjectScenario(w, s, "tsv")
+		documentCSVDecodeObjectScenario(t, w, s, "tsv")
 	case "roundtrip-csv":
 		documentCSVRoundTripScenario(w, s, "csv")

@@ -7,13 +7,10 @@ import (
 )

 type DataTreeNavigator interface {
-	// given the context and an expressionNode,
+	// given the context and a expressionNode,
 	// this will process the against the given expressionNode and return
 	// a new context of matching candidates
 	GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error)
-
-	DeeplyAssign(context Context, path []interface{}, rhsNode *CandidateNode) error
-	DeeplyAssignKey(context Context, path []interface{}, rhsNode *CandidateNode) error
 }

 type dataTreeNavigator struct {
@@ -23,41 +20,6 @@ func NewDataTreeNavigator() DataTreeNavigator {
 	return &dataTreeNavigator{}
 }

-func (d *dataTreeNavigator) DeeplyAssign(context Context, path []interface{}, rhsCandidateNode *CandidateNode) error {
-	return d.deeplyAssign(context, path, rhsCandidateNode, false)
-}
-
-func (d *dataTreeNavigator) DeeplyAssignKey(context Context, path []interface{}, rhsCandidateNode *CandidateNode) error {
-	return d.deeplyAssign(context, path, rhsCandidateNode, true)
-}
-
-func (d *dataTreeNavigator) deeplyAssign(context Context, path []interface{}, rhsCandidateNode *CandidateNode, targetKey bool) error {
-
-	assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
-
-	if rhsCandidateNode.Kind == MappingNode {
-		log.Debug("DeeplyAssign: deeply merging object")
-		// if the rhs is a map, we need to deeply merge it in.
-		// otherwise we'll clobber any existing fields
-		assignmentOp = &Operation{OperationType: multiplyAssignOpType, Preferences: multiplyPreferences{
-			AppendArrays:  true,
-			TraversePrefs: traversePreferences{DontFollowAlias: true},
-			AssignPrefs:   assignPreferences{},
-		}}
-	}
-
-	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}
-
-	assignmentOpNode := &ExpressionNode{
-		Operation: assignmentOp,
-		LHS:       createTraversalTree(path, traversePreferences{}, targetKey),
-		RHS:       &ExpressionNode{Operation: rhsOp},
-	}
-
-	_, err := d.GetMatchingNodes(context, assignmentOpNode)
-	return err
-}
-
 func (d *dataTreeNavigator) GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error) {
 	if expressionNode == nil {
 		log.Debugf("getMatchingNodes - nothing to do")
@@ -69,6 +31,7 @@ func (d *dataTreeNavigator) GetMatchingNodes(context Context, expressionNode *Ex
 			log.Debug(NodeToString(el.Value.(*CandidateNode)))
 		}
 	}
+	log.Debug(">>")
 	handler := expressionNode.Operation.OperationType.Handler
 	if handler != nil {
 		return handler(d, context, expressionNode)
@@ -1,10 +1,44 @@
 package yqlib

 import (
+	"fmt"
 	"io"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+type InputFormat uint
+
+const (
+	YamlInputFormat = 1 << iota
+	XMLInputFormat
+	PropertiesInputFormat
+	Base64InputFormat
+	JsonInputFormat
+	CSVObjectInputFormat
+	TSVObjectInputFormat
 )

 type Decoder interface {
-	Init(reader io.Reader) error
-	Decode() (*CandidateNode, error)
+	Init(reader io.Reader)
+	Decode(node *yaml.Node) error
+}
+
+func InputFormatFromString(format string) (InputFormat, error) {
+	switch format {
+	case "yaml", "y":
+		return YamlInputFormat, nil
+	case "xml", "x":
+		return XMLInputFormat, nil
+	case "props", "p":
+		return PropertiesInputFormat, nil
+	case "json", "ndjson", "j":
+		return JsonInputFormat, nil
+	case "csv", "c":
+		return CSVObjectInputFormat, nil
+	case "tsv", "t":
+		return TSVObjectInputFormat, nil
+	default:
+		return 0, fmt.Errorf("unknown format '%v' please use [yaml|xml|props]", format)
+	}
 }
@@ -4,31 +4,10 @@ import (
 	"bytes"
 	"encoding/base64"
 	"io"
-	"strings"
+
+	yaml "gopkg.in/yaml.v3"
 )

-type base64Padder struct {
-	count uint64
-	io.Reader
-}
-
-func (c *base64Padder) pad(buf []byte) (int, error) {
-	pad := strings.Repeat("=", int(4-c.count%4))
-	n, err := strings.NewReader(pad).Read(buf)
-	c.count += uint64(n)
-	return n, err
-}
-
-func (c *base64Padder) Read(buf []byte) (int, error) {
-	n, err := c.Reader.Read(buf)
-	c.count += uint64(n)
-
-	if err == io.EOF && c.count%4 != 0 {
-		return c.pad(buf)
-	}
-	return n, err
-}
-
 type base64Decoder struct {
 	reader       io.Reader
 	finished     bool
@@ -40,22 +19,21 @@ func NewBase64Decoder() Decoder {
 	return &base64Decoder{finished: false, encoding: *base64.StdEncoding}
 }

-func (dec *base64Decoder) Init(reader io.Reader) error {
-	dec.reader = &base64Padder{Reader: reader}
+func (dec *base64Decoder) Init(reader io.Reader) {
+	dec.reader = reader
 	dec.readAnything = false
 	dec.finished = false
-	return nil
 }

-func (dec *base64Decoder) Decode() (*CandidateNode, error) {
+func (dec *base64Decoder) Decode(rootYamlNode *yaml.Node) error {
 	if dec.finished {
-		return nil, io.EOF
+		return io.EOF
 	}
 	base64Reader := base64.NewDecoder(&dec.encoding, dec.reader)
 	buf := new(bytes.Buffer)

 	if _, err := buf.ReadFrom(base64Reader); err != nil {
-		return nil, err
+		return err
 	}
 	if buf.Len() == 0 {
 		dec.finished = true
@@ -64,9 +42,12 @@ func (dec *base64Decoder) Decode() (*CandidateNode, error) {
 		// otherwise if we've already read some bytes, and now we get
 		// an empty string, then we are done.
 		if dec.readAnything {
-			return nil, io.EOF
+			return io.EOF
 		}
 	}
 	dec.readAnything = true
-	return CreateStringScalarNode(buf.String()), nil
+	rootYamlNode.Kind = yaml.ScalarNode
+	rootYamlNode.Tag = "!!str"
+	rootYamlNode.Value = buf.String()
+	return nil
 }
@@ -5,70 +5,73 @@ import (
 	"errors"
 	"io"

-	"github.com/dimchansky/utfbom"
+	yaml "gopkg.in/yaml.v3"
 )

 type csvObjectDecoder struct {
-	prefs    CsvPreferences
-	reader   csv.Reader
-	finished bool
+	separator rune
+	reader    csv.Reader
+	finished  bool
 }

-func NewCSVObjectDecoder(prefs CsvPreferences) Decoder {
-	return &csvObjectDecoder{prefs: prefs}
+func NewCSVObjectDecoder(separator rune) Decoder {
+	return &csvObjectDecoder{separator: separator}
 }

-func (dec *csvObjectDecoder) Init(reader io.Reader) error {
-	cleanReader, enc := utfbom.Skip(reader)
-	log.Debugf("Detected encoding: %s\n", enc)
-	dec.reader = *csv.NewReader(cleanReader)
-	dec.reader.Comma = dec.prefs.Separator
+func (dec *csvObjectDecoder) Init(reader io.Reader) {
+	dec.reader = *csv.NewReader(reader)
+	dec.reader.Comma = dec.separator
 	dec.finished = false
-	return nil
 }

-func (dec *csvObjectDecoder) convertToNode(content string) *CandidateNode {
+func (dec *csvObjectDecoder) convertToYamlNode(content string) *yaml.Node {
 	node, err := parseSnippet(content)
-	// if we're not auto-parsing, then we wont put in parsed objects or arrays
-	// but we still parse scalars
-	if err != nil || (!dec.prefs.AutoParse && node.Kind != ScalarNode) {
-		return CreateScalarNode(content, content)
+	if err != nil {
+		return createScalarNode(content, content)
 	}
 	return node
 }

-func (dec *csvObjectDecoder) createObject(headerRow []string, contentRow []string) *CandidateNode {
-	objectNode := &CandidateNode{Kind: MappingNode, Tag: "!!map"}
+func (dec *csvObjectDecoder) createObject(headerRow []string, contentRow []string) *yaml.Node {
+	objectNode := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}

 	for i, header := range headerRow {
-		objectNode.AddKeyValueChild(CreateScalarNode(header, header), dec.convertToNode(contentRow[i]))
+		objectNode.Content = append(
+			objectNode.Content,
+			createScalarNode(header, header),
+			dec.convertToYamlNode(contentRow[i]))
 	}
 	return objectNode
 }

-func (dec *csvObjectDecoder) Decode() (*CandidateNode, error) {
+func (dec *csvObjectDecoder) Decode(rootYamlNode *yaml.Node) error {
 	if dec.finished {
-		return nil, io.EOF
+		return io.EOF
 	}
 	headerRow, err := dec.reader.Read()
 	log.Debugf(": headerRow%v", headerRow)
 	if err != nil {
-		return nil, err
+		return err
 	}

-	rootArray := &CandidateNode{Kind: SequenceNode, Tag: "!!seq"}
+	rootArray := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}

 	contentRow, err := dec.reader.Read()

 	for err == nil && len(contentRow) > 0 {
 		log.Debugf("Adding contentRow: %v", contentRow)
-		rootArray.AddChild(dec.createObject(headerRow, contentRow))
+		rootArray.Content = append(rootArray.Content, dec.createObject(headerRow, contentRow))
 		contentRow, err = dec.reader.Read()
 		log.Debugf("Read next contentRow: %v, %v", contentRow, err)
 	}
 	if !errors.Is(err, io.EOF) {
-		return nil, err
+		return err
 	}

-	return rootArray, nil
+	log.Debugf("finished, contentRow%v", contentRow)
+	log.Debugf("err: %v", err)
+
+	rootYamlNode.Kind = yaml.DocumentNode
+	rootYamlNode.Content = []*yaml.Node{rootArray}
+	return nil
 }
@@ -1,46 +0,0 @@
-//go:build !yq_noyaml
-
-//
-// NOTE this is still a WIP - not yet ready.
-//
-
-package yqlib
-
-import (
-	"io"
-
-	yaml "github.com/goccy/go-yaml"
-	"github.com/goccy/go-yaml/ast"
-)
-
-type goccyYamlDecoder struct {
-	decoder yaml.Decoder
-	cm      yaml.CommentMap
-}
-
-func NewGoccyYAMLDecoder() Decoder {
-	return &goccyYamlDecoder{}
-}
-
-func (dec *goccyYamlDecoder) Init(reader io.Reader) error {
-	dec.cm = yaml.CommentMap{}
-	dec.decoder = *yaml.NewDecoder(reader, yaml.CommentToMap(dec.cm), yaml.UseOrderedMap())
-	return nil
-}
-
-func (dec *goccyYamlDecoder) Decode() (*CandidateNode, error) {
-
-	var ast ast.Node
-
-	err := dec.decoder.Decode(&ast)
-	if err != nil {
-		return nil, err
-	}
-
-	candidateNode := &CandidateNode{}
-	if err := candidateNode.UnmarshalGoccyYAML(ast, dec.cm); err != nil {
-		return nil, err
-	}
-
-	return candidateNode, nil
-}
@@ -1,11 +1,11 @@
-//go:build !yq_nojson
-
 package yqlib

 import (
+	"fmt"
 	"io"

 	"github.com/goccy/go-json"
+	yaml "gopkg.in/yaml.v3"
 )

 type jsonDecoder struct {
@@ -16,18 +16,70 @@ func NewJSONDecoder() Decoder {
 	return &jsonDecoder{}
 }

-func (dec *jsonDecoder) Init(reader io.Reader) error {
+func (dec *jsonDecoder) Init(reader io.Reader) {
 	dec.decoder = *json.NewDecoder(reader)
+}
+
+func (dec *jsonDecoder) Decode(rootYamlNode *yaml.Node) error {
+
+	var dataBucket interface{}
+	log.Debug("going to decode")
+	err := dec.decoder.Decode(&dataBucket)
+	if err != nil {
+		return err
+	}
+	node, err := dec.convertToYamlNode(dataBucket)
+
+	if err != nil {
+		return err
+	}
+	rootYamlNode.Kind = yaml.DocumentNode
+	rootYamlNode.Content = []*yaml.Node{node}
 	return nil
 }

-func (dec *jsonDecoder) Decode() (*CandidateNode, error) {
-
-	var dataBucket CandidateNode
-	err := dec.decoder.Decode(&dataBucket)
-	if err != nil {
-		return nil, err
+func (dec *jsonDecoder) convertToYamlNode(data interface{}) (*yaml.Node, error) {
+	switch data := data.(type) {
+	case nil:
+		return createScalarNode(nil, "null"), nil
+	case float64, float32:
+		// json decoder returns ints as float.
+		return parseSnippet(fmt.Sprintf("%v", data))
+	case int, int64, int32, string, bool:
+		return createScalarNode(data, fmt.Sprintf("%v", data)), nil
+	case map[string]interface{}:
+		return dec.parseMap(data)
+	case []interface{}:
+		return dec.parseArray(data)
+	default:
+		return nil, fmt.Errorf("unrecognised type :(")
 	}
+}

-	return &dataBucket, nil
+func (dec *jsonDecoder) parseMap(dataMap map[string]interface{}) (*yaml.Node, error) {
+
+	var yamlMap = &yaml.Node{Kind: yaml.MappingNode}
+
+	for key, value := range dataMap {
+		yamlValue, err := dec.convertToYamlNode(value)
+		if err != nil {
+			return nil, err
+		}
+		yamlMap.Content = append(yamlMap.Content, createScalarNode(key, fmt.Sprintf("%v", key)), yamlValue)
+	}
+	return yamlMap, nil
+}
+
+func (dec *jsonDecoder) parseArray(dataArray []interface{}) (*yaml.Node, error) {
+
+	var yamlMap = &yaml.Node{Kind: yaml.SequenceNode}
+
+	for _, value := range dataArray {
+		yamlValue, err := dec.convertToYamlNode(value)
+		if err != nil {
+			return nil, err
+		}
+		yamlMap.Content = append(yamlMap.Content, yamlValue)
+	}
+	return yamlMap, nil
 }
@@ -1,165 +0,0 @@
-//go:build !yq_nolua
-
-package yqlib
-
-import (
-	"fmt"
-	"io"
-	"math"
-
-	lua "github.com/yuin/gopher-lua"
-)
-
-type luaDecoder struct {
-	reader   io.Reader
-	finished bool
-	prefs    LuaPreferences
-}
-
-func NewLuaDecoder(prefs LuaPreferences) Decoder {
-	return &luaDecoder{
-		prefs: prefs,
-	}
-}
-
-func (dec *luaDecoder) Init(reader io.Reader) error {
-	dec.reader = reader
-	return nil
-}
-
-func (dec *luaDecoder) convertToYamlNode(ls *lua.LState, lv lua.LValue) *CandidateNode {
-	switch lv.Type() {
-	case lua.LTNil:
-		return &CandidateNode{
-			Kind:  ScalarNode,
-			Tag:   "!!null",
-			Value: "",
-		}
-	case lua.LTBool:
-		return &CandidateNode{
-			Kind:  ScalarNode,
-			Tag:   "!!bool",
-			Value: lv.String(),
-		}
-	case lua.LTNumber:
-		n := float64(lua.LVAsNumber(lv))
-		// various special case floats
-		if math.IsNaN(n) {
-			return &CandidateNode{
-				Kind:  ScalarNode,
-				Tag:   "!!float",
-				Value: ".nan",
-			}
-		}
-		if math.IsInf(n, 1) {
-			return &CandidateNode{
-				Kind:  ScalarNode,
-				Tag:   "!!float",
-				Value: ".inf",
-			}
-		}
-		if math.IsInf(n, -1) {
-			return &CandidateNode{
-				Kind:  ScalarNode,
-				Tag:   "!!float",
-				Value: "-.inf",
-			}
-		}
-
-		// does it look like an integer?
-		if n == float64(int(n)) {
-			return &CandidateNode{
-				Kind:  ScalarNode,
-				Tag:   "!!int",
-				Value: lv.String(),
-			}
-		}
-
-		return &CandidateNode{
-			Kind:  ScalarNode,
-			Tag:   "!!float",
-			Value: lv.String(),
-		}
-	case lua.LTString:
-		return &CandidateNode{
-			Kind:  ScalarNode,
-			Tag:   "!!str",
-			Value: lv.String(),
-		}
-	case lua.LTFunction:
-		return &CandidateNode{
-			Kind:  ScalarNode,
-			Tag:   "tag:lua.org,2006,function",
-			Value: lv.String(),
-		}
-	case lua.LTTable:
-		// Simultaneously create a sequence and a map, pick which one to return
-		// based on whether all keys were consecutive integers
-		i := 1
-		yaml_sequence := &CandidateNode{
-			Kind: SequenceNode,
-			Tag:  "!!seq",
-		}
-		yaml_map := &CandidateNode{
-			Kind: MappingNode,
-			Tag:  "!!map",
-		}
-		t := lv.(*lua.LTable)
-		k, v := ls.Next(t, lua.LNil)
-		for k != lua.LNil {
-			if ki, ok := k.(lua.LNumber); i != 0 && ok && math.Mod(float64(ki), 1) == 0 && int(ki) == i {
-				i++
-			} else {
-				i = 0
-			}
-			newKey := dec.convertToYamlNode(ls, k)
-
-			yv := dec.convertToYamlNode(ls, v)
-			yaml_map.AddKeyValueChild(newKey, yv)
-
-			if i != 0 {
-				yaml_sequence.AddChild(yv)
-			}
-			k, v = ls.Next(t, k)
-		}
-		if i != 0 {
-			return yaml_sequence
-		}
-		return yaml_map
-	default:
-		return &CandidateNode{
-			Kind:        ScalarNode,
-			LineComment: fmt.Sprintf("Unhandled Lua type: %s", lv.Type().String()),
-			Tag:         "!!null",
-			Value:       lv.String(),
-		}
-	}
-}
-
-func (dec *luaDecoder) decideTopLevelNode(ls *lua.LState) *CandidateNode {
-	if ls.GetTop() == 0 {
-		// no items were explicitly returned, encode the globals table instead
-		return dec.convertToYamlNode(ls, ls.Get(lua.GlobalsIndex))
-	}
-	return dec.convertToYamlNode(ls, ls.Get(1))
-}
-
-func (dec *luaDecoder) Decode() (*CandidateNode, error) {
-	if dec.finished {
-		return nil, io.EOF
-	}
-	ls := lua.NewState(lua.Options{SkipOpenLibs: true})
-	defer ls.Close()
-	fn, err := ls.Load(dec.reader, "@input")
-	if err != nil {
-		return nil, err
-	}
-	ls.Push(fn)
-	err = ls.PCall(0, lua.MultRet, nil)
-	if err != nil {
-		return nil, err
-	}
-	firstNode := dec.decideTopLevelNode(ls)
-	dec.finished = true
-	return firstNode, nil
-}
@@ -0,0 +1,122 @@
+package yqlib
+
+import (
+	"bytes"
+	"io"
+	"strconv"
+	"strings"
+
+	"github.com/magiconair/properties"
+	"gopkg.in/yaml.v3"
+)
+
+type propertiesDecoder struct {
+	reader   io.Reader
+	finished bool
+	d        DataTreeNavigator
+}
+
+func NewPropertiesDecoder() Decoder {
+	return &propertiesDecoder{d: NewDataTreeNavigator(), finished: false}
+}
+
+func (dec *propertiesDecoder) Init(reader io.Reader) {
+	dec.reader = reader
+	dec.finished = false
+}
+
+func parsePropKey(key string) []interface{} {
+	pathStrArray := strings.Split(key, ".")
+	path := make([]interface{}, len(pathStrArray))
+	for i, pathStr := range pathStrArray {
+		num, err := strconv.ParseInt(pathStr, 10, 32)
+		if err == nil {
+			path[i] = num
+		} else {
+			path[i] = pathStr
+		}
+	}
+	return path
+}
+
+func (dec *propertiesDecoder) processComment(c string) string {
+	if c == "" {
+		return ""
+	}
+	return "# " + c
+}
+
+func (dec *propertiesDecoder) applyProperty(properties *properties.Properties, context Context, key string) error {
+	value, _ := properties.Get(key)
+	path := parsePropKey(key)
+
+	rhsNode := &yaml.Node{
+		Value:       value,
+		Tag:         "!!str",
+		Kind:        yaml.ScalarNode,
+		LineComment: dec.processComment(properties.GetComment(key)),
+	}
+
+	rhsNode.Tag = guessTagFromCustomType(rhsNode)
+
+	rhsCandidateNode := &CandidateNode{
+		Path: path,
+		Node: rhsNode,
+	}
+
+	assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
+
+	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}
+
+	assignmentOpNode := &ExpressionNode{
+		Operation: assignmentOp,
+		LHS:       createTraversalTree(path, traversePreferences{}, false),
+		RHS:       &ExpressionNode{Operation: rhsOp},
+	}
+
+	_, err := dec.d.GetMatchingNodes(context, assignmentOpNode)
+	return err
+}
+
+func (dec *propertiesDecoder) Decode(rootYamlNode *yaml.Node) error {
+	if dec.finished {
+		return io.EOF
+	}
+	buf := new(bytes.Buffer)
+
+	if _, err := buf.ReadFrom(dec.reader); err != nil {
+		return err
+	}
+	if buf.Len() == 0 {
+		dec.finished = true
+		return io.EOF
+	}
+	properties, err := properties.LoadString(buf.String())
+	if err != nil {
+		return err
+	}
+	properties.DisableExpansion = true
+
+	rootMap := &CandidateNode{
+		Node: &yaml.Node{
+			Kind: yaml.MappingNode,
+			Tag:  "!!map",
+		},
+	}
+
+	context := Context{}
+	context = context.SingleChildContext(rootMap)
+
+	for _, key := range properties.Keys() {
+		if err := dec.applyProperty(properties, context, key); err != nil {
+			return err
+		}
+
+	}
+
+	rootYamlNode.Kind = yaml.DocumentNode
+	rootYamlNode.Content = []*yaml.Node{rootMap.Node}
+	dec.finished = true
+	return nil
+
+}
@@ -3,11 +3,10 @@ package yqlib
 import (
 	"bufio"
 	"bytes"
-	"fmt"
 	"strings"
 )

-type FormatScenario struct {
+type formatScenario struct {
 	input          string
 	indent         int
 	expression     string
@@ -16,25 +15,22 @@ type FormatScenario struct {
 	subdescription string
 	skipDoc        bool
 	scenarioType   string
-	expectedError  string
 }

-func processFormatScenario(s FormatScenario, decoder Decoder, encoder Encoder) (string, error) {
+func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) string {
+
 	var output bytes.Buffer
 	writer := bufio.NewWriter(&output)

 	if decoder == nil {
-		decoder = NewYamlDecoder(ConfiguredYamlPreferences)
+		decoder = NewYamlDecoder()
 	}

-	log.Debugf("reading docs")
 	inputs, err := readDocuments(strings.NewReader(s.input), "sample.yml", 0, decoder)
 	if err != nil {
-		return "", err
+		panic(err)
 	}

-	log.Debugf("done reading the documents")
-
 	expression := s.expression
 	if expression == "" {
 		expression = "."
@@ -43,33 +39,22 @@ func processFormatScenario(s FormatScenario, decoder Decoder, encoder Encoder) (
 	exp, err := getExpressionParser().ParseExpression(expression)

 	if err != nil {
-		return "", err
+		panic(err)
 	}

 	context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: inputs}, exp)

-	log.Debugf("Going to print: %v", NodesToString(context.MatchingNodes))
-
 	if err != nil {
-		return "", err
+		panic(err)
 	}

 	printer := NewPrinter(encoder, NewSinglePrinterWriter(writer))
 	err = printer.PrintResults(context.MatchingNodes)
 	if err != nil {
-		return "", err
+		panic(err)
 	}
 	writer.Flush()

-	return output.String(), nil
-}
-
-func mustProcessFormatScenario(s FormatScenario, decoder Decoder, encoder Encoder) string {
-
-	result, err := processFormatScenario(s, decoder, encoder)
-	if err != nil {
-		panic(fmt.Errorf("Bad scenario %v: %w", s.description, err))
-	}
-	return result
+	return output.String()

 }
@@ -1,357 +0,0 @@
-//go:build !yq_notoml
-
-package yqlib
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"io"
-	"strconv"
-	"time"
-
-	toml "github.com/pelletier/go-toml/v2/unstable"
-)
-
-type tomlDecoder struct {
-	parser   toml.Parser
-	finished bool
-	d        DataTreeNavigator
-	rootMap  *CandidateNode
-}
-
-func NewTomlDecoder() Decoder {
-	return &tomlDecoder{
-		finished: false,
-		d:        NewDataTreeNavigator(),
-	}
-}
-
-func (dec *tomlDecoder) Init(reader io.Reader) error {
-	dec.parser = toml.Parser{}
-	buf := new(bytes.Buffer)
-	_, err := buf.ReadFrom(reader)
-	if err != nil {
-		return err
-	}
-	dec.parser.Reset(buf.Bytes())
-	dec.rootMap = &CandidateNode{
-		Kind: MappingNode,
-		Tag:  "!!map",
-	}
-	return nil
-}
-
-func (dec *tomlDecoder) getFullPath(tomlNode *toml.Node) []interface{} {
-	path := make([]interface{}, 0)
-	for {
-		path = append(path, string(tomlNode.Data))
-		tomlNode = tomlNode.Next()
-		if tomlNode == nil {
-			return path
-		}
-	}
-}
-
-func (dec *tomlDecoder) processKeyValueIntoMap(rootMap *CandidateNode, tomlNode *toml.Node) error {
-	value := tomlNode.Value()
-	path := dec.getFullPath(value.Next())
-	log.Debug("processKeyValueIntoMap: %v", path)
-
-	valueNode, err := dec.decodeNode(value)
-	if err != nil {
-		return err
-	}
-
-	context := Context{}
-	context = context.SingleChildContext(rootMap)
-
-	return dec.d.DeeplyAssign(context, path, valueNode)
-}
-
-func (dec *tomlDecoder) decodeKeyValuesIntoMap(rootMap *CandidateNode, tomlNode *toml.Node) (bool, error) {
-	log.Debug("decodeKeyValuesIntoMap -- processing first (current) entry")
-	if err := dec.processKeyValueIntoMap(rootMap, tomlNode); err != nil {
-		return false, err
-	}
-
-	for dec.parser.NextExpression() {
-		nextItem := dec.parser.Expression()
-		log.Debug("decodeKeyValuesIntoMap -- next exp, its a %v", nextItem.Kind)
-
-		if nextItem.Kind == toml.KeyValue {
-			if err := dec.processKeyValueIntoMap(rootMap, nextItem); err != nil {
-				return false, err
-			}
-		} else {
-			// run out of key values
-			log.Debug("done in decodeKeyValuesIntoMap, gota a %v", nextItem.Kind)
-			return true, nil
-		}
-	}
-	log.Debug("no more things to read in")
-	return false, nil
-}
-
-func (dec *tomlDecoder) createInlineTableMap(tomlNode *toml.Node) (*CandidateNode, error) {
-	content := make([]*CandidateNode, 0)
-	log.Debug("createInlineTableMap")
-
-	iterator := tomlNode.Children()
-	for iterator.Next() {
-		child := iterator.Node()
-		if child.Kind != toml.KeyValue {
-			return nil, fmt.Errorf("only keyvalue pairs are supported in inlinetables, got %v instead", child.Kind)
-		}
-
-		keyValues := &CandidateNode{
-			Kind: MappingNode,
-			Tag:  "!!map",
-		}
-
-		if err := dec.processKeyValueIntoMap(keyValues, child); err != nil {
-			return nil, err
-		}
-
-		content = append(content, keyValues.Content...)
-	}
-
-	return &CandidateNode{
-		Kind:    MappingNode,
-		Tag:     "!!map",
-		Content: content,
-	}, nil
-}
-
-func (dec *tomlDecoder) createArray(tomlNode *toml.Node) (*CandidateNode, error) {
-	content := make([]*CandidateNode, 0)
-	iterator := tomlNode.Children()
-	for iterator.Next() {
-		child := iterator.Node()
-		yamlNode, err := dec.decodeNode(child)
-		if err != nil {
-			return nil, err
-		}
-		content = append(content, yamlNode)
-	}
-
-	return &CandidateNode{
-		Kind:    SequenceNode,
-		Tag:     "!!seq",
-		Content: content,
-	}, nil
-
-}
-
-func (dec *tomlDecoder) createStringScalar(tomlNode *toml.Node) (*CandidateNode, error) {
-	content := string(tomlNode.Data)
-	return CreateScalarNode(content, content), nil
-}
-
-func (dec *tomlDecoder) createBoolScalar(tomlNode *toml.Node) (*CandidateNode, error) {
-	content := string(tomlNode.Data)
-	return CreateScalarNode(content == "true", content), nil
-}
-
-func (dec *tomlDecoder) createIntegerScalar(tomlNode *toml.Node) (*CandidateNode, error) {
-	content := string(tomlNode.Data)
-	_, num, err := parseInt64(content)
-	return CreateScalarNode(num, content), err
-}
-
-func (dec *tomlDecoder) createDateTimeScalar(tomlNode *toml.Node) (*CandidateNode, error) {
-	content := string(tomlNode.Data)
-	val, err := parseDateTime(time.RFC3339, content)
-	return CreateScalarNode(val, content), err
-}
-
-func (dec *tomlDecoder) createFloatScalar(tomlNode *toml.Node) (*CandidateNode, error) {
-	content := string(tomlNode.Data)
-	num, err := strconv.ParseFloat(content, 64)
-	return CreateScalarNode(num, content), err
-}
-
-func (dec *tomlDecoder) decodeNode(tomlNode *toml.Node) (*CandidateNode, error) {
-	switch tomlNode.Kind {
-	case toml.Key, toml.String:
-		return dec.createStringScalar(tomlNode)
-	case toml.Bool:
-		return dec.createBoolScalar(tomlNode)
-	case toml.Integer:
-		return dec.createIntegerScalar(tomlNode)
-	case toml.DateTime:
-		return dec.createDateTimeScalar(tomlNode)
-	case toml.Float:
-		return dec.createFloatScalar(tomlNode)
-	case toml.Array:
-		return dec.createArray(tomlNode)
-	case toml.InlineTable:
-		return dec.createInlineTableMap(tomlNode)
-	default:
-		return nil, fmt.Errorf("unsupported type %v", tomlNode.Kind)
-	}
-
-}
-
-func (dec *tomlDecoder) Decode() (*CandidateNode, error) {
-	if dec.finished {
-		return nil, io.EOF
-	}
-	//
-	// toml library likes to panic
-	var deferredError error
-	defer func() { //catch or finally
-		if r := recover(); r != nil {
-			var ok bool
-			deferredError, ok = r.(error)
-			if !ok {
-				deferredError = fmt.Errorf("pkg: %v", r)
-			}
-		}
-	}()
-
-	log.Debug("ok here we go")
-	var runAgainstCurrentExp = false
-	var err error
-	for runAgainstCurrentExp || dec.parser.NextExpression() {
-
-		if runAgainstCurrentExp {
-			log.Debug("running against current exp")
-		}
-
-		currentNode := dec.parser.Expression()
-
-		log.Debug("currentNode: %v ", currentNode.Kind)
-		runAgainstCurrentExp, err = dec.processTopLevelNode(currentNode)
-		if err != nil {
-			return dec.rootMap, err
-		}
-
-	}
-
-	err = dec.parser.Error()
-	if err != nil {
-		return nil, err
-	}
-
-	// must have finished
-	dec.finished = true
-
-	if len(dec.rootMap.Content) == 0 {
-		return nil, io.EOF
-	}
-
-	return dec.rootMap, deferredError
-
-}
-
-func (dec *tomlDecoder) processTopLevelNode(currentNode *toml.Node) (bool, error) {
-	var runAgainstCurrentExp bool
-	var err error
-	log.Debug("processTopLevelNode: Going to process %v state is current %v", currentNode.Kind, NodeToString(dec.rootMap))
-	if currentNode.Kind == toml.Table {
-		runAgainstCurrentExp, err = dec.processTable(currentNode)
-	} else if currentNode.Kind == toml.ArrayTable {
-		runAgainstCurrentExp, err = dec.processArrayTable(currentNode)
-	} else {
-		runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(dec.rootMap, currentNode)
-	}
-
-	log.Debug("processTopLevelNode: DONE Processing state is now %v", NodeToString(dec.rootMap))
-	return runAgainstCurrentExp, err
-}
-
-func (dec *tomlDecoder) processTable(currentNode *toml.Node) (bool, error) {
-	log.Debug("Enter processTable")
-	fullPath := dec.getFullPath(currentNode.Child())
-	log.Debug("fullpath: %v", fullPath)
-
-	tableNodeValue := &CandidateNode{
-		Kind:    MappingNode,
-		Tag:     "!!map",
-		Content: make([]*CandidateNode, 0),
-	}
-
-	var tableValue *toml.Node
-	runAgainstCurrentExp := false
-	var err error
-	hasValue := dec.parser.NextExpression()
-	// check to see if there is any table data
-	if hasValue {
-		tableValue = dec.parser.Expression()
-		// next expression is not table data, so we are done
-		if tableValue.Kind != toml.KeyValue {
-			log.Debug("got an empty table, returning")
-			return true, nil
-		}
-
-		runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)
-		if err != nil && !errors.Is(io.EOF, err) {
-			return false, err
-		}
-	}
-
-	c := Context{}
-	c = c.SingleChildContext(dec.rootMap)
-	err = dec.d.DeeplyAssign(c, fullPath, tableNodeValue)
-	if err != nil {
-		return false, err
-	}
-	return runAgainstCurrentExp, nil
-}
-
-func (dec *tomlDecoder) arrayAppend(context Context, path []interface{}, rhsNode *CandidateNode) error {
-	log.Debug("arrayAppend to path: %v,%v", path, NodeToString(rhsNode))
-	rhsCandidateNode := &CandidateNode{
-		Kind:    SequenceNode,
-		Tag:     "!!seq",
-		Content: []*CandidateNode{rhsNode},
-	}
-
-	assignmentOp := &Operation{OperationType: addAssignOpType}
-
-	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}
-
-	assignmentOpNode := &ExpressionNode{
-		Operation: assignmentOp,
-		LHS:       createTraversalTree(path, traversePreferences{}, false),
-		RHS:       &ExpressionNode{Operation: rhsOp},
-	}
-
-	_, err := dec.d.GetMatchingNodes(context, assignmentOpNode)
-	return err
-}
-
-func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error) {
-	log.Debug("Entering processArrayTable")
-	fullPath := dec.getFullPath(currentNode.Child())
-	log.Debug("Fullpath: %v", fullPath)
-
-	// need to use the array append exp to add another entry to
-	// this array: fullpath += [ thing ]
-
-	hasValue := dec.parser.NextExpression()
-	if !hasValue {
-		return false, fmt.Errorf("error retrieving table %v value: %w", fullPath, dec.parser.Error())
-	}
-
-	tableNodeValue := &CandidateNode{
-		Kind: MappingNode,
-		Tag:  "!!map",
-	}
-
-	tableValue := dec.parser.Expression()
-	runAgainstCurrentExp, err := dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)
-	log.Debugf("table node err: %w", err)
-	if err != nil && !errors.Is(io.EOF, err) {
-		return false, err
-	}
-	c := Context{}
-
-	c = c.SingleChildContext(dec.rootMap)
-
-	// += function
-	err = dec.arrayAppend(c, fullPath, tableNodeValue)
-
-	return runAgainstCurrentExp, err
-}
@@ -1,52 +0,0 @@
-package yqlib
-
-import (
-	"bytes"
-	"io"
-	"net/url"
-)
-
-type uriDecoder struct {
-	reader       io.Reader
-	finished     bool
-	readAnything bool
-}
-
-func NewUriDecoder() Decoder {
-	return &uriDecoder{finished: false}
-}
-
-func (dec *uriDecoder) Init(reader io.Reader) error {
-	dec.reader = reader
-	dec.readAnything = false
-	dec.finished = false
-	return nil
-}
-
-func (dec *uriDecoder) Decode() (*CandidateNode, error) {
-	if dec.finished {
-		return nil, io.EOF
-	}
-
-	buf := new(bytes.Buffer)
-
-	if _, err := buf.ReadFrom(dec.reader); err != nil {
-		return nil, err
-	}
-	if buf.Len() == 0 {
-		dec.finished = true
-
-		// if we've read _only_ an empty string, lets return that
-		// otherwise if we've already read some bytes, and now we get
-		// an empty string, then we are done.
-		if dec.readAnything {
-			return nil, io.EOF
-		}
-	}
-	newValue, err := url.QueryUnescape(buf.String())
-	if err != nil {
-		return nil, err
-	}
-	dec.readAnything = true
-	return CreateStringScalarNode(newValue), nil
-}
@@ -0,0 +1,348 @@
+package yqlib
+
+import (
+	"encoding/xml"
+	"errors"
+	"io"
+	"strings"
+	"unicode"
+
+	"golang.org/x/net/html/charset"
+	yaml "gopkg.in/yaml.v3"
+)
+
+type xmlDecoder struct {
+	reader          io.Reader
+	readAnything    bool
+	attributePrefix string
+	contentName     string
+	strictMode      bool
+	keepNamespace   bool
+	useRawToken     bool
+	finished        bool
+}
+
+func NewXMLDecoder(attributePrefix string, contentName string, strictMode bool, keepNamespace bool, useRawToken bool) Decoder {
+	if contentName == "" {
+		contentName = "content"
+	}
+	return &xmlDecoder{
+		attributePrefix: attributePrefix,
+		contentName:     contentName,
+		finished:        false,
+		strictMode:      strictMode,
+		keepNamespace:   keepNamespace,
+		useRawToken:     useRawToken,
+	}
+}
+
+func (dec *xmlDecoder) Init(reader io.Reader) {
+	dec.reader = reader
+	dec.readAnything = false
+	dec.finished = false
+}
+
+func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yaml.Node, error) {
+	yamlNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+	for _, child := range nodes {
+		yamlChild, err := dec.convertToYamlNode(child)
+		if err != nil {
+			return nil, err
+		}
+		yamlNode.Content = append(yamlNode.Content, yamlChild)
+	}
+
+	return yamlNode, nil
+}
+
+func (dec *xmlDecoder) processComment(c string) string {
+	if c == "" {
+		return ""
+	}
+	return "#" + strings.TrimRight(c, " ")
+}
+
+func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
+	log.Debug("createMap: headC: %v, footC: %v", n.HeadComment, n.FootComment)
+	yamlNode := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
+
+	if len(n.Data) > 0 {
+		label := dec.contentName
+		labelNode := createScalarNode(label, label)
+		labelNode.HeadComment = dec.processComment(n.HeadComment)
+		labelNode.FootComment = dec.processComment(n.FootComment)
+		yamlNode.Content = append(yamlNode.Content, labelNode, createScalarNode(n.Data, n.Data))
+	}
+
+	for i, keyValuePair := range n.Children {
+		label := keyValuePair.K
+		children := keyValuePair.V
+		labelNode := createScalarNode(label, label)
+		var valueNode *yaml.Node
+		var err error
+
+		if i == 0 {
+			labelNode.HeadComment = dec.processComment(n.HeadComment)
+
+		}
+
+		// if i == len(n.Children)-1 {
+		labelNode.FootComment = dec.processComment(keyValuePair.FootComment)
+		// }
+
+		log.Debug("len of children in %v is %v", label, len(children))
+		if len(children) > 1 {
+			valueNode, err = dec.createSequence(children)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			// comment hack for maps of scalars
+			// if the value is a scalar, the head comment of the scalar needs to go on the key?
+			// add tests for <z/> as well as multiple <ds> of inputXmlWithComments > yaml
+			if len(children[0].Children) == 0 && children[0].HeadComment != "" {
+				labelNode.HeadComment = labelNode.HeadComment + "\n" + strings.TrimSpace(children[0].HeadComment)
+				children[0].HeadComment = ""
+			}
+			valueNode, err = dec.convertToYamlNode(children[0])
+			if err != nil {
+				return nil, err
+			}
+		}
+		yamlNode.Content = append(yamlNode.Content, labelNode, valueNode)
+	}
+
+	return yamlNode, nil
+}
+
+func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yaml.Node, error) {
+	if len(n.Children) > 0 {
+		return dec.createMap(n)
+	}
+	scalar := createScalarNode(n.Data, n.Data)
+	if n.Data == "" {
+		scalar = createScalarNode(nil, "")
+	}
+	log.Debug("scalar headC: %v, footC: %v", n.HeadComment, n.FootComment)
+	scalar.HeadComment = dec.processComment(n.HeadComment)
+	scalar.LineComment = dec.processComment(n.LineComment)
+	scalar.FootComment = dec.processComment(n.FootComment)
+
+	return scalar, nil
+}
+
+func (dec *xmlDecoder) Decode(rootYamlNode *yaml.Node) error {
+	if dec.finished {
+		return io.EOF
+	}
+	root := &xmlNode{}
+	// cant use xj - it doesn't keep map order.
+	err := dec.decodeXML(root)
+
+	if err != nil {
+		return err
+	}
+	firstNode, err := dec.convertToYamlNode(root)
+
+	if err != nil {
+		return err
+	} else if firstNode.Tag == "!!null" {
+		dec.finished = true
+		if dec.readAnything {
+			return io.EOF
+		}
+	}
+	dec.readAnything = true
+	rootYamlNode.Kind = yaml.DocumentNode
+	rootYamlNode.Content = []*yaml.Node{firstNode}
+	dec.finished = true
+	return nil
+}
+
+type xmlNode struct {
+	Children    []*xmlChildrenKv
+	HeadComment string
+	FootComment string
+	LineComment string
+	Data        string
+}
+
+type xmlChildrenKv struct {
+	K           string
+	V           []*xmlNode
+	FootComment string
+}
+
+// AddChild appends a node to the list of children
+func (n *xmlNode) AddChild(s string, c *xmlNode) {
+
+	if n.Children == nil {
+		n.Children = make([]*xmlChildrenKv, 0)
+	}
+	log.Debug("looking for %s", s)
+	// see if we can find an existing entry to add to
+	for _, childEntry := range n.Children {
+		if childEntry.K == s {
+			log.Debug("found it, appending an entry%s", s)
+			childEntry.V = append(childEntry.V, c)
+			log.Debug("yay len of children in %v is %v", s, len(childEntry.V))
+			return
+		}
+	}
+	log.Debug("not there, making a new one %s", s)
+	n.Children = append(n.Children, &xmlChildrenKv{K: s, V: []*xmlNode{c}})
+}
+
+type element struct {
+	parent *element
+	n      *xmlNode
+	label  string
+	state  string
+}
+
+// this code is heavily based on https://github.com/basgys/goxml2json
+// main changes are to decode into a structure that preserves the original order
+// of the map keys.
+func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
+	xmlDec := xml.NewDecoder(dec.reader)
+	xmlDec.Strict = dec.strictMode
+	// That will convert the charset if the provided XML is non-UTF-8
+	xmlDec.CharsetReader = charset.NewReaderLabel
+
+	// Create first element from the root node
+	elem := &element{
+		parent: nil,
+		n:      root,
+	}
+
+	getToken := func() (xml.Token, error) {
+		if dec.useRawToken {
+			return xmlDec.RawToken()
+		}
+		return xmlDec.Token()
+	}
+
+	for {
+		t, e := getToken()
+		if e != nil && !errors.Is(e, io.EOF) {
+			return e
+		}
+		if t == nil {
+			break
+		}
+
+		switch se := t.(type) {
+		case xml.StartElement:
+			log.Debug("start element %v", se.Name.Local)
+			elem.state = "started"
+			// Build new a new current element and link it to its parent
+			elem = &element{
+				parent: elem,
+				n:      &xmlNode{},
+				label:  se.Name.Local,
+			}
+
+			// Extract attributes as children
+			for _, a := range se.Attr {
+				if dec.keepNamespace {
+					if a.Name.Space != "" {
+						a.Name.Local = a.Name.Space + ":" + a.Name.Local
+					}
+				}
+				elem.n.AddChild(dec.attributePrefix+a.Name.Local, &xmlNode{Data: a.Value})
+			}
+		case xml.CharData:
+			// Extract XML data (if any)
+			elem.n.Data = trimNonGraphic(string(se))
+			if elem.n.Data != "" {
+				elem.state = "chardata"
+				log.Debug("chardata [%v] for %v", elem.n.Data, elem.label)
+			}
+		case xml.EndElement:
+			log.Debug("end element %v", elem.label)
+			elem.state = "finished"
+			// And add it to its parent list
+			if elem.parent != nil {
+				elem.parent.n.AddChild(elem.label, elem.n)
+			}
+
+			// Then change the current element to its parent
+			elem = elem.parent
+		case xml.Comment:
+
+			commentStr := string(xml.CharData(se))
+			if elem.state == "started" {
+				applyFootComment(elem, commentStr)
+
+			} else if elem.state == "chardata" {
+				log.Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
+				elem.n.LineComment = joinFilter([]string{elem.n.LineComment, commentStr})
+			} else {
+				log.Debug("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
+				elem.n.HeadComment = joinFilter([]string{elem.n.HeadComment, commentStr})
+			}
+
+		}
+	}
+
+	return nil
+}
+
+func applyFootComment(elem *element, commentStr string) {
+
+	// first lets try to put the comment on the last child
+	if len(elem.n.Children) > 0 {
+		lastChildIndex := len(elem.n.Children) - 1
+		childKv := elem.n.Children[lastChildIndex]
+		log.Debug("got a foot comment for %v: [%v]", childKv.K, commentStr)
+		childKv.FootComment = joinFilter([]string{elem.n.FootComment, commentStr})
+	} else {
+		log.Debug("got a foot comment for %v: [%v]", elem.label, commentStr)
+		elem.n.FootComment = joinFilter([]string{elem.n.FootComment, commentStr})
+	}
+}
+
+func joinFilter(rawStrings []string) string {
+	stringsToJoin := make([]string, 0)
+	for _, str := range rawStrings {
+		if str != "" {
+			stringsToJoin = append(stringsToJoin, str)
+		}
+	}
+	return strings.Join(stringsToJoin, " ")
+}
+
+// trimNonGraphic returns a slice of the string s, with all leading and trailing
+// non graphic characters and spaces removed.
+//
+// Graphic characters include letters, marks, numbers, punctuation, symbols,
+// and spaces, from categories L, M, N, P, S, Zs.
+// Spacing characters are set by category Z and property Pattern_White_Space.
+func trimNonGraphic(s string) string {
+	if s == "" {
+		return s
+	}
+
+	var first *int
+	var last int
+	for i, r := range []rune(s) {
+		if !unicode.IsGraphic(r) || unicode.IsSpace(r) {
+			continue
+		}
+
+		if first == nil {
+			f := i // copy i
+			first = &f
+			last = i
+		} else {
+			last = i
+		}
+	}
+
+	// If first is nil, it means there are no graphic characters
+	if first == nil {
+		return ""
+	}
+
+	return string([]rune(s)[*first : last+1])
+}
@@ -1,157 +1,23 @@
 package yqlib

 import (
-	"bufio"
-	"bytes"
-	"errors"
 	"io"
-	"regexp"
-	"strings"

 	yaml "gopkg.in/yaml.v3"
 )

 type yamlDecoder struct {
 	decoder yaml.Decoder
-
-	prefs YamlPreferences
-
-	// work around of various parsing issues by yaml.v3 with document headers
-	leadingContent string
-	bufferRead     bytes.Buffer
-
-	// anchor map persists over multiple documents for convenience.
-	anchorMap map[string]*CandidateNode
-
-	readAnything  bool
-	firstFile     bool
-	documentIndex uint
 }

-func NewYamlDecoder(prefs YamlPreferences) Decoder {
-	return &yamlDecoder{prefs: prefs, firstFile: true}
+func NewYamlDecoder() Decoder {
+	return &yamlDecoder{}
 }

-func (dec *yamlDecoder) processReadStream(reader *bufio.Reader) (io.Reader, string, error) {
-	var commentLineRegEx = regexp.MustCompile(`^\s*#`)
-	var yamlDirectiveLineRegEx = regexp.MustCompile(`^\s*%YA`)
-	var sb strings.Builder
-	for {
-		peekBytes, err := reader.Peek(4)
-		if errors.Is(err, io.EOF) {
-			// EOF are handled else where..
-			return reader, sb.String(), nil
-		} else if err != nil {
-			return reader, sb.String(), err
-		} else if string(peekBytes[0]) == "\n" {
-			_, err := reader.ReadString('\n')
-			sb.WriteString("\n")
-			if errors.Is(err, io.EOF) {
-				return reader, sb.String(), nil
-			} else if err != nil {
-				return reader, sb.String(), err
-			}
-		} else if string(peekBytes) == "--- " {
-			_, err := reader.ReadString(' ')
-			sb.WriteString("$yqDocSeparator$\n")
-			if errors.Is(err, io.EOF) {
-				return reader, sb.String(), nil
-			} else if err != nil {
-				return reader, sb.String(), err
-			}
-		} else if string(peekBytes) == "---\n" {
-			_, err := reader.ReadString('\n')
-			sb.WriteString("$yqDocSeparator$\n")
-			if errors.Is(err, io.EOF) {
-				return reader, sb.String(), nil
-			} else if err != nil {
-				return reader, sb.String(), err
-			}
-		} else if commentLineRegEx.MatchString(string(peekBytes)) || yamlDirectiveLineRegEx.MatchString(string(peekBytes)) {
-			line, err := reader.ReadString('\n')
-			sb.WriteString(line)
-			if errors.Is(err, io.EOF) {
-				return reader, sb.String(), nil
-			} else if err != nil {
-				return reader, sb.String(), err
-			}
-		} else {
-			return reader, sb.String(), nil
-		}
-	}
+func (dec *yamlDecoder) Init(reader io.Reader) {
+	dec.decoder = *yaml.NewDecoder(reader)
 }

-func (dec *yamlDecoder) Init(reader io.Reader) error {
-	readerToUse := reader
-	leadingContent := ""
-	dec.bufferRead = bytes.Buffer{}
-	var err error
-	// if we 'evaluating together' - we only process the leading content
-	// of the first file - this ensures comments from subsequent files are
-	// merged together correctly.
-	if dec.prefs.LeadingContentPreProcessing && (!dec.prefs.EvaluateTogether || dec.firstFile) {
-		readerToUse, leadingContent, err = dec.processReadStream(bufio.NewReader(reader))
-		if err != nil {
-			return err
-		}
-	} else if !dec.prefs.LeadingContentPreProcessing {
-		// if we're not process the leading content
-		// keep a copy of what we've read. This is incase its a
-		// doc with only comments - the decoder will return nothing
-		// then we can read the comments from bufferRead
-		readerToUse = io.TeeReader(reader, &dec.bufferRead)
-	}
-	dec.leadingContent = leadingContent
-	dec.readAnything = false
-	dec.decoder = *yaml.NewDecoder(readerToUse)
-	dec.firstFile = false
-	dec.documentIndex = 0
-	dec.anchorMap = make(map[string]*CandidateNode)
-	return nil
-}
-
-func (dec *yamlDecoder) Decode() (*CandidateNode, error) {
-	var yamlNode yaml.Node
-	err := dec.decoder.Decode(&yamlNode)
-
-	if errors.Is(err, io.EOF) && dec.leadingContent != "" && !dec.readAnything {
-		// force returning an empty node with a comment.
-		dec.readAnything = true
-		return dec.blankNodeWithComment(), nil
-	} else if errors.Is(err, io.EOF) && !dec.prefs.LeadingContentPreProcessing && !dec.readAnything {
-		// didn't find any yaml,
-		// check the tee buffer, maybe there were comments
-		dec.readAnything = true
-		dec.leadingContent = dec.bufferRead.String()
-		if dec.leadingContent != "" {
-			return dec.blankNodeWithComment(), nil
-		}
-		return nil, err
-	} else if err != nil {
-		return nil, err
-	}
-
-	candidateNode := CandidateNode{document: dec.documentIndex}
-	// don't bother with the DocumentNode
-	err = candidateNode.UnmarshalYAML(yamlNode.Content[0], dec.anchorMap)
-	if err != nil {
-		return nil, err
-	}
-
-	candidateNode.HeadComment = yamlNode.HeadComment + candidateNode.HeadComment
-	candidateNode.FootComment = yamlNode.FootComment + candidateNode.FootComment
-
-	if dec.leadingContent != "" {
-		candidateNode.LeadingContent = dec.leadingContent
-		dec.leadingContent = ""
-	}
-	dec.readAnything = true
-	dec.documentIndex++
-	return &candidateNode, nil
-}
-
-func (dec *yamlDecoder) blankNodeWithComment() *CandidateNode {
-	node := CreateScalarNode(nil, "")
-	node.LeadingContent = dec.leadingContent
-	return node
+func (dec *yamlDecoder) Decode(rootYamlNode *yaml.Node) error {
+	return dec.decoder.Decode(rootYamlNode)
 }
@@ -0,0 +1,6 @@
+
+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
@@ -9,6 +9,12 @@ Add behaves differently according to the type of the LHS:
 Use `+=` as a relative append assign for things like increment. Note that `.a += .x` is equivalent to running `.a = .a + .x`.


+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
 ## Concatenate arrays
 Given a sample.yml file of:
 ```yaml
@@ -86,23 +92,6 @@ will output
 a: ['dog', 'cat']
 ```

-## Prepend to existing array
-Given a sample.yml file of:
-```yaml
-a:
-  - dog
-```
-then
-```bash
-yq '.a = ["cat"] + .a' sample.yml
-```
-will output
-```yaml
-a:
-  - cat
-  - dog
-```
-
 ## Add new object to array
 Given a sample.yml file of:
 ```yaml
@@ -2,6 +2,12 @@

 This operator is used to provide alternative (or default) values when a particular expression is either null or false.

+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
 ## LHS is defined
 Given a sample.yml file of:
 ```yaml
@@ -73,36 +79,3 @@ will output
 cat
 ```

-## Update or create - entity exists
-This initialises `a` if it's not present
-
-Given a sample.yml file of:
-```yaml
-a: 1
-```
-then
-```bash
-yq '(.a // (.a = 0)) += 1' sample.yml
-```
-will output
-```yaml
-a: 2
-```
-
-## Update or create - entity does not exist
-This initialises `a` if it's not present
-
-Given a sample.yml file of:
-```yaml
-b: camel
-```
-then
-```bash
-yq '(.a // (.a = 0)) += 1' sample.yml
-```
-will output
-```yaml
-b: camel
-a: 1
-```
-
@@ -5,6 +5,12 @@ Use the `alias` and `anchor` operators to read and write yaml aliases and anchor
 `yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.


+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
 ## Merge one map
 see https://yaml.org/type/merge.html

@@ -302,7 +308,7 @@ foobar:
 ```

 ## Dereference and update a field
-Use explode with multiply to dereference an object
+`Use explode with multiply to dereference an object

 Given a sample.yml file of:
 ```yaml
@@ -1,28 +0,0 @@
-# Array to Map
-
-Use this operator to convert an array to..a map. The indices are used as map keys, null values in the array are skipped over.
-
-Behind the scenes, this is implemented using reduce:
-
-```
-(.[] | select(. != null) ) as $i ireduce({}; .[$i | key] = $i)
-```
-
-## Simple example
-Given a sample.yml file of:
-```yaml
-cool:
-  - null
-  - null
-  - hello
-```
-then
-```bash
-yq '.cool |= array_to_map' sample.yml
-```
-will output
-```yaml
-cool:
-  2: hello
-```
-
@@ -3,14 +3,15 @@
 This operator is used to update node values. It can be used in either the:

 ### plain form: `=`
-Which will set the LHS node values equal to the RHS node values. The RHS expression is run against the matching nodes in the pipeline.
+Which will assign the LHS node values to the RHS node values. The RHS expression is run against the matching nodes in the pipeline.

 ### relative form: `|=`
-This will do a similar thing to the plain form, but the RHS expression is run with _each LHS node as context_. This is useful for updating values based on old values, e.g. increment.
+This will do a similar thing to the plain form, however, the RHS expression is run against _the LHS nodes_. This is useful for updating values based on old values, e.g. increment.
+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;

-
-### Flags
- `c` clobber custom tags
+`yq e <exp> <file>`
+{% endhint %}

 ## Create yaml file
 Running
@@ -203,7 +204,7 @@ a:
 ```

 ## Update node value that has an anchor
-Anchor will remain
+Anchor will remaple

 Given a sample.yml file of:
 ```yaml
@@ -234,37 +235,3 @@ a:
    - bogs
 ```

-## Custom types are maintained by default
-Given a sample.yml file of:
-```yaml
-a: !cat meow
-b: !dog woof
-```
-then
-```bash
-yq '.a = .b' sample.yml
-```
-will output
-```yaml
-a: !cat woof
-b: !dog woof
-```
-
-## Custom types: clobber
-Use the `c` option to clobber custom tags
-
-Given a sample.yml file of:
-```yaml
-a: !cat meow
-b: !dog woof
-```
-then
-```bash
-yq '.a =c .b' sample.yml
-```
-will output
-```yaml
-a: !dog woof
-b: !dog woof
-```
-
@@ -4,7 +4,7 @@ The `or` and `and` operators take two parameters and return a boolean result.

 `not` flips a boolean from true to false, or vice versa. 

-`any` will return `true` if there are any `true` values in an array sequence, and `all` will return true if _all_ elements in an array are true.
+`any` will return `true` if there are any `true` values in a array sequence, and `all` will return true if _all_ elements in an array are true.

 `any_c(condition)` and `all_c(condition)` are like `any` and `all` but they take a condition expression that is used against each element to determine if it's `true`. Note: in `jq` you can simply pass a condition to `any` or `all` and it simply works - `yq` isn't that clever..yet

@@ -16,6 +16,12 @@ These are most commonly used with the `select` operator to filter particular nod
 - comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare)
 - select operator [here](https://mikefarah.gitbook.io/yq/operators/select)

+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
 ## `or` example
 Running
 ```bash
@@ -26,24 +32,6 @@ will output
 true
 ```

-## "yes" and "no" are strings
-In the yaml 1.2 standard, support for yes/no as booleans was dropped - they are now considered strings. See '10.2.1.2. Boolean' in https://yaml.org/spec/1.2.2/
-
-Given a sample.yml file of:
-```yaml
- yes
- no
-```
-then
-```bash
-yq '.[] | tag' sample.yml
-```
-will output
-```yaml
-!!str
-!!str
-```
-
 ## `and` example
 Running
 ```bash
@@ -3,6 +3,12 @@
 This creates an array using the expression between the square brackets.


+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
 ## Collect empty
 Running
 ```bash
@@ -2,7 +2,11 @@

 Returns the column of the matching node. Starts from 1, 0 indicates there was no column data.

-Column is the number of characters that precede that node on the line it starts.
+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}

 ## Returns column of _value_ node
 Given a sample.yml file of:
@@ -1,18 +1,22 @@
 # Comment Operators

-Use these comment operators to set or retrieve comments. Note that line comments on maps/arrays are actually set on the _key_ node as opposed to the _value_ (map/array). See below for examples.
+Use these comment operators to set or retrieve comments.

 Like the `=` and `|=` assign operators, the same syntax applies when updating comments:

 ### plain form: `=`
-This will set the LHS nodes' comments equal to the expression on the RHS. The RHS is run against the matching nodes in the pipeline
+This will assign the LHS nodes comments to the expression on the RHS. The RHS is run against the matching nodes in the pipeline

 ### relative form: `|=` 
-This is similar to the plain form, but it evaluates the RHS with _each matching LHS node as context_. This is useful if you want to set the comments as a relative expression of the node, for instance its value or path.
+Similar to the plain form, however the RHS evaluates against each matching LHS node! This is useful if you want to set the comments as a relative expression of the node, for instance its value or path.
+
+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}

 ## Set line comment
-Set the comment on the key node for more reliability (see below).
-
 Given a sample.yml file of:
 ```yaml
 a: cat
@@ -26,24 +30,6 @@ will output
 a: cat # single
 ```

-## Set line comment of a maps/arrays
-For maps and arrays, you need to set the line comment on the _key_ node. This will also work for scalars.
-
-Given a sample.yml file of:
-```yaml
-a:
-  b: things
-```
-then
-```bash
-yq '(.a | key) line_comment="single"' sample.yml
-```
-will output
-```yaml
-a: # single
-  b: things
-```
-
 ## Use update assign to perform relative updates
 Given a sample.yml file of:
 ```yaml
@@ -60,121 +46,6 @@ a: cat # cat
 b: dog # dog
 ```

-## Where is the comment - map key example
-The underlying yaml parser can assign comments in a document to surprising nodes. Use an expression like this to find where you comment is. 'p' indicates the path, 'isKey' is if the node is a map key (as opposed to a map value).
-From this, you can see the 'hello-world-comment' is actually on the 'hello' key
-
-Given a sample.yml file of:
-```yaml
-hello: # hello-world-comment
-  message: world
-```
-then
-```bash
-yq '[... | {"p": path | join("."), "isKey": is_key, "hc": headComment, "lc": lineComment, "fc": footComment}]' sample.yml
-```
-will output
-```yaml
- p: ""
-  isKey: false
-  hc: ""
-  lc: ""
-  fc: ""
- p: hello
-  isKey: true
-  hc: ""
-  lc: hello-world-comment
-  fc: ""
- p: hello
-  isKey: false
-  hc: ""
-  lc: ""
-  fc: ""
- p: hello.message
-  isKey: true
-  hc: ""
-  lc: ""
-  fc: ""
- p: hello.message
-  isKey: false
-  hc: ""
-  lc: ""
-  fc: ""
-```
-
-## Retrieve comment - map key example
-From the previous example, we know that the comment is on the 'hello' _key_ as a lineComment
-
-Given a sample.yml file of:
-```yaml
-hello: # hello-world-comment
-  message: world
-```
-then
-```bash
-yq '.hello | key | line_comment' sample.yml
-```
-will output
-```yaml
-hello-world-comment
-```
-
-## Where is the comment - array example
-The underlying yaml parser can assign comments in a document to surprising nodes. Use an expression like this to find where you comment is. 'p' indicates the path, 'isKey' is if the node is a map key (as opposed to a map value).
-From this, you can see the 'under-name-comment' is actually on the first child
-
-Given a sample.yml file of:
-```yaml
-name:
-  # under-name-comment
-  - first-array-child
-```
-then
-```bash
-yq '[... | {"p": path | join("."), "isKey": is_key, "hc": headComment, "lc": lineComment, "fc": footComment}]' sample.yml
-```
-will output
-```yaml
- p: ""
-  isKey: false
-  hc: ""
-  lc: ""
-  fc: ""
- p: name
-  isKey: true
-  hc: ""
-  lc: ""
-  fc: ""
- p: name
-  isKey: false
-  hc: ""
-  lc: ""
-  fc: ""
- p: name.0
-  isKey: false
-  hc: under-name-comment
-  lc: ""
-  fc: ""
-```
-
-## Retrieve comment - array example
-From the previous example, we know that the comment is on the first child as a headComment
-
-Given a sample.yml file of:
-```yaml
-name:
-  # under-name-comment
-  - first-array-child
-```
-then
-```bash
-yq '.name[0] | headComment' sample.yml
-```
-will output
-```yaml
-under-name-comment
-```
-
 ## Set head comment
 Given a sample.yml file of:
 ```yaml
@@ -187,26 +58,8 @@ yq '. head_comment="single"' sample.yml
 will output
 ```yaml
 # single
-a: cat
-```

-## Set head comment of a map entry
-Given a sample.yml file of:
-```yaml
-f: foo
-a:
-  b: cat
-```
-then
-```bash
-yq '(.a | key) head_comment="single"' sample.yml
-```
-will output
-```yaml
-f: foo
-# single
-a:
-  b: cat
+a: cat
 ```

 ## Set foot comment, using an expression
@@ -245,8 +98,6 @@ Note the use of `...` to ensure key nodes are included.

 Given a sample.yml file of:
 ```yaml
-# hi
-
 a: cat # comment
 # great
 b: # key comment
@@ -264,8 +115,6 @@ b:
 ## Get line comment
 Given a sample.yml file of:
 ```yaml
-# welcome!
-
 a: cat # meow
 # have a great day
 ```
@@ -294,7 +143,6 @@ yq '. | head_comment' sample.yml
 will output
 ```yaml
 welcome!
-
 ```

 ## Head comment with document split
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Mike Farah	d8f8661567	Fixing from review	2022-08-01 10:16:59 +10:00
Mike Farah	9628aaf8c5	Added CSV roundtrip	2022-08-01 10:05:26 +10:00
Mike Farah	cb0bae128f	Adding CSV decoder	2022-07-31 17:35:36 +10:00
Mike Farah	183a42c249	WIP: adding CSV decoder	2022-07-31 17:31:27 +10:00