Fixing from review

Added CSV roundtrip
Adding CSV decoder
2026-08-24 16:39:58 +08:00 · 2022-08-01 10:16:59 +10:00 · 2022-08-01 10:05:26 +10:00 · 2022-07-31 17:35:36 +10:00 · 2022-07-31 17:31:27 +10:00
263 changed files with 1593 additions and 8668 deletions
@@ -60,23 +60,3 @@ jobs:
            -t "${IMAGE_NAME}:4" \
            -t "${IMAGE_NAME}:latest" \
            .
-          
-          cd github-action
-          docker buildx build \
-            --label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
-            --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
-            --label "org.opencontainers.image.description=yq is a portable command-line YAML processor" \
-            --label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
-            --label "org.opencontainers.image.licenses=MIT" \
-            --label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \
-            --label "org.opencontainers.image.source=https://github.com/mikefarah/yq" \
-            --label "org.opencontainers.image.title=yq" \
-            --label "org.opencontainers.image.url=https://mikefarah.gitbook.io/yq/" \
-            --label "org.opencontainers.image.version=${IMAGE_VERSION}" \
-            --platform "${PLATFORMS}" \
-            --pull \
-            --push \
-            -t "${IMAGE_NAME}:${IMAGE_VERSION}-githubaction" \
-            -t "${IMAGE_NAME}:4-githubaction" \
-            -t "${IMAGE_NAME}:latest-githubaction" \
-            .
@@ -10,10 +10,10 @@ jobs:
    runs-on: ubuntu-latest
    steps:

-    - name: Set up Go
-      uses: actions/setup-go@v4
+    - name: Set up Go 1.18
+      uses: actions/setup-go@v3
      with:
-        go-version: '^1.20'
+        go-version: 1.18
      id: go

    - name: Check out code into the Go module directory
@@ -10,18 +10,18 @@ jobs:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@v3
        with:
-          go-version: '^1.20'
-          check-latest: true
+          go-version: '^1.18'
      - name: Compile man page markup
        id: gen-man-page-md
        run: |
          ./scripts/generate-man-page-md.sh
+          echo "::set-output name=man-page-md::man.md"

      - name: Get the version
        id: get_version
-        run: echo "VERSION=${GITHUB_REF##*/}" >> "${GITHUB_OUTPUT}"
+        run: echo ::set-output name=VERSION::${GITHUB_REF##*/}

      - name: Generate man page
        uses: docker://pandoc/core:2.14.2
@@ -35,7 +35,7 @@ jobs:
            --variable=header:"yq (https://github.com/mikefarah/yq/) version ${{ steps.get_version.outputs.VERSION }}"
            --variable=author:"Mike Farah"
            --output=yq.1
-            man.md
+            ${{ steps.gen-man-page-md.outputs.man-page-md }}

      - name: Cross compile
        run: |
@@ -1,23 +0,0 @@
-name: Release Snap
-
-on:
-  release:
-    types: [released]
-
-  # Allows you to run this workflow manually from the Actions tab
-  workflow_dispatch:
-
-jobs:
-    buildSnap:
-      environment: snap
-      runs-on: ubuntu-latest
-      steps:
-        - uses: actions/checkout@v3
-        - uses: snapcore/action-build@v1
-          id: build
-        - uses: snapcore/action-publish@v1
-          env:
-            SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.STORE_LOGIN }}
-          with:
-            snap: ${{ steps.build.outputs.snap }}
-            release: stable
@@ -1,8 +1,5 @@
 name: Test Yq Action
-on: 
-  push:
-  workflow_dispatch:
-
+on: [push]
 permissions:
  contents: read

@@ -15,14 +12,11 @@ jobs:
    steps:
      - uses: actions/checkout@v3
      - name: Get test
-        id: get_value
        uses: mikefarah/yq@master
        with:
-          cmd: yq '.test' examples/multiline-text.yaml
-      - name: Multiline test
-        run: echo "### It was [${{ steps.get_value.outputs.result }}]" >> $GITHUB_STEP_SUMMARY
+          cmd: yq eval '.a' examples/sample.yaml
      - name: Write inplace test
        id: lookupSdkVersion
        uses: mikefarah/yq@master
        with:
-          cmd: yq -i '.b.c = 5' examples/sample.yaml
+          cmd: yq eval -i '.b.c = 5' examples/sample.yaml
@@ -41,8 +41,6 @@ yq*.snap

 test.yml
 test*.yml
-test*.xml
-test*.toml
 test*.yaml
 0.yml
 1.yml
@@ -58,9 +56,4 @@ _build
 debian/files

 # intellij
-/.idea
-
-# vscode
-.vscode
-
-yq3
+/.idea
@@ -1,6 +1,6 @@
 # Development

-1. Install (Golang)[https://golang.org/]
+1. Install (golang)[https://golang.org/]
 1. Run  `scripts/devtools.sh` to install the required devtools
 2. Run `make [local] vendor` to install the vendor dependencies
 2. Run `make [local] test` to ensure you can run the existing tests
@@ -12,7 +12,7 @@

 # Documentation

-The documentation is a bit of a mixed bag (sorry in advance, I do plan on simplifying it...) - with some parts automatically generated and stiched together and some statically defined.
+The documentation is a bit of a mixed bag (sorry in advanced, I do plan on simplifying it...) - with some parts automatically generated and stiched together and some statically defined.

 Documentation is written in markdown, and is published in the 'gitbook' branch.

@@ -1,10 +1,10 @@
-FROM golang:1.20.5 as builder
+FROM golang:1.18.4 as builder

 WORKDIR /go/src/mikefarah/yq

 COPY . .

-RUN CGO_ENABLED=0 go build -ldflags "-s -w" .
+RUN CGO_ENABLED=0 go build .
 # RUN ./scripts/test.sh -- this too often times out in the github pipeline.
 RUN ./scripts/acceptance.sh

@@ -1,4 +1,4 @@
-FROM golang:1.20.5
+FROM golang:1.18.4

 COPY scripts/devtools.sh /opt/devtools.sh

@@ -7,7 +7,6 @@ export GIT_DESCRIBE = $(shell git describe --tags --always)
 LDFLAGS :=
 LDFLAGS += -X main.GitCommit=${GIT_COMMIT}${GIT_DIRTY}
 LDFLAGS += -X main.GitDescribe=${GIT_DESCRIBE}
-LDFLAGS += -w -s

 GITHUB_TOKEN ?=

@@ -3,10 +3,31 @@
 ![Build](https://github.com/mikefarah/yq/workflows/Build/badge.svg)  ![Docker Pulls](https://img.shields.io/docker/pulls/mikefarah/yq.svg) ![Github Releases (by Release)](https://img.shields.io/github/downloads/mikefarah/yq/total.svg) ![Go Report](https://goreportcard.com/badge/github.com/mikefarah/yq) ![CodeQL](https://github.com/mikefarah/yq/workflows/CodeQL/badge.svg)


-a lightweight and portable command-line YAML, JSON and XML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json, xml, properties, csv and tsv. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.
+a lightweight and portable command-line YAML, JSON and XML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json and xml. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.

 yq is written in go - so you can download a dependency free binary for your platform and you are good to go! If you prefer there are a variety of package managers that can be used as well as Docker and Podman, all listed below.

+## Notice for v4.x versions prior to 4.18.1
+Since 4.18.1, yq's 'eval/e' command is the _default_ command and no longer needs to be specified.
+
+Older versions will still need to specify 'eval/e'.
+
+Similarly, '-' is no longer required as a filename to read from STDIN (unless reading from one or more files).
+
+TLDR:
+
+Prior to 4.18.1 
+```bash
+yq e '.cool' - < file.yaml
+```
+
+4.18+ 
+```bash
+yq '.cool' < file.yaml
+```
+
+When merging multiple files together, `eval-all/ea` is still required to tell `yq` to run the expression against all the document at once.
+
 ## Quick Usage Guide

 Read a value:
@@ -47,22 +68,17 @@ yq -i '

 Convert JSON to YAML
 ```bash
-yq -Poy sample.json
+yq -P sample.json
 ```

 See the [documentation](https://mikefarah.gitbook.io/yq/) for more examples.

-Take a look at the discussions for [common questions](https://github.com/mikefarah/yq/discussions/categories/q-a), and [cool ideas](https://github.com/mikefarah/yq/discussions/categories/show-and-tell)
-
 ## Install

 ### [Download the latest binary](https://github.com/mikefarah/yq/releases/latest)

 ### wget
-Use wget to download, gzipped pre-compiled binaries:
-
-
-For instance, VERSION=v4.2.0 and BINARY=yq_linux_amd64
+Use wget to download the pre-compiled binaries:

 #### Compressed via tar.gz
 ```bash
@@ -77,12 +93,7 @@ wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -O /
    chmod +x /usr/bin/yq
 ```

-#### Latest version
-
-```bash
-wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq &&\
-    chmod +x /usr/bin/yq
-```
+For instance, VERSION=v4.2.0 and BINARY=yq_linux_amd64

 ### MacOS / Linux via Homebrew:
 Using [Homebrew](https://brew.sh/)
@@ -196,27 +207,12 @@ RUN apk add --no-cache tzdata
 USER yq
 ```

-#### Podman with SELinux
-
-If you are using podman with SELinux, you will need to set the shared volume flag `:z` on the volume mount:
-
-```
-v "${PWD}":/workdir:z
-```
-
 ### GitHub Action
 ```
  - name: Set foobar to cool
    uses: mikefarah/yq@master
    with:
      cmd: yq -i '.foo.bar = "cool"' 'config.yml'
-  - name: Get an entry with a variable that might contain dots or spaces
-    id: get_username
-    uses: mikefarah/yq@master
-    with:
-      cmd: yq '.all.children.["${{ matrix.ip_address }}"].username' ops/inventories/production.yml
-  - name: Reuse a variable obtained in another step
-    run: echo ${{ steps.get_username.outputs.result }}
 ```

 See https://mikefarah.gitbook.io/yq/usage/github-action for more.
@@ -229,19 +225,7 @@ go install github.com/mikefarah/yq/v4@latest
 ## Community Supported Installation methods
 As these are supported by the community :heart: - however, they may be out of date with the officially supported releases.

-_Please note that the Debian package (previously supported by @rmescandon) is no longer maintained. Please use an alternative installation method._
-
-
-### Nix
-
-```
-nix profile install nixpkgs#yq-go
-```
-
-See [here](https://search.nixos.org/packages?channel=unstable&show=yq-go&from=0&size=50&sort=relevance&type=packages&query=yq-go)
-
-
-### Webi
+# Webi

 ```
 webi yq
@@ -264,13 +248,6 @@ choco install yq
 ```
 Supported by @chillum (https://chocolatey.org/packages/yq)

-and
-
-### Winget
-winget install yq
-
-https://winget.run/pkg/MikeFarah/yq
-
 ### Mac:
 Using [MacPorts](https://www.macports.org/)
 ```
@@ -288,6 +265,15 @@ Supported by Tuan Hoang
 https://pkgs.alpinelinux.org/package/edge/community/x86/yq


+### On Ubuntu 16.04 or higher from Debian package:
+```sh
+sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64
+sudo add-apt-repository ppa:rmescandon/yq
+sudo apt update
+sudo apt install yq -y
+```
+Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
+
 ## Features
 - [Detailed documentation with many examples](https://mikefarah.gitbook.io/yq/)
 - Written in portable go, so you can download a lovely dependency free binary
@@ -304,10 +290,10 @@ https://pkgs.alpinelinux.org/package/edge/community/x86/yq
 - Keeps yaml formatting and comments when updating (though there are issues with whitespace)
 - [Decode/Encode base64 data](https://mikefarah.gitbook.io/yq/operators/encode-decode)
 - [Load content from other files](https://mikefarah.gitbook.io/yq/operators/load)
- [Convert to/from json/ndjson](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
+- [Convert to/from json](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
 - [Convert to/from xml](https://mikefarah.gitbook.io/yq/v/v4.x/usage/xml)
 - [Convert to/from properties](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties)
- [Convert to/from csv/tsv](https://mikefarah.gitbook.io/yq/usage/csv-tsv)
+- [Convert to csv/tsv](https://mikefarah.gitbook.io/yq/usage/csv-tsv)
 - [General shell completion scripts (bash/zsh/fish/powershell)](https://mikefarah.gitbook.io/yq/v/v4.x/commands/shell-completion)
 - [Reduce](https://mikefarah.gitbook.io/yq/operators/reduce) to merge multiple files or sum an array or other fancy things.
 - [Github Action](https://mikefarah.gitbook.io/yq/usage/github-action) to use in your automated pipeline (thanks @devorbitus)
@@ -362,6 +348,5 @@ Use "yq [command] --help" for more information about a command.
 ## Known Issues / Missing Features
 - `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details)
 - Powershell has its own...[opinions on quoting yq](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks#quotes-in-windows-powershell)
- "yes", "no" were dropped as boolean values in the yaml 1.2 standard - which is the standard yq assumes.

 See [tips and tricks](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks) for more common problems and solutions.
@@ -9,7 +9,7 @@ EOL

 testEmptyEval() {
  X=$(./yq e test.yml)
-  expected="# comment"
+  expected=$(cat test.yml)
  assertEquals 0 $?
  assertEquals "$expected" "$X"
 }
@@ -1,43 +0,0 @@
-#!/bin/bash
-
-setUp() {
-  rm test*.yml || true
-  
-}
-
-testLineCountFirstLineComment() {
-  cat >test.yml <<EOL
-#test123 
-abc: 123 
-test123: 123123 
-#comment 
-lalilu: lalilu
-EOL
-
-  X=$(./yq '.lalilu | line' --header-preprocess=false < test.yml)
-  assertEquals "5" "$X"
-}
-
-testArrayOfDocs() {
-  cat >test.yml <<EOL
---
-# leading comment doc 1
-a: 1
---
-# leading comment doc 2
-a: 2
-EOL
-
-read -r -d '' expected << EOM
- # leading comment doc 1
-  a: 1
- # leading comment doc 2
-  a: 2
-EOM
-
-  X=$(./yq ea '[.]' --header-preprocess=false < test.yml)
-  assertEquals "$expected" "$X"
-
-}
-
-source ./scripts/shunit2
@@ -1,249 +0,0 @@
-#!/bin/bash
-
-setUp() {
-  rm test*.yml 2>/dev/null || true
-  rm test*.toml 2>/dev/null || true
-  rm test*.tfstate 2>/dev/null || true
-  rm test*.json 2>/dev/null || true
-  rm test*.properties 2>/dev/null || true
-  rm test*.csv 2>/dev/null || true
-  rm test*.tsv 2>/dev/null || true
-  rm test*.xml 2>/dev/null || true
-}
-
-testInputJson() {
-  cat >test.json <<EOL
-{ "mike" : { "things": "cool" } }
-EOL
-
-  read -r -d '' expected << EOM
-{
-  "mike": {
-    "things": "cool"
-  }
-}
-EOM
-
-  X=$(./yq test.json)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea test.json)
-  assertEquals "$expected" "$X"
-}
-
-testInputToml() {
-  cat >test.toml <<EOL
-[owner]
-name = "Tom Preston-Werner"
-dob = 1979-05-27T07:32:00-08:00
-EOL
-
-  read -r -d '' expected << EOM
-owner:
-  name: Tom Preston-Werner
-  dob: 1979-05-27T07:32:00-08:00
-EOM
-
-  X=$(./yq -oy test.toml)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea -oy test.toml)
-  assertEquals "$expected" "$X"
-}
-
-testInputTfstate() {
-  cat >test.tfstate <<EOL
-{ "mike" : { "things": "cool" } }
-EOL
-
-  read -r -d '' expected << EOM
-{"mike": {"things": "cool"}}
-EOM
-
-  X=$(./yq test.tfstate)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea test.tfstate)
-  assertEquals "$expected" "$X"
-}
-
-testInputJsonOutputYaml() {
-  cat >test.json <<EOL
-{ "mike" : { "things": "cool" } }
-EOL
-
-  read -r -d '' expected << EOM
-mike:
-  things: cool
-EOM
-
-  X=$(./yq test.json -oy)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea test.json -oy)
-  assertEquals "$expected" "$X"
-}
-
-testInputProperties() {
-  cat >test.properties <<EOL
-mike.things = hello
-EOL
-
-  read -r -d '' expected << EOM
-mike.things = hello
-EOM
-
-  X=$(./yq e test.properties)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq test.properties)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea test.properties)
-  assertEquals "$expected" "$X"
-}
-
-testInputPropertiesGitHubAction() {
-  cat >test.properties <<EOL
-mike.things = hello
-EOL
-
-  read -r -d '' expected << EOM
-mike.things = hello
-EOM
-
-  X=$(cat /dev/null | ./yq e test.properties)
-  assertEquals "$expected" "$X"
-
-  X=$(cat /dev/null | ./yq ea test.properties)
-  assertEquals "$expected" "$X"
-}
-
-testInputCSV() {
-  cat >test.csv <<EOL
-fruit,yumLevel
-apple,5
-banana,4
-EOL
-
-  read -r -d '' expected << EOM
-fruit,yumLevel
-apple,5
-banana,4
-EOM
-
-  X=$(./yq e test.csv)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea test.csv)
-  assertEquals "$expected" "$X"
-}
-
-testInputCSVUTF8() {
-  read -r -d '' expected << EOM
-id,first,last
-1,john,smith
-1,jane,smith
-EOM
-
-  X=$(./yq utf8.csv)
-  assertEquals "$expected" "$X"
-}
-
-testInputTSV() {
-  cat >test.tsv <<EOL
-fruit	yumLevel
-apple	5
-banana	4
-EOL
-
-  read -r -d '' expected << EOM
-fruit	yumLevel
-apple	5
-banana	4
-EOM
-
-  X=$(./yq e test.tsv)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea test.tsv)
-  assertEquals "$expected" "$X"
-}
-
-
-
-
-testInputXml() {
-  cat >test.xml <<EOL
-<cat legs="4">BiBi</cat>
-EOL
-
-  read -r -d '' expected << EOM
-<cat legs="4">BiBi</cat>
-EOM
-
-  X=$(./yq e test.xml)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea test.xml)
-  assertEquals "$expected" "$X"
-}
-
-testInputXmlNamespaces() {
-  cat >test.xml <<EOL
-<?xml version="1.0"?>
-<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
-</map>
-EOL
-
-  read -r -d '' expected << EOM
-<?xml version="1.0"?>
-<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url"></map>
-EOM
-
-  X=$(./yq e test.xml)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea test.xml)
-  assertEquals "$expected" "$X"
-}
-
-
-
-testInputXmlStrict() {
-  cat >test.xml <<EOL
-<?xml version="1.0"?>
-<!DOCTYPE root [
-<!ENTITY writer "Catherine.">
-<!ENTITY copyright "(r) Great">
-]>
-<root>
-    <item>&writer;&copyright;</item>
-</root>
-EOL
-
-  X=$(./yq --xml-strict-mode test.xml  2>&1)
-  assertEquals 1 $?
-  assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X"
-
-  X=$(./yq ea --xml-strict-mode test.xml  2>&1)
-  assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X"
-}
-
-testInputXmlGithubAction() {
-  cat >test.xml <<EOL
-<cat legs="4">BiBi</cat>
-EOL
-
-  read -r -d '' expected << EOM
-<cat legs="4">BiBi</cat>
-EOM
-
-  X=$(cat /dev/null | ./yq e test.xml)
-  assertEquals "$expected" "$X"
-
-  X=$(cat /dev/null | ./yq ea test.xml)
-  assertEquals "$expected" "$X"
-}
-
-source ./scripts/shunit2
@@ -63,20 +63,6 @@ EOM
  assertEquals "$expected" "$X"
 }

-testInputCSVUTF8() {
-  read -r -d '' expected << EOM
- id: 1
-  first: john
-  last: smith
- id: 1
-  first: jane
-  last: smith
-EOM
-
-  X=$(./yq -p=csv utf8.csv)
-  assertEquals "$expected" "$X"
-}
-
 testInputTSV() {
  cat >test.tsv <<EOL
 fruit	yumLevel
@@ -109,7 +95,7 @@ EOL
  read -r -d '' expected << EOM
 cat:
  +content: BiBi
-  +@legs: "4"
+  +legs: "4"
 EOM

  X=$(./yq e -p=xml test.yml)
@@ -120,44 +106,23 @@ EOM
 }

 testInputXmlNamespaces() {
-  cat >test.xml <<EOL
+  cat >test.yml <<EOL
 <?xml version="1.0"?>
 <map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
 </map>
 EOL

  read -r -d '' expected << EOM
-+p_xml: version="1.0"
 map:
-  +@xmlns: some-namespace
-  +@xmlns:xsi: some-instance
-  +@xsi:schemaLocation: some-url
+  +xmlns: some-namespace
+  +xmlns:xsi: some-instance
+  +xsi:schemaLocation: some-url
 EOM

-  X=$(./yq e -p=xml test.xml)
+  X=$(./yq e -p=xml test.yml)
  assertEquals "$expected" "$X"

-  X=$(./yq ea -p=xml test.xml)
-  assertEquals "$expected" "$X"
-}
-
-testInputXmlRoundtrip() {
-  cat >test.yml <<EOL
-<?xml version="1.0"?>
-<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
-<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">Meow</map>
-EOL
-
-  read -r -d '' expected << EOM
-<?xml version="1.0"?>
-<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
-<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">Meow</map>
-EOM
-
-  X=$(./yq -p=xml -o=xml test.yml)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea -p=xml -o=xml test.yml)
+  X=$(./yq ea -p=xml test.yml)
  assertEquals "$expected" "$X"
 }

@@ -174,11 +139,11 @@ testInputXmlStrict() {
 </root>
 EOL

-  X=$(./yq -p=xml --xml-strict-mode test.yml -o=xml 2>&1)
+  X=$(./yq -p=xml --xml-strict-mode test.yml 2>&1)
  assertEquals 1 $?
  assertEquals "Error: bad file 'test.yml': XML syntax error on line 7: invalid character entity &writer;" "$X"

-  X=$(./yq ea -p=xml --xml-strict-mode test.yml -o=xml 2>&1)
+  X=$(./yq ea -p=xml --xml-strict-mode test.yml 2>&1)
  assertEquals "Error: bad file 'test.yml': XML syntax error on line 7: invalid character entity &writer;" "$X"
 }

@@ -190,7 +155,7 @@ EOL
  read -r -d '' expected << EOM
 cat:
  +content: BiBi
-  +@legs: "4"
+  +legs: "4"
 EOM

  X=$(cat /dev/null | ./yq e -p=xml test.yml)
@@ -1,8 +1,5 @@
 #!/bin/bash

-
-# examples where header-preprocess is required
-
 setUp() {
  rm test*.yml || true
  cat >test.yml <<EOL
@@ -34,82 +31,6 @@ EOM
  assertEquals "$expected" "$X"
 }

-
-testLeadingSeperatorWithNewlinesNewDoc() {
-  cat >test.yml <<EOL
-# hi peeps
-# cool
-
-
---
-a: test
---
-b: cool
-EOL
-
-  read -r -d '' expected << EOM
-# hi peeps
-# cool
-
-
---
-a: thing
---
-b: cool
-EOM
-
-  X=$(./yq e '(select(di == 0) | .a) = "thing"' - < test.yml)
-  assertEquals "$expected" "$X"
-}
-
-testLeadingSeperatorWithNewlinesMoreComments() {
-  cat >test.yml <<EOL
-# hi peeps
-# cool
-
---
-# great
-
-a: test
---
-b: cool
-EOL
-
-  read -r -d '' expected << EOM
-# hi peeps
-# cool
-
---
-# great
-
-a: thing
---
-b: cool
-EOM
-
-  X=$(./yq e '(select(di == 0) | .a) = "thing"' - < test.yml)
-  assertEquals "$expected" "$X"
-}
-
-
-testLeadingSeperatorWithDirective() {
-  cat >test.yml <<EOL
-%YAML 1.1
---
-this: should really work
-EOL
-
-  read -r -d '' expected << EOM
-%YAML 1.1
---
-this: should really work
-EOM
-
-  X=$(./yq < test.yml)
-  assertEquals "$expected" "$X"
-}
-
-
 testLeadingSeperatorPipeIntoEvalSeq() {
  X=$(./yq e - < test.yml)
  expected=$(cat test.yml)
@@ -1,286 +0,0 @@
-#!/bin/bash
-
-setUp() {
-    rm test*.yml || true
-}
-
-## Convenient bash shortcut to read records of NUL separated values
-## from stdin the safe way. See example usage in the next tests.
-read-0() {
-    local eof="" IFS=''
-    while [ "$1" ]; do
-        ## - The `-r` avoids bad surprise with '\n' and other interpreted
-        ##   sequences that can be read.
-        ## - The `-d ''` is the (strange?) way to refer to NUL delimiter.
-        ## - The `--` is how to avoid unpleasant surprises if your
-        ##   "$1" starts with "-" (minus) sign. This protection also
-        ##   will produce a readable error if you want to try to start
-        ##   your variable names with a "-".
-        read -r -d '' -- "$1" || eof=1
-        shift
-    done
-    [ -z "$eof" ] ## fail on EOF
-}
-
-## Convenient bash shortcut to be used with the next function `p-err`
-## to read NUL separated values the safe way AND catch any errors from
-## the process creating the stream of NUL separated data.  See example
-## usage in the tests.
-read-0-err() {
-    local ret="$1" eof="" idx=0 last=
-    read -r -- "${ret?}" <<<"0"
-    shift
-    while [ "$1" ]; do
-        last=$idx
-        read -r -d '' -- "$1" || {
-            ## Put this last value in ${!ret}
-            eof="$1"
-            read -r -- "$ret" <<<"${!eof}"
-            break
-        }
-        ((idx++))
-        shift
-    done
-    [ -z "$eof" ] || {
-        if [ "$last" != 0 ]; then
-            ## Uhoh, we have no idea if the errorlevel of the internal
-            ## command was properly delimited with a NUL char, and
-            ## anyway something went really wrong at least about the
-            ## number of fields separated by NUL char and the one
-            ## expected.
-            echo "Error: read-0-err couldn't fill all value $ret = '${!ret}', '$eof', '${!eof}'" >&2
-            read -r -- "$ret" <<<"not-enough-values"
-        else
-            if ! [[ "${!ret}" =~ ^[0-9]+$ && "${!ret}" -ge 0 && "${!ret}" -le 127 ]]; then
-                ## This could happen if you don't use `p-err` wrapper,
-                ## or used stdout in unexpected ways in your inner
-                ## command.
-                echo "Error: last value is not a number, did you finish with an errorlevel ?" >&2
-                read -r -- "$ret" <<<"last-value-not-a-number"
-            fi
-        fi
-        false
-    }
-}
-
-## Simply runs command given as argument and adds errorlevel in the
-## standard output. Is expected to be used in tandem with
-## `read-0-err`.
-p-err() {
-    local exp="$1"
-    "$@"
-    printf "%s" "$?"
-}
-
-wyq-r() {
-    local exp="$1"
-    ./yq e -0 -r=false "$1"
-    printf "%s" "$?"
-}
-
-testBasicUsageRaw() {
-  cat >test.yml <<EOL
-a: foo
-b: bar
-EOL
-
-  printf "foo\0bar\0" > expected.out
-
-  ## We need to compare binary content here. We have to filter the compared
-  ## content through a representation that gets rid of NUL chars but accurately
-  ## transcribe the content.
-  ## Also as it would be nice to have a pretty output in case the test fails,
-  ## we use here 'hd': a widely available shortcut to 'hexdump' that will
-  ## pretty-print any binary to it's hexadecimal representation.
-  ##
-  ## Note that the standard `assertEquals` compare its arguments
-  ## value, but they can't hold NUL characters (this comes from the
-  ## limitation of the C API of `exec*(..)` functions that requires
-  ## `const char *arv[]`). And these are NUL terminated strings.  As a
-  ## consequence, the NUL characters gets removed in bash arguments.
-  assertEquals "$(hd expected.out)" \
-               "$(./yq e -0 '.a, .b' test.yml | hd)"
-
-  rm expected.out
-}
-
-testBasicUsage() {
-  local a b
-  cat >test.yml <<EOL
-a: foo
-b: bar
-EOL
-
-  ## We provide 2 values, and ask to fill 2 variables.
-  read-0 a b < <(./yq e -0 '.a, .b' test.yml)
-  assertEquals "$?" "0"      ## Everything is fine
-  assertEquals "foo" "$a"    ## Values are correctly parsed
-  assertEquals "bar" "$b"
-
-  a=YYY ; b=XXX
-  ## Not enough values provided to fill `a` and `b`.
-  read-0 a b < <(./yq e -0 '.a' test.yml)
-  assertEquals "$?" "1"      ## An error was emitted
-  assertEquals "foo" "$a"    ## First value was correctly parsed
-  assertEquals "" "$b"       ## Second was still reset
-
-  ## Error from inner command are not catchable !. Use
-  ## `read-0-err`/`p-err` for that.
-  read-0 a < <(printf "\0"; ./yq e -0 'xxx' test.yml; )
-  assertEquals "$?" "0"
-
-}
-
-testBasicUsageJson() {
-  cat >test.yml <<EOL
-a:
-  x: foo
-b: bar
-EOL
-
-  read-0 a b < <(./yq e -0 -o=json '.a, .b' test.yml)
-
-  assertEquals '{
-  "x": "foo"
-}' "$a"
-  assertEquals '"bar"' "$b"
-
-}
-
-testFailWithValueContainingNUL() {
-  local a b c
-  ## Note that value of field 'a' actually contains a NUL char !
-  cat >test.yml <<EOL
-a: "foo\u0000bar"
-b: 1
-c: |
-  wiz
-  boom
-EOL
-
-  ## We are looking for trouble with asking to separated fields with NUL
-  ## char and requested value `.a` actually contains itself a NUL char !
-  read-0 a b c < <(./yq e -0 '.a, .b, .c' test.yml)
-  assertNotEquals "0" "$?"   ## read-0 failed to fill all values
-
-  ## But here, we can request for one value, even if `./yq` fails
-  read-0 b < <(./yq e -0 '.b, .a' test.yml)
-  assertEquals "0" "$?"   ## read-0 succeeds at feeding the first value
-  ## Note: to catch the failure of `yq`, see in the next tests the usage
-  ## of `read-0-err`.
-
-  ## using -r=false solves any NUL containing value issues, but keeps
-  ## all in YAML representation:
-  read-0 a b c < <(./yq e -0 -r=false '.a, .b, .c' test.yml)
-  assertEquals "0" "$?"    ## All goes well despite asking for `a` value
-
-  assertEquals '"foo\0bar"' "$a"   ## This is a YAML string representation
-  assertEquals '1' "$b"
-  assertEquals '|
-  wiz
-  boom' "$c"
-}
-
-testStandardLoop() {
-    local E a b res
-
-    ## Here everything is normal: 4 values, that will be paired
-    ## in key/values.
-    cat >test.yml <<EOL
- yay
- wiz
- hop
- pow
-EOL
-
-    res=""
-    while read-0-err E a b; do
-        res+="$a: $b;"
-    done < <(p-err ./yq -0 '.[]' test.yml)
-
-    assertEquals "0" "$E"                     ## errorlevel of internal command
-    assertEquals "yay: wiz;hop: pow;" "$res"  ## expected result
-}
-
-testStandardLoopWithoutEnoughValues() {
-    local E a b res
-
-    ## Here 5 values, there will be a missing value when reading
-    ## pairs of value.
-    cat >test.yml <<EOL
- yay
- wiz
- hop
- pow
- kwak
-EOL
-
-    res=""
-    ## The loop will succeed 2 times then fail
-    while read-0-err E a b; do
-        res+="$a: $b;"
-    done < <(p-err ./yq -0 '.[]' test.yml)
-
-    assertEquals "not-enough-values" "$E"     ## Not enough value error
-    assertEquals "yay: wiz;hop: pow;" "$res"  ## the 2 full key/value pairs
-
-}
-
-testStandardLoopWithInternalCmdError() {
-    local E a b res
-
-    ## Note the third value contains a NUL char !
-    cat >test.yml <<EOL
- yay
- wiz
- "foo\0bar"
- hop
- pow
-EOL
-
-    res=""
-    ## It should be only upon the second pass in the loop that
-    ## read-0-err will catch the fact that there is an error !
-    while read-0-err E a b; do
-        res+="$a: $b;"
-    done < <(p-err ./yq -0 '.[]' test.yml)
-    assertEquals "1" "$E"            ## Internal command errorlevel (from `./yq`)
-    assertEquals "yay: wiz;" "$res"  ## first 2 values were ok at least
-
-}
-
-testStandardLoopNotEnoughErrorEatsCmdError() {
-    local E a b res
-
-    ## Because of possible edge cases where the internal errorlevel
-    ## reported by `p-err` in the standard output might be mangled
-    ## with the unfinished record, `read-0-err E ...` will NOT report
-    ## the internal command error in the variable E and instead will
-    ## store the value 'not-enough-values'. In real world, anyway, you
-    ## will want to react the same if the internal command failed
-    ## and/or you didn't get as much values as expected while
-    ## reading. Keep in mind also that standard error is not
-    ## swallowed, so you can read reports from the inner command AND
-    ## from `read-0-err`.
-
-    ## Here, note that the fourth value contains a NUL char !
-    cat >test.yml <<EOL
- yay
- wiz
- hop
- "foo\0bar"
- pow
-EOL
-
-    res=""
-    ## It should be only upon the second loop that read-0-err will catch
-    ## the fact that there are not enough data to fill the requested variables
-    while read-0-err E a b; do
-        res+="$a: $b;"
-    done < <(p-err ./yq -0 '.[]' test.yml)
-    assertEquals "not-enough-values" "$E"          ## Not enough values error eats internal error !
-    assertEquals "yay: wiz;" "$res"  ## first 2 values were ok at least
-}
-
-
-source ./scripts/shunit2
@@ -48,55 +48,6 @@ EOM
  assertEquals "$expected" "$X"
 }

-testOutputYamlRawDefault() {
-  cat >test.yml <<EOL
-a: "cat"
-EOL
-
-  X=$(./yq e  '.a' test.yml)
-  assertEquals "cat" "$X"
-
-  X=$(./yq ea '.a' test.yml)
-  assertEquals "cat" "$X"
-}
-
-testOutputYamlRawOff() {
-  cat >test.yml <<EOL
-a: "cat"
-EOL
-
-  X=$(./yq e -r=false '.a' test.yml)
-  assertEquals "\"cat\"" "$X"
-
-  X=$(./yq ea -r=false '.a' test.yml)
-  assertEquals "\"cat\"" "$X"
-}
-
-testOutputJsonRaw() {
-  cat >test.yml <<EOL
-a: cat
-EOL
-
-  X=$(./yq e -r --output-format=json '.a' test.yml)
-  assertEquals "cat" "$X"
-
-  X=$(./yq ea -r --output-format=json '.a' test.yml)
-  assertEquals "cat" "$X"
-}
-
-testOutputJsonDefault() {
-  cat >test.yml <<EOL
-a: cat
-EOL
-
-  X=$(./yq e --output-format=json '.a' test.yml)
-  assertEquals "\"cat\"" "$X"
-
-  X=$(./yq ea --output-format=json '.a' test.yml)
-  assertEquals "\"cat\"" "$X"
-}
-
-
 testOutputJsonShort() {
  cat >test.yml <<EOL
 a: {b: ["cat"]}
@@ -121,11 +72,11 @@ EOM

 testOutputProperties() {
  cat >test.yml <<EOL
-a: {b: {c: ["cat cat"]}}
+a: {b: {c: ["cat"]}}
 EOL

  read -r -d '' expected << EOM
-a.b.c.0 = cat cat
+a.b.c.0 = cat
 EOM

  X=$(./yq e --output-format=props test.yml)
@@ -135,30 +86,13 @@ EOM
  assertEquals "$expected" "$X"
 }

-testOutputPropertiesDontUnwrap() {
-  cat >test.yml <<EOL
-a: {b: {c: ["cat cat"]}}
-EOL
-
-  read -r -d '' expected << EOM
-a.b.c.0 = "cat cat"
-EOM
-
-  X=$(./yq e -r=false --output-format=props test.yml)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea -r=false --output-format=props test.yml)
-  assertEquals "$expected" "$X"
-}
-
-
 testOutputPropertiesShort() {
  cat >test.yml <<EOL
-a: {b: {c: ["cat cat"]}}
+a: {b: {c: ["cat"]}}
 EOL

  read -r -d '' expected << EOM
-a.b.c.0 = cat cat
+a.b.c.0 = cat
 EOM

  X=$(./yq e -o=p test.yml)
@@ -252,7 +186,7 @@ EOM

 testOutputXmComplex() {
  cat >test.yml <<EOL
-a: {b: {c: ["cat", "dog"], +@f: meow}}
+a: {b: {c: ["cat", "dog"], +f: meow}}
 EOL

  read -r -d '' expected << EOM
@@ -12,6 +12,6 @@ outputs:
    description: "The complete result from the yq command being run"
 runs:
  using: 'docker'
-  image: 'docker://mikefarah/yq:4-githubaction'
+  image: 'github-action/Dockerfile'
  args:
    - ${{ inputs.cmd }}
@@ -1,15 +1,18 @@
 package cmd

-var unwrapScalarFlag = newUnwrapFlag()
-
-var unwrapScalar = false
+var leadingContentPreProcessing = true
+var unwrapScalar = true

 var writeInplace = false
 var outputToJSON = false
+var outputFormat = "yaml"
+var inputFormat = "yaml"

-var outputFormat = ""
-
-var inputFormat = ""
+var xmlAttributePrefix = "+"
+var xmlContentName = "+content"
+var xmlStrictMode = false
+var xmlKeepNamespace = true
+var xmlUseRawToken = true

 var exitStatus = false
 var forceColor = false
@@ -18,7 +21,6 @@ var colorsEnabled = false
 var indent = 2
 var noDocSeparators = false
 var nullInput = false
-var nulSepOutput = false
 var verbose = false
 var version = false
 var prettyPrint = false
@@ -75,7 +75,7 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 		return err
 	}

-	decoder, err := configureDecoder(true)
+	decoder, err := configureDecoder()
 	if err != nil {
 		return err
 	}
@@ -84,15 +84,9 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 	if err != nil {
 		return err
 	}
-	encoder, err := configureEncoder()
-	if err != nil {
-		return err
-	}
+	encoder := configureEncoder(format)

 	printer := yqlib.NewPrinter(encoder, printerWriter)
-	if nulSepOutput {
-		printer.SetNulSepOutput(true)
-	}

 	if frontMatter != "" {
 		frontMatterHandler := yqlib.NewFrontMatterHandler(args[0])
@@ -115,13 +109,13 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 	switch len(args) {
 	case 0:
 		if nullInput {
-			err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(expression), printer)
+			err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(expression), printer, "")
 		} else {
 			cmd.Println(cmd.UsageString())
 			return nil
 		}
 	default:
-		err = allAtOnceEvaluator.EvaluateFiles(processExpression(expression), args, printer, decoder)
+		err = allAtOnceEvaluator.EvaluateFiles(processExpression(expression), args, printer, leadingContentPreProcessing, decoder)
 	}

 	completedSuccessfully = err == nil
@@ -93,17 +93,11 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 	if err != nil {
 		return err
 	}
-	encoder, err := configureEncoder()
-	if err != nil {
-		return err
-	}
+	encoder := configureEncoder(format)

 	printer := yqlib.NewPrinter(encoder, printerWriter)
-	if nulSepOutput {
-		printer.SetNulSepOutput(true)
-	}

-	decoder, err := configureDecoder(false)
+	decoder, err := configureDecoder()
 	if err != nil {
 		return err
 	}
@@ -129,13 +123,13 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 	switch len(args) {
 	case 0:
 		if nullInput {
-			err = streamEvaluator.EvaluateNew(processExpression(expression), printer)
+			err = streamEvaluator.EvaluateNew(processExpression(expression), printer, "")
 		} else {
 			cmd.Println(cmd.UsageString())
 			return nil
 		}
 	default:
-		err = streamEvaluator.EvaluateFiles(processExpression(expression), args, printer, decoder)
+		err = streamEvaluator.EvaluateFiles(processExpression(expression), args, printer, leadingContentPreProcessing, decoder)
 	}
 	completedSuccessfully = err == nil

@@ -35,7 +35,7 @@ yq -P sample.json
 			return evaluateSequence(cmd, args)

 		},
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
 			cmd.SetOut(cmd.OutOrStdout())

 			var format = logging.MustStringFormatter(
@@ -47,13 +47,14 @@ yq -P sample.json
 			if verbose {
 				backend.SetLevel(logging.DEBUG, "")
 			} else {
-				backend.SetLevel(logging.WARNING, "")
+				backend.SetLevel(logging.ERROR, "")
 			}

 			logging.SetBackend(backend)
 			yqlib.InitExpressionParser()
-
-			return nil
+			yqlib.XMLPreferences.AttributePrefix = xmlAttributePrefix
+			yqlib.XMLPreferences.ContentName = xmlContentName
+			yqlib.XMLPreferences.StrictMode = xmlStrictMode
 		},
 	}

@@ -65,18 +66,14 @@ yq -P sample.json
 		panic(err)
 	}

-	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", "[auto|a|yaml|y|json|j|props|p|xml|x|tsv|t|csv|c] output format type.")
-	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", "[auto|a|yaml|y|props|p|xml|x|tsv|t|csv|c|toml] parse format for input. Note that json is a subset of yaml.")
+	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "yaml", "[yaml|y|json|j|props|p|xml|x] output format type.")
+	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "yaml", "[yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.")

-	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
-	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.StrictMode, "xml-strict-mode", yqlib.ConfiguredXMLPreferences.StrictMode, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.KeepNamespace, "xml-keep-namespace", yqlib.ConfiguredXMLPreferences.KeepNamespace, "enables keeping namespace after parsing attributes")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.UseRawToken, "xml-raw-token", yqlib.ConfiguredXMLPreferences.UseRawToken, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")
-	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ProcInstPrefix, "xml-proc-inst-prefix", yqlib.ConfiguredXMLPreferences.ProcInstPrefix, "prefix for xml processing instructions (e.g. <?xml version=\"1\"?>)")
-	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.DirectiveName, "xml-directive-name", yqlib.ConfiguredXMLPreferences.DirectiveName, "name for xml directives (e.g. <!DOCTYPE thing cat>)")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", yqlib.ConfiguredXMLPreferences.SkipProcInst, "skip over process instructions (e.g. <?xml version=\"1\"?>)")
-	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", yqlib.ConfiguredXMLPreferences.SkipDirectives, "skip over directives (e.g. <!DOCTYPE thing cat>)")
+	rootCmd.PersistentFlags().StringVar(&xmlAttributePrefix, "xml-attribute-prefix", "+", "prefix for xml attributes")
+	rootCmd.PersistentFlags().StringVar(&xmlContentName, "xml-content-name", "+content", "name for xml content (if no attribute name is present).")
+	rootCmd.PersistentFlags().BoolVar(&xmlStrictMode, "xml-strict-mode", false, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
+	rootCmd.PersistentFlags().BoolVar(&xmlKeepNamespace, "xml-keep-namespace", true, "enables keeping namespace after parsing attributes")
+	rootCmd.PersistentFlags().BoolVar(&xmlUseRawToken, "xml-raw-token", true, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")

 	rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.")
 	rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)")
@@ -84,10 +81,7 @@ yq -P sample.json
 	rootCmd.PersistentFlags().IntVarP(&indent, "indent", "I", 2, "sets indent level for output")
 	rootCmd.Flags().BoolVarP(&version, "version", "V", false, "Print version information and quit")
 	rootCmd.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the file inplace of first file given.")
-	rootCmd.PersistentFlags().VarP(unwrapScalarFlag, "unwrapScalar", "r", "unwrap scalar, print the value with no quotes, colors or comments. Defaults to true for yaml")
-	rootCmd.PersistentFlags().Lookup("unwrapScalar").NoOptDefVal = "true"
-	rootCmd.PersistentFlags().BoolVarP(&nulSepOutput, "nul-output", "0", false, "Use NUL char to separate values. If unwrap scalar is also set, fail if unwrapped scalar contains NUL char.")
-
+	rootCmd.PersistentFlags().BoolVarP(&unwrapScalar, "unwrapScalar", "r", true, "unwrap scalar, print the value with no quotes, colors or comments")
 	rootCmd.PersistentFlags().BoolVarP(&prettyPrint, "prettyPrint", "P", false, "pretty print, shorthand for '... style = \"\"'")
 	rootCmd.PersistentFlags().BoolVarP(&exitStatus, "exit-status", "e", false, "set exit status if there are no matches or null or false is returned")

@@ -95,7 +89,7 @@ yq -P sample.json
 	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
 	rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
 	rootCmd.PersistentFlags().StringVarP(&forceExpression, "expression", "", "", "forcibly set the expression argument. Useful when yq argument detection thinks your expression is a file.")
-	rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredYamlPreferences.LeadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.")
+	rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.")

 	rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter.")
 	rootCmd.PersistentFlags().StringVarP(&splitFileExpFile, "split-exp-file", "", "", "Use a file to specify the split-exp expression.")
@@ -42,7 +42,7 @@ $ yq shell-completion fish > ~/.config/fish/completions/yq.fish
 `,
 	DisableFlagsInUseLine: true,
 	ValidArgs:             []string{"bash", "zsh", "fish", "powershell"},
-	Args:                  cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs),
+	Args:                  cobra.ExactValidArgs(1),
 	RunE: func(cmd *cobra.Command, args []string) error {
 		var err error = nil
 		switch args[0] {
@@ -1,46 +0,0 @@
-package cmd
-
-import (
-	"strconv"
-
-	"github.com/spf13/pflag"
-)
-
-type boolFlag interface {
-	pflag.Value
-	IsExplicitySet() bool
-	IsSet() bool
-}
-
-type unwrapScalarFlagStrc struct {
-	explicitySet bool
-	value        bool
-}
-
-func newUnwrapFlag() boolFlag {
-	return &unwrapScalarFlagStrc{value: true}
-}
-
-func (f *unwrapScalarFlagStrc) IsExplicitySet() bool {
-	return f.explicitySet
-}
-
-func (f *unwrapScalarFlagStrc) IsSet() bool {
-	return f.value
-}
-
-func (f *unwrapScalarFlagStrc) String() string {
-	return strconv.FormatBool(f.value)
-}
-
-func (f *unwrapScalarFlagStrc) Set(value string) error {
-
-	v, err := strconv.ParseBool(value)
-	f.value = v
-	f.explicitySet = true
-	return err
-}
-
-func (*unwrapScalarFlagStrc) Type() string {
-	return "bool"
-}
@@ -10,10 +10,6 @@ import (
 	"gopkg.in/op/go-logging.v1"
 )

-func isAutomaticOutputFormat() bool {
-	return outputFormat == "" || outputFormat == "auto" || outputFormat == "a"
-}
-
 func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 	cmd.SilenceUsage = true

@@ -57,81 +53,17 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 		return "", nil, fmt.Errorf("cannot pass files in when using null-input flag")
 	}

-	inputFilename := ""
-	if len(args) > 0 {
-		inputFilename = args[0]
-	}
-	if inputFormat == "" || inputFormat == "auto" || inputFormat == "a" {
-
-		inputFormat = yqlib.FormatFromFilename(inputFilename)
-
-		_, err := yqlib.InputFormatFromString(inputFormat)
-		if err != nil {
-			// unknown file type, default to yaml
-			yqlib.GetLogger().Debug("Unknown file format extension '%v', defaulting to yaml", inputFormat)
-			inputFormat = "yaml"
-			if isAutomaticOutputFormat() {
-				outputFormat = "yaml"
-			}
-		} else if isAutomaticOutputFormat() {
-			// automatic input worked, we can do it for output too unless specified
-			if inputFormat == "json" {
-				yqlib.GetLogger().Warning("JSON file output is now JSON by default (instead of yaml). Use '-oy' or '--output-format=yaml' for yaml output")
-			}
-			outputFormat = inputFormat
-		}
-	} else if isAutomaticOutputFormat() {
-		// backwards compatibility -
-		// before this was introduced, `yq -pcsv things.csv`
-		// would produce *yaml* output.
-		//
-		outputFormat = yqlib.FormatFromFilename(inputFilename)
-		if inputFilename != "-" {
-			yqlib.GetLogger().Warning("yq default output is now 'auto' (based on the filename extension). Normally yq would output '%v', but for backwards compatibility 'yaml' has been set. Please use -oy to specify yaml, or drop the -p flag.", outputFormat)
-		}
-		outputFormat = "yaml"
-	}
-
-	outputFormatType, err := yqlib.OutputFormatFromString(outputFormat)
-
-	if err != nil {
-		return "", nil, err
-	}
-	yqlib.GetLogger().Debug("Using input format %v", inputFormat)
-	yqlib.GetLogger().Debug("Using output format %v", outputFormat)
-
-	if outputFormatType == yqlib.YamlOutputFormat ||
-		outputFormatType == yqlib.PropsOutputFormat {
-		unwrapScalar = true
-	}
-	if unwrapScalarFlag.IsExplicitySet() {
-		unwrapScalar = unwrapScalarFlag.IsSet()
-	}
-
-	//copy preference form global setting
-	yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
-
-	yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
-
 	return expression, args, nil
 }

-func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
+func configureDecoder() (yqlib.Decoder, error) {
 	yqlibInputFormat, err := yqlib.InputFormatFromString(inputFormat)
 	if err != nil {
 		return nil, err
 	}
-	yqlibDecoder, err := createDecoder(yqlibInputFormat, evaluateTogether)
-	if yqlibDecoder == nil {
-		return nil, fmt.Errorf("no support for %s input format", inputFormat)
-	}
-	return yqlibDecoder, err
-}
-
-func createDecoder(format yqlib.InputFormat, evaluateTogether bool) (yqlib.Decoder, error) {
-	switch format {
+	switch yqlibInputFormat {
 	case yqlib.XMLInputFormat:
-		return yqlib.NewXMLDecoder(yqlib.ConfiguredXMLPreferences), nil
+		return yqlib.NewXMLDecoder(xmlAttributePrefix, xmlContentName, xmlStrictMode, xmlKeepNamespace, xmlUseRawToken), nil
 	case yqlib.PropertiesInputFormat:
 		return yqlib.NewPropertiesDecoder(), nil
 	case yqlib.JsonInputFormat:
@@ -140,14 +72,9 @@ func createDecoder(format yqlib.InputFormat, evaluateTogether bool) (yqlib.Decod
 		return yqlib.NewCSVObjectDecoder(','), nil
 	case yqlib.TSVObjectInputFormat:
 		return yqlib.NewCSVObjectDecoder('\t'), nil
-	case yqlib.TomlInputFormat:
-		return yqlib.NewTomlDecoder(), nil
-	case yqlib.YamlInputFormat:
-		prefs := yqlib.ConfiguredYamlPreferences
-		prefs.EvaluateTogether = evaluateTogether
-		return yqlib.NewYamlDecoder(prefs), nil
 	}
-	return nil, fmt.Errorf("invalid decoder: %v", format)
+
+	return yqlib.NewYamlDecoder(), nil
 }

 func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yqlib.PrinterWriter, error) {
@@ -167,38 +94,22 @@ func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yq
 	return printerWriter, nil
 }

-func configureEncoder() (yqlib.Encoder, error) {
-	yqlibOutputFormat, err := yqlib.OutputFormatFromString(outputFormat)
-	if err != nil {
-		return nil, err
-	}
-	yqlibEncoder, err := createEncoder(yqlibOutputFormat)
-	if yqlibEncoder == nil {
-		return nil, fmt.Errorf("no support for %s output format", outputFormat)
-	}
-	return yqlibEncoder, err
-}
-
-func createEncoder(format yqlib.PrinterOutputFormat) (yqlib.Encoder, error) {
+func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
 	switch format {
 	case yqlib.JSONOutputFormat:
-		return yqlib.NewJSONEncoder(indent, colorsEnabled, unwrapScalar), nil
+		return yqlib.NewJSONEncoder(indent, colorsEnabled)
 	case yqlib.PropsOutputFormat:
-		return yqlib.NewPropertiesEncoder(unwrapScalar), nil
+		return yqlib.NewPropertiesEncoder(unwrapScalar)
 	case yqlib.CSVOutputFormat:
-		return yqlib.NewCsvEncoder(','), nil
+		return yqlib.NewCsvEncoder(',')
 	case yqlib.TSVOutputFormat:
-		return yqlib.NewCsvEncoder('\t'), nil
+		return yqlib.NewCsvEncoder('\t')
 	case yqlib.YamlOutputFormat:
-		return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences), nil
+		return yqlib.NewYamlEncoder(indent, colorsEnabled, !noDocSeparators, unwrapScalar)
 	case yqlib.XMLOutputFormat:
-		return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences), nil
-	case yqlib.TomlOutputFormat:
-		return yqlib.NewTomlEncoder(), nil
-	case yqlib.ShellVariablesOutputFormat:
-		return yqlib.NewShellVariablesEncoder(), nil
+		return yqlib.NewXMLEncoder(indent, xmlAttributePrefix, xmlContentName)
 	}
-	return nil, fmt.Errorf("invalid encoder: %v", format)
+	panic("invalid encoder")
 }

 // this is a hack to enable backwards compatibility with githubactions (which pipe /dev/null into everything)
@@ -11,7 +11,7 @@ var (
 	GitDescribe string

 	// Version is main version number that is being run at the moment.
-	Version = "v4.34.2"
+	Version = "4.26.1"

 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release
@@ -103,7 +103,7 @@ yq (4.16.2) focal; urgency=medium
  * Improved extract-checksum.sh
  * Bump github.com/spf13/cobra from 1.2.1 to 1.3.0 (#1039)
  * enable more linters (#1043)
-  * Bump Golang compiler #1037
+  * Bump golang compiler #1037

 -- Roberto Mier Escandon <rmescandon@gmail.com>  Tue, 21 Dec 2021 09:41:44 +0000

@@ -1,2 +1,11 @@
- [cat, dog, frog, cow]
- [apple, banana, grape, mango]
+---
+- become: true
+  gather_facts: false
+  hosts: lalaland
+  name: "Apply smth"
+  roles:
+    - lala
+    - land
+  serial: 1
+- become: false
+  gather_facts: true
@@ -1 +1,12 @@
-["foobar", "foobaz", "blarp"]
+# --------------------------------------------------
+# It's a test with comment
+# --------------------------------------------------
+test:
+  - name: a
+  - name: b
+  - name: c
+groups:
+  - name: a
+  - name: b
+  - name: c
+  - name: c
@@ -1 +1,14 @@
-<cat>3</cat>
+<!-- before cat -->
+<cat>
+	<!-- in cat before -->
+	<x>3<!-- multi
+line comment
+for x --></x>
+	<y>
+		<!-- in y before -->
+		<d><!-- in d before -->4<!-- in d after --></d>
+		<!-- in y after -->
+	</y>
+	<!-- in_cat_after -->
+</cat>
+<!-- after cat -->
@@ -1,26 +0,0 @@
-
-
-# This is a TOML document
-
-title = "TOML Example"
-
-[owner]
-name = "Tom Preston-Werner"
-dob = 1979-05-27T07:32:00-08:00
-
-[database]
-enabled = true
-ports = [ 8000, 8001, 8002 ]
-data = [ ["delta", "phi"], [3.14] ]
-temp_targets = { cpu = 79.5, case = 72.0 }
-
-[servers]
-
-[servers.alpha]
-ip = "10.0.0.1"
-role = "frontend"
-
-[servers.beta]
-ip = "10.0.0.2"
-role = "backend"
-
@@ -1,3 +1,3 @@
 name,numberOfCats,likesApples,height
-,1,true,168.8
+Gary,1,true,168.8
 Samantha's Rabbit,2,false,-188.8
@@ -1,4 +0,0 @@
---
-# comment
-# about things
-a: cat
@@ -2,8 +2,7 @@ FROM mikefarah/yq:4

 COPY entrypoint.sh /entrypoint.sh

-# github action recommendation is to run as root.
-# https://docs.github.com/en/actions/creating-actions/dockerfile-support-for-github-actions#user
-USER root
+# this seems to be the default user in github actions
+USER 1001

 ENTRYPOINT ["/entrypoint.sh"]
@@ -2,11 +2,8 @@
 set -e
 echo "::debug::\$cmd: $1"
 RESULT=$(eval "$1")
+RESULT="${RESULT//'%'/'%25'}"
+RESULT="${RESULT//$'\n'/'%0A'}"
+RESULT="${RESULT//$'\r'/'%0D'}"
 echo "::debug::\$RESULT: $RESULT"
-# updating from 
-# https://github.com/orgs/community/discussions/26288#discussioncomment-3876281
-# https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-output-parameter
-delimiter=$(cat /proc/sys/kernel/random/uuid)
-echo "result<<${delimiter}" >> "${GITHUB_OUTPUT}"
-echo "${RESULT}" >> "${GITHUB_OUTPUT}"
-echo "${delimiter}" >> "${GITHUB_OUTPUT}"
+echo ::set-output name=result::"$RESULT"
@@ -1,32 +1,30 @@
 module github.com/mikefarah/yq/v4

 require (
-	github.com/a8m/envsubst v1.4.2
-	github.com/alecthomas/participle/v2 v2.0.0
-	github.com/alecthomas/repr v0.2.0
-	github.com/dimchansky/utfbom v1.1.1
-	github.com/elliotchance/orderedmap v1.5.0
-	github.com/fatih/color v1.15.0
-	github.com/goccy/go-json v0.10.2
-	github.com/goccy/go-yaml v1.11.0
+	github.com/a8m/envsubst v1.3.0
+	github.com/alecthomas/participle/v2 v2.0.0-beta.4
+	github.com/alecthomas/repr v0.1.0
+	github.com/elliotchance/orderedmap v1.4.0
+	github.com/fatih/color v1.13.0
+	github.com/goccy/go-json v0.9.10
+	github.com/goccy/go-yaml v1.9.5
 	github.com/jinzhu/copier v0.3.5
-	github.com/magiconair/properties v1.8.7
-	github.com/pelletier/go-toml/v2 v2.0.8
+	github.com/magiconair/properties v1.8.6
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
-	github.com/spf13/cobra v1.7.0
-	github.com/spf13/pflag v1.0.5
-	golang.org/x/net v0.12.0
-	golang.org/x/text v0.11.0
+	github.com/spf13/cobra v1.5.0
+	golang.org/x/net v0.0.0-20220708220712-1185a9018129
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.1
 )

 require (
-	github.com/inconshreveable/mousetrap v1.1.0 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.17 // indirect
-	golang.org/x/sys v0.10.0 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e // indirect
+	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 )

-go 1.20
+go 1.18
@@ -1,68 +1,81 @@
-github.com/a8m/envsubst v1.4.2 h1:4yWIHXOLEJHQEFd4UjrWDrYeYlV7ncFWJOCBRLOZHQg=
-github.com/a8m/envsubst v1.4.2/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
-github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk=
-github.com/alecthomas/participle/v2 v2.0.0 h1:Fgrq+MbuSsJwIkw3fEj9h75vDP0Er5JzepJ0/HNHv0g=
-github.com/alecthomas/participle/v2 v2.0.0/go.mod h1:rAKZdJldHu8084ojcWevWAL8KmEU+AT+Olodb+WoN2Y=
-github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk=
-github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/a8m/envsubst v1.3.0 h1:GmXKmVssap0YtlU3E230W98RWtWCyIZzjtf1apWWyAg=
+github.com/a8m/envsubst v1.3.0/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
+github.com/alecthomas/assert/v2 v2.0.3 h1:WKqJODfOiQG0nEJKFKzDIG3E29CN2/4zR9XGJzKIkbg=
+github.com/alecthomas/participle/v2 v2.0.0-beta.4 h1:ublfGBm+x+p2j7KotHhrUMbKtejT7M0Gv1Mt1u3absw=
+github.com/alecthomas/participle/v2 v2.0.0-beta.4/go.mod h1:RC764t6n4L8D8ITAJv0qdokritYSNR3wV5cVwmIEaMM=
+github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE=
+github.com/alecthomas/repr v0.1.0/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
-github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
-github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
-github.com/elliotchance/orderedmap v1.5.0 h1:1IsExUsjv5XNBD3ZdC7jkAAqLWOOKdbPTmkHx63OsBg=
-github.com/elliotchance/orderedmap v1.5.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
-github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
-github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
-github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
-github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
-github.com/goccy/go-yaml v1.11.0 h1:n7Z+zx8S9f9KgzG6KtQKf+kwqXZlLNR2F6018Dgau54=
-github.com/goccy/go-yaml v1.11.0/go.mod h1:H+mJrWtjPTJAHvRbV09MCK9xYwODM+wRTVFFTWckfng=
+github.com/elliotchance/orderedmap v1.4.0 h1:wZtfeEONCbx6in1CZyE6bELEt/vFayMvsxqI5SgsR+A=
+github.com/elliotchance/orderedmap v1.4.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
+github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
+github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
+github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
+github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
+github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
+github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
+github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
+github.com/goccy/go-json v0.9.10 h1:hCeNmprSNLB8B8vQKWl6DpuH0t60oEs+TAk9a7CScKc=
+github.com/goccy/go-json v0.9.10/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/goccy/go-yaml v1.9.5 h1:Eh/+3uk9kLxG4koCX6lRMAPS1OaMSAi+FJcya0INdB0=
+github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
-github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
-github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg=
 github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
-github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
-github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
-github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
-github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
-github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ=
-github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4=
+github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
+github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=
+github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
+github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
+github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
-github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
+github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
+github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
-github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
-golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
-golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
-golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20220708220712-1185a9018129 h1:vucSRfWwTsoXro7P+3Cjlr6flUMtzCwzlvkxEQtHHB0=
+golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e h1:NHvCuwuS43lGnYhten69ZWqi2QOj/CiDNcKbVqwVoew=
+golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0=
 golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW8s2qTSe3wGBtvo0MbVQG/c5k8RE=
 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -37,7 +37,7 @@ The `=` operator then pipes the 'root' context through the `rhs` expression of `
 dog
 ```

-Both sides have now been evaluated, so now the operator copies across the value from the RHS (`.b`) to the LHS (`.a`), and it returns the now updated context:
+Both sides have now been evaluated, so now the operator copies across the value from the RHS (`.b`) to the the LHS (`.a`), and it returns the now updated context:

 ```yaml
 a: dog
@@ -8,7 +8,7 @@ import (

 // A yaml expression evaluator that runs the expression once against all files/nodes in memory.
 type Evaluator interface {
-	EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error
+	EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error

 	// EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes
 	EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error)
@@ -46,16 +46,21 @@ func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCand
 	return context.MatchingNodes, nil
 }

-func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error {
+func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error {
 	fileIndex := 0
+	firstFileLeadingContent := ""

 	var allDocuments = list.New()
 	for _, filename := range filenames {
-		reader, err := readStream(filename)
+		reader, leadingContent, err := readStream(filename, fileIndex == 0 && leadingContentPreProcessing)
 		if err != nil {
 			return err
 		}

+		if fileIndex == 0 {
+			firstFileLeadingContent = leadingContent
+		}
+
 		fileDocuments, err := readDocuments(reader, filename, fileIndex, decoder)
 		if err != nil {
 			return err
@@ -70,9 +75,11 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
 			Filename:       "",
 			Node:           &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
 			FileIndex:      0,
-			LeadingContent: "",
+			LeadingContent: firstFileLeadingContent,
 		}
 		allDocuments.PushBack(candidateNode)
+	} else {
+		allDocuments.Front().Value.(*CandidateNode).LeadingContent = firstFileLeadingContent
 	}

 	matches, err := e.EvaluateCandidateNodes(expression, allDocuments)
@@ -157,7 +157,7 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignP
 	n.Node.Kind = other.Node.Kind

 	// don't clobber custom tags...
-	if prefs.ClobberCustomTags || strings.HasPrefix(n.Node.Tag, "!!") || n.Node.Tag == "" {
+	if strings.HasPrefix(n.Node.Tag, "!!") || n.Node.Tag == "" {
 		n.Node.Tag = other.Node.Tag
 	}

@@ -1,25 +0,0 @@
-//go:build linux
-
-package yqlib
-
-import (
-	"io/fs"
-	"os"
-	"syscall"
-)
-
-func changeOwner(info fs.FileInfo, file *os.File) error {
-	if stat, ok := info.Sys().(*syscall.Stat_t); ok {
-		uid := int(stat.Uid)
-		gid := int(stat.Gid)
-
-		err := os.Chown(file.Name(), uid, gid)
-		if err != nil {
-			// this happens with snap confinement
-			// not really a big issue as users can chown
-			// the file themselves if required.
-			log.Info("Skipping chown: %v", err)
-		}
-	}
-	return nil
-}
@@ -1,12 +0,0 @@
-//go:build !linux
-
-package yqlib
-
-import (
-	"io/fs"
-	"os"
-)
-
-func changeOwner(info fs.FileInfo, file *os.File) error {
-	return nil
-}
@@ -3,6 +3,7 @@ package yqlib
 import (
 	"fmt"
 	"strconv"
+	"time"

 	yaml "gopkg.in/yaml.v3"
 )
@@ -15,7 +16,7 @@ type compareTypePref struct {
 func compareOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- compareOperator")
 	prefs := expressionNode.Operation.Preferences.(compareTypePref)
-	return crossFunction(d, context, expressionNode, compare(prefs), true)
+	return crossFunction(d, context.ReadOnlyClone(), expressionNode, compare(prefs), true)
 }

 func compare(prefs compareTypePref) func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
@@ -53,12 +54,12 @@ func compare(prefs compareTypePref) func(d *dataTreeNavigator, context Context,
 }

 func compareDateTime(layout string, prefs compareTypePref, lhs *yaml.Node, rhs *yaml.Node) (bool, error) {
-	lhsTime, err := parseDateTime(layout, lhs.Value)
+	lhsTime, err := time.Parse(layout, lhs.Value)
 	if err != nil {
 		return false, err
 	}

-	rhsTime, err := parseDateTime(layout, rhs.Value)
+	rhsTime, err := time.Parse(layout, rhs.Value)
 	if err != nil {
 		return false, err
 	}
@@ -79,8 +80,8 @@ func compareScalars(context Context, prefs compareTypePref, lhs *yaml.Node, rhs

 	isDateTime := lhs.Tag == "!!timestamp"
 	// if the lhs is a string, it might be a timestamp in a custom format.
-	if lhsTag == "!!str" {
-		_, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value)
+	if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
+		_, err := time.Parse(context.GetDateTimeLayout(), lhs.Value)
 		isDateTime = err == nil
 	}
 	if isDateTime {
@@ -126,10 +127,6 @@ func compareScalars(context Context, prefs compareTypePref, lhs *yaml.Node, rhs
 			return lhs.Value > rhs.Value, nil
 		}
 		return lhs.Value < rhs.Value, nil
-	} else if lhsTag == "!!null" && rhsTag == "!!null" && prefs.OrEqual {
-		return true, nil
-	} else if lhsTag == "!!null" || rhsTag == "!!null" {
-		return false, nil
 	}

 	return false, fmt.Errorf("%v not yet supported for comparison", lhs.Tag)
@@ -14,20 +14,6 @@ var compareOperatorScenarios = []expressionScenario{
 			"D0, P[a], (!!bool)::true\n",
 		},
 	},
-	{
-		skipDoc:    true,
-		expression: "(.k | length) >= 0",
-		expected: []string{
-			"D0, P[k], (!!bool)::true\n",
-		},
-	},
-	{
-		skipDoc:    true,
-		expression: `"2022-01-30T15:53:09Z" > "2020-01-30T15:53:09Z"`,
-		expected: []string{
-			"D0, P[], (!!bool)::true\n",
-		},
-	},
 	{
 		skipDoc:    true,
 		document:   "a: 5\nb: 4",
@@ -299,28 +285,28 @@ var compareOperatorScenarios = []expressionScenario{
 		description: "Both sides are null: > is false",
 		expression:  ".a > .b",
 		expected: []string{
-			"D0, P[a], (!!bool)::false\n",
+			"D0, P[], (!!bool)::false\n",
 		},
 	},
 	{
 		skipDoc:    true,
 		expression: ".a < .b",
 		expected: []string{
-			"D0, P[a], (!!bool)::false\n",
+			"D0, P[], (!!bool)::false\n",
 		},
 	},
 	{
 		description: "Both sides are null: >= is true",
 		expression:  ".a >= .b",
 		expected: []string{
-			"D0, P[a], (!!bool)::true\n",
+			"D0, P[], (!!bool)::true\n",
 		},
 	},
 	{
 		skipDoc:    true,
 		expression: ".a <= .b",
 		expected: []string{
-			"D0, P[a], (!!bool)::true\n",
+			"D0, P[], (!!bool)::true\n",
 		},
 	},

@@ -364,7 +350,7 @@ var compareOperatorScenarios = []expressionScenario{
 		document:   `a: 5`,
 		expression: ".b <= .a",
 		expected: []string{
-			"D0, P[b], (!!bool)::false\n",
+			"D0, P[a], (!!bool)::false\n",
 		},
 	},
 	{
@@ -372,7 +358,7 @@ var compareOperatorScenarios = []expressionScenario{
 		document:   `a: 5`,
 		expression: ".b < .a",
 		expected: []string{
-			"D0, P[b], (!!bool)::false\n",
+			"D0, P[a], (!!bool)::false\n",
 		},
 	},
 }
@@ -12,9 +12,7 @@ const csvSimple = `name,numberOfCats,likesApples,height
 Gary,1,true,168.8
 Samantha's Rabbit,2,false,-188.8
 `
-const csvMissing = `name,numberOfCats,likesApples,height
-,null,,168.8
-`
+
 const expectedUpdatedSimpleCsv = `name,numberOfCats,likesApples,height
 Gary,3,true,168.8
 Samantha's Rabbit,2,false,-188.8
@@ -112,13 +110,6 @@ var csvScenarios = []formatScenario{
 		expected:       csvSimpleMissingData,
 		scenarioType:   "encode-csv",
 	},
-	{
-		description:  "decode csv missing",
-		skipDoc:      true,
-		input:        csvMissing,
-		expected:     csvMissing,
-		scenarioType: "roundtrip-csv",
-	},
 	{
 		description:    "Parse CSV into an array of objects",
 		subdescription: "First row is assumed to be the header row.",
@@ -126,14 +117,6 @@ var csvScenarios = []formatScenario{
 		expected:       expectedYamlFromCSV,
 		scenarioType:   "decode-csv-object",
 	},
-	{
-		description:  "Scalar roundtrip",
-		skipDoc:      true,
-		input:        "mike\ncat",
-		expression:   ".[0].mike",
-		expected:     "cat\n",
-		scenarioType: "roundtrip-csv",
-	},
 	{
 		description:    "Parse TSV into an array of objects",
 		subdescription: "First row is assumed to be the header row.",
@@ -153,21 +136,21 @@ var csvScenarios = []formatScenario{
 func testCSVScenario(t *testing.T, s formatScenario) {
 	switch s.scenarioType {
 	case "encode-csv":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(',')), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder(',')), s.description)
 	case "encode-tsv":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder('\t')), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder('\t')), s.description)
 	case "decode-csv-object":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewYamlEncoder(2, false, true, true)), s.description)
 	case "decode-tsv-object":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, ConfiguredYamlPreferences)), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder('\t'), NewYamlEncoder(2, false, true, true)), s.description)
 	case "roundtrip-csv":
-		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewCSVObjectDecoder(','), NewCsvEncoder(',')), s.description)
+		test.AssertResultWithContext(t, s.expected, processFormatScenario(s, NewCSVObjectDecoder(','), NewCsvEncoder(',')), s.description)
 	default:
 		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
 	}
 }

-func documentCSVDecodeObjectScenario(w *bufio.Writer, s formatScenario, formatType string) {
+func documentCSVDecodeObjectScenario(t *testing.T, w *bufio.Writer, s formatScenario, formatType string) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))

 	if s.subdescription != "" {
@@ -188,7 +171,7 @@ func documentCSVDecodeObjectScenario(w *bufio.Writer, s formatScenario, formatTy
 	}

 	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n",
-		mustProcessFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, ConfiguredYamlPreferences))),
+		processFormatScenario(s, NewCSVObjectDecoder(separator), NewYamlEncoder(s.indent, false, true, true))),
 	)
 }

@@ -220,7 +203,7 @@ func documentCSVEncodeScenario(w *bufio.Writer, s formatScenario, formatType str
 	}

 	writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
-		mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(separator))),
+		processFormatScenario(s, NewYamlDecoder(), NewCsvEncoder(separator))),
 	)
 }

@@ -252,7 +235,7 @@ func documentCSVRoundTripScenario(w *bufio.Writer, s formatScenario, formatType
 	}

 	writeOrPanic(w, fmt.Sprintf("```%v\n%v```\n\n", formatType,
-		mustProcessFormatScenario(s, NewCSVObjectDecoder(separator), NewCsvEncoder(separator))),
+		processFormatScenario(s, NewCSVObjectDecoder(separator), NewCsvEncoder(separator))),
 	)
 }

@@ -267,9 +250,9 @@ func documentCSVScenario(t *testing.T, w *bufio.Writer, i interface{}) {
 	case "encode-tsv":
 		documentCSVEncodeScenario(w, s, "tsv")
 	case "decode-csv-object":
-		documentCSVDecodeObjectScenario(w, s, "csv")
+		documentCSVDecodeObjectScenario(t, w, s, "csv")
 	case "decode-tsv-object":
-		documentCSVDecodeObjectScenario(w, s, "tsv")
+		documentCSVDecodeObjectScenario(t, w, s, "tsv")
 	case "roundtrip-csv":
 		documentCSVRoundTripScenario(w, s, "csv")

@@ -4,7 +4,6 @@ import (
 	"fmt"

 	logging "gopkg.in/op/go-logging.v1"
-	yaml "gopkg.in/yaml.v3"
 )

 type DataTreeNavigator interface {
@@ -12,8 +11,6 @@ type DataTreeNavigator interface {
 	// this will process the against the given expressionNode and return
 	// a new context of matching candidates
 	GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error)
-
-	DeeplyAssign(context Context, path []interface{}, rhsNode *yaml.Node) error
 }

 type dataTreeNavigator struct {
@@ -23,27 +20,6 @@ func NewDataTreeNavigator() DataTreeNavigator {
 	return &dataTreeNavigator{}
 }

-func (d *dataTreeNavigator) DeeplyAssign(context Context, path []interface{}, rhsNode *yaml.Node) error {
-
-	rhsCandidateNode := &CandidateNode{
-		Path: path,
-		Node: rhsNode,
-	}
-
-	assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
-
-	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}
-
-	assignmentOpNode := &ExpressionNode{
-		Operation: assignmentOp,
-		LHS:       createTraversalTree(path, traversePreferences{}, false),
-		RHS:       &ExpressionNode{Operation: rhsOp},
-	}
-
-	_, err := d.GetMatchingNodes(context, assignmentOpNode)
-	return err
-}
-
 func (d *dataTreeNavigator) GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error) {
 	if expressionNode == nil {
 		log.Debugf("getMatchingNodes - nothing to do")
@@ -3,7 +3,8 @@ package yqlib
 import (
 	"fmt"
 	"io"
-	"strings"
+
+	yaml "gopkg.in/yaml.v3"
 )

 type InputFormat uint
@@ -16,22 +17,20 @@ const (
 	JsonInputFormat
 	CSVObjectInputFormat
 	TSVObjectInputFormat
-	TomlInputFormat
-	UriInputFormat
 )

 type Decoder interface {
-	Init(reader io.Reader) error
-	Decode() (*CandidateNode, error)
+	Init(reader io.Reader)
+	Decode(node *yaml.Node) error
 }

 func InputFormatFromString(format string) (InputFormat, error) {
 	switch format {
-	case "yaml", "yml", "y":
+	case "yaml", "y":
 		return YamlInputFormat, nil
 	case "xml", "x":
 		return XMLInputFormat, nil
-	case "properties", "props", "p":
+	case "props", "p":
 		return PropertiesInputFormat, nil
 	case "json", "ndjson", "j":
 		return JsonInputFormat, nil
@@ -39,25 +38,7 @@ func InputFormatFromString(format string) (InputFormat, error) {
 		return CSVObjectInputFormat, nil
 	case "tsv", "t":
 		return TSVObjectInputFormat, nil
-	case "toml":
-		return TomlInputFormat, nil
 	default:
-		return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml]", format)
+		return 0, fmt.Errorf("unknown format '%v' please use [yaml|xml|props]", format)
 	}
 }
-
-func FormatFromFilename(filename string) string {
-
-	if filename != "" {
-		GetLogger().Debugf("checking file extension '%s' for auto format detection", filename)
-		nPos := strings.LastIndex(filename, ".")
-		if nPos > -1 {
-			format := filename[nPos+1:]
-			GetLogger().Debugf("detected format '%s'", format)
-			return format
-		}
-	}
-
-	GetLogger().Debugf("using default inputFormat 'yaml'")
-	return "yaml"
-}
@@ -4,33 +4,10 @@ import (
 	"bytes"
 	"encoding/base64"
 	"io"
-	"strings"

 	yaml "gopkg.in/yaml.v3"
 )

-type base64Padder struct {
-	count uint64
-	io.Reader
-}
-
-func (c *base64Padder) pad(buf []byte) (int, error) {
-	pad := strings.Repeat("=", int(4-c.count%4))
-	n, err := strings.NewReader(pad).Read(buf)
-	c.count += uint64(n)
-	return n, err
-}
-
-func (c *base64Padder) Read(buf []byte) (int, error) {
-	n, err := c.Reader.Read(buf)
-	c.count += uint64(n)
-
-	if err == io.EOF && c.count%4 != 0 {
-		return c.pad(buf)
-	}
-	return n, err
-}
-
 type base64Decoder struct {
 	reader       io.Reader
 	finished     bool
@@ -42,22 +19,21 @@ func NewBase64Decoder() Decoder {
 	return &base64Decoder{finished: false, encoding: *base64.StdEncoding}
 }

-func (dec *base64Decoder) Init(reader io.Reader) error {
-	dec.reader = &base64Padder{Reader: reader}
+func (dec *base64Decoder) Init(reader io.Reader) {
+	dec.reader = reader
 	dec.readAnything = false
 	dec.finished = false
-	return nil
 }

-func (dec *base64Decoder) Decode() (*CandidateNode, error) {
+func (dec *base64Decoder) Decode(rootYamlNode *yaml.Node) error {
 	if dec.finished {
-		return nil, io.EOF
+		return io.EOF
 	}
 	base64Reader := base64.NewDecoder(&dec.encoding, dec.reader)
 	buf := new(bytes.Buffer)

 	if _, err := buf.ReadFrom(base64Reader); err != nil {
-		return nil, err
+		return err
 	}
 	if buf.Len() == 0 {
 		dec.finished = true
@@ -66,15 +42,12 @@ func (dec *base64Decoder) Decode() (*CandidateNode, error) {
 		// otherwise if we've already read some bytes, and now we get
 		// an empty string, then we are done.
 		if dec.readAnything {
-			return nil, io.EOF
+			return io.EOF
 		}
 	}
 	dec.readAnything = true
-	return &CandidateNode{
-		Node: &yaml.Node{
-			Kind:  yaml.ScalarNode,
-			Tag:   "!!str",
-			Value: buf.String(),
-		},
-	}, nil
+	rootYamlNode.Kind = yaml.ScalarNode
+	rootYamlNode.Tag = "!!str"
+	rootYamlNode.Value = buf.String()
+	return nil
 }
@@ -5,7 +5,6 @@ import (
 	"errors"
 	"io"

-	"github.com/dimchansky/utfbom"
 	yaml "gopkg.in/yaml.v3"
 )

@@ -19,13 +18,10 @@ func NewCSVObjectDecoder(separator rune) Decoder {
 	return &csvObjectDecoder{separator: separator}
 }

-func (dec *csvObjectDecoder) Init(reader io.Reader) error {
-	cleanReader, enc := utfbom.Skip(reader)
-	log.Debugf("Detected encoding: %s\n", enc)
-	dec.reader = *csv.NewReader(cleanReader)
+func (dec *csvObjectDecoder) Init(reader io.Reader) {
+	dec.reader = *csv.NewReader(reader)
 	dec.reader.Comma = dec.separator
 	dec.finished = false
-	return nil
 }

 func (dec *csvObjectDecoder) convertToYamlNode(content string) *yaml.Node {
@@ -48,14 +44,14 @@ func (dec *csvObjectDecoder) createObject(headerRow []string, contentRow []strin
 	return objectNode
 }

-func (dec *csvObjectDecoder) Decode() (*CandidateNode, error) {
+func (dec *csvObjectDecoder) Decode(rootYamlNode *yaml.Node) error {
 	if dec.finished {
-		return nil, io.EOF
+		return io.EOF
 	}
 	headerRow, err := dec.reader.Read()
 	log.Debugf(": headerRow%v", headerRow)
 	if err != nil {
-		return nil, err
+		return err
 	}

 	rootArray := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
@@ -69,13 +65,13 @@ func (dec *csvObjectDecoder) Decode() (*CandidateNode, error) {
 		log.Debugf("Read next contentRow: %v, %v", contentRow, err)
 	}
 	if !errors.Is(err, io.EOF) {
-		return nil, err
+		return err
 	}

-	return &CandidateNode{
-		Node: &yaml.Node{
-			Kind:    yaml.DocumentNode,
-			Content: []*yaml.Node{rootArray},
-		},
-	}, nil
+	log.Debugf("finished, contentRow%v", contentRow)
+	log.Debugf("err: %v", err)
+
+	rootYamlNode.Kind = yaml.DocumentNode
+	rootYamlNode.Content = []*yaml.Node{rootArray}
+	return nil
 }
@@ -1,5 +1,3 @@
-//go:build !yq_nojson
-
 package yqlib

 import (
@@ -18,68 +16,63 @@ func NewJSONDecoder() Decoder {
 	return &jsonDecoder{}
 }

-func (dec *jsonDecoder) Init(reader io.Reader) error {
+func (dec *jsonDecoder) Init(reader io.Reader) {
 	dec.decoder = *json.NewDecoder(reader)
-	return nil
 }

-func (dec *jsonDecoder) Decode() (*CandidateNode, error) {
+func (dec *jsonDecoder) Decode(rootYamlNode *yaml.Node) error {

-	var dataBucket orderedMap
+	var dataBucket interface{}
 	log.Debug("going to decode")
 	err := dec.decoder.Decode(&dataBucket)
 	if err != nil {
-		return nil, err
+		return err
 	}
-	node, err := dec.convertToYamlNode(&dataBucket)
+	node, err := dec.convertToYamlNode(dataBucket)

 	if err != nil {
-		return nil, err
+		return err
 	}
-
-	return &CandidateNode{
-		Node: &yaml.Node{
-			Kind:    yaml.DocumentNode,
-			Content: []*yaml.Node{node},
-		},
-	}, nil
+	rootYamlNode.Kind = yaml.DocumentNode
+	rootYamlNode.Content = []*yaml.Node{node}
+	return nil
 }

-func (dec *jsonDecoder) convertToYamlNode(data *orderedMap) (*yaml.Node, error) {
-	if data == nil {
+func (dec *jsonDecoder) convertToYamlNode(data interface{}) (*yaml.Node, error) {
+	switch data := data.(type) {
+	case nil:
 		return createScalarNode(nil, "null"), nil
+	case float64, float32:
+		// json decoder returns ints as float.
+		return parseSnippet(fmt.Sprintf("%v", data))
+	case int, int64, int32, string, bool:
+		return createScalarNode(data, fmt.Sprintf("%v", data)), nil
+	case map[string]interface{}:
+		return dec.parseMap(data)
+	case []interface{}:
+		return dec.parseArray(data)
+	default:
+		return nil, fmt.Errorf("unrecognised type :(")
 	}
-	if data.kv == nil {
-		switch rawData := data.altVal.(type) {
-		case nil:
-			return createScalarNode(nil, "null"), nil
-		case float64, float32:
-			// json decoder returns ints as float.
-			return parseSnippet(fmt.Sprintf("%v", rawData))
-		case int, int64, int32, string, bool:
-			return createScalarNode(rawData, fmt.Sprintf("%v", rawData)), nil
-		case []*orderedMap:
-			return dec.parseArray(rawData)
-		default:
-			return nil, fmt.Errorf("unrecognised type :( %v", rawData)
-		}
-	}
+}

-	var yamlMap = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
-	for _, keyValuePair := range data.kv {
-		yamlValue, err := dec.convertToYamlNode(&keyValuePair.V)
+func (dec *jsonDecoder) parseMap(dataMap map[string]interface{}) (*yaml.Node, error) {
+
+	var yamlMap = &yaml.Node{Kind: yaml.MappingNode}
+
+	for key, value := range dataMap {
+		yamlValue, err := dec.convertToYamlNode(value)
 		if err != nil {
 			return nil, err
 		}
-		yamlMap.Content = append(yamlMap.Content, createScalarNode(keyValuePair.K, keyValuePair.K), yamlValue)
+		yamlMap.Content = append(yamlMap.Content, createScalarNode(key, fmt.Sprintf("%v", key)), yamlValue)
 	}
 	return yamlMap, nil
-
 }

-func (dec *jsonDecoder) parseArray(dataArray []*orderedMap) (*yaml.Node, error) {
+func (dec *jsonDecoder) parseArray(dataArray []interface{}) (*yaml.Node, error) {

-	var yamlMap = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+	var yamlMap = &yaml.Node{Kind: yaml.SequenceNode}

 	for _, value := range dataArray {
 		yamlValue, err := dec.convertToYamlNode(value)
@@ -2,7 +2,6 @@ package yqlib

 import (
 	"bytes"
-	"fmt"
 	"io"
 	"strconv"
 	"strings"
@@ -21,10 +20,9 @@ func NewPropertiesDecoder() Decoder {
 	return &propertiesDecoder{d: NewDataTreeNavigator(), finished: false}
 }

-func (dec *propertiesDecoder) Init(reader io.Reader) error {
+func (dec *propertiesDecoder) Init(reader io.Reader) {
 	dec.reader = reader
 	dec.finished = false
-	return nil
 }

 func parsePropKey(key string) []interface{} {
@@ -48,26 +46,31 @@ func (dec *propertiesDecoder) processComment(c string) string {
 	return "# " + c
 }

-func (dec *propertiesDecoder) applyPropertyComments(context Context, path []interface{}, comments []string) error {
-	assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
+func (dec *propertiesDecoder) applyProperty(properties *properties.Properties, context Context, key string) error {
+	value, _ := properties.Get(key)
+	path := parsePropKey(key)
+
+	rhsNode := &yaml.Node{
+		Value:       value,
+		Tag:         "!!str",
+		Kind:        yaml.ScalarNode,
+		LineComment: dec.processComment(properties.GetComment(key)),
+	}
+
+	rhsNode.Tag = guessTagFromCustomType(rhsNode)

 	rhsCandidateNode := &CandidateNode{
 		Path: path,
-		Node: &yaml.Node{
-			Tag:         "!!str",
-			Value:       fmt.Sprintf("%v", path[len(path)-1]),
-			HeadComment: dec.processComment(strings.Join(comments, "\n")),
-			Kind:        yaml.ScalarNode,
-		},
+		Node: rhsNode,
 	}

-	rhsCandidateNode.Node.Tag = guessTagFromCustomType(rhsCandidateNode.Node)
+	assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}

-	rhsOp := &Operation{OperationType: referenceOpType, CandidateNode: rhsCandidateNode}
+	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}

 	assignmentOpNode := &ExpressionNode{
 		Operation: assignmentOp,
-		LHS:       createTraversalTree(path, traversePreferences{}, true),
+		LHS:       createTraversalTree(path, traversePreferences{}, false),
 		RHS:       &ExpressionNode{Operation: rhsOp},
 	}

@@ -75,45 +78,22 @@ func (dec *propertiesDecoder) applyPropertyComments(context Context, path []inte
 	return err
 }

-func (dec *propertiesDecoder) applyProperty(context Context, properties *properties.Properties, key string) error {
-	value, _ := properties.Get(key)
-	path := parsePropKey(key)
-
-	propertyComments := properties.GetComments(key)
-	if len(propertyComments) > 0 {
-		err := dec.applyPropertyComments(context, path, propertyComments)
-		if err != nil {
-			return nil
-		}
-	}
-
-	rhsNode := &yaml.Node{
-		Value: value,
-		Tag:   "!!str",
-		Kind:  yaml.ScalarNode,
-	}
-
-	rhsNode.Tag = guessTagFromCustomType(rhsNode)
-
-	return dec.d.DeeplyAssign(context, path, rhsNode)
-}
-
-func (dec *propertiesDecoder) Decode() (*CandidateNode, error) {
+func (dec *propertiesDecoder) Decode(rootYamlNode *yaml.Node) error {
 	if dec.finished {
-		return nil, io.EOF
+		return io.EOF
 	}
 	buf := new(bytes.Buffer)

 	if _, err := buf.ReadFrom(dec.reader); err != nil {
-		return nil, err
+		return err
 	}
 	if buf.Len() == 0 {
 		dec.finished = true
-		return nil, io.EOF
+		return io.EOF
 	}
 	properties, err := properties.LoadString(buf.String())
 	if err != nil {
-		return nil, err
+		return err
 	}
 	properties.DisableExpansion = true

@@ -128,18 +108,15 @@ func (dec *propertiesDecoder) Decode() (*CandidateNode, error) {
 	context = context.SingleChildContext(rootMap)

 	for _, key := range properties.Keys() {
-		if err := dec.applyProperty(context, properties, key); err != nil {
-			return nil, err
+		if err := dec.applyProperty(properties, context, key); err != nil {
+			return err
 		}

 	}
-	dec.finished = true

-	return &CandidateNode{
-		Node: &yaml.Node{
-			Kind:    yaml.DocumentNode,
-			Content: []*yaml.Node{rootMap.Node},
-		},
-	}, nil
+	rootYamlNode.Kind = yaml.DocumentNode
+	rootYamlNode.Content = []*yaml.Node{rootMap.Node}
+	dec.finished = true
+	return nil

 }
@@ -3,7 +3,6 @@ package yqlib
 import (
 	"bufio"
 	"bytes"
-	"fmt"
 	"strings"
 )

@@ -16,20 +15,20 @@ type formatScenario struct {
 	subdescription string
 	skipDoc        bool
 	scenarioType   string
-	expectedError  string
 }

-func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (string, error) {
+func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) string {
+
 	var output bytes.Buffer
 	writer := bufio.NewWriter(&output)

 	if decoder == nil {
-		decoder = NewYamlDecoder(ConfiguredYamlPreferences)
+		decoder = NewYamlDecoder()
 	}

 	inputs, err := readDocuments(strings.NewReader(s.input), "sample.yml", 0, decoder)
 	if err != nil {
-		return "", err
+		panic(err)
 	}

 	expression := s.expression
@@ -40,31 +39,22 @@ func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (
 	exp, err := getExpressionParser().ParseExpression(expression)

 	if err != nil {
-		return "", err
+		panic(err)
 	}

 	context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: inputs}, exp)

 	if err != nil {
-		return "", err
+		panic(err)
 	}

 	printer := NewPrinter(encoder, NewSinglePrinterWriter(writer))
 	err = printer.PrintResults(context.MatchingNodes)
 	if err != nil {
-		return "", err
+		panic(err)
 	}
 	writer.Flush()

-	return output.String(), nil
-}
-
-func mustProcessFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) string {
-
-	result, err := processFormatScenario(s, decoder, encoder)
-	if err != nil {
-		panic(fmt.Errorf("Bad scenario %v: %w", s.description, err))
-	}
-	return result
+	return output.String()

 }
@@ -1,369 +0,0 @@
-//go:build !yq_notoml
-
-package yqlib
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"io"
-	"strconv"
-	"time"
-
-	toml "github.com/pelletier/go-toml/v2/unstable"
-	yaml "gopkg.in/yaml.v3"
-)
-
-type tomlDecoder struct {
-	parser   toml.Parser
-	finished bool
-	d        DataTreeNavigator
-	rootMap  *CandidateNode
-}
-
-func NewTomlDecoder() Decoder {
-	return &tomlDecoder{
-		finished: false,
-		d:        NewDataTreeNavigator(),
-	}
-}
-
-func (dec *tomlDecoder) Init(reader io.Reader) error {
-	dec.parser = toml.Parser{}
-	buf := new(bytes.Buffer)
-	_, err := buf.ReadFrom(reader)
-	if err != nil {
-		return err
-	}
-	dec.parser.Reset(buf.Bytes())
-	dec.rootMap = &CandidateNode{
-		Node: &yaml.Node{
-			Kind: yaml.MappingNode,
-			Tag:  "!!map",
-		}}
-	return nil
-}
-
-func (dec *tomlDecoder) getFullPath(tomlNode *toml.Node) []interface{} {
-	path := make([]interface{}, 0)
-	for {
-		path = append(path, string(tomlNode.Data))
-		tomlNode = tomlNode.Next()
-		if tomlNode == nil {
-			return path
-		}
-	}
-}
-
-func (dec *tomlDecoder) processKeyValueIntoMap(rootMap *CandidateNode, tomlNode *toml.Node) error {
-	value := tomlNode.Value()
-	path := dec.getFullPath(value.Next())
-	log.Debug("!!!processKeyValueIntoMap: %v", path)
-
-	valueNode, err := dec.decodeNode(value)
-	if err != nil {
-		return err
-	}
-	context := Context{}
-	context = context.SingleChildContext(rootMap)
-
-	return dec.d.DeeplyAssign(context, path, valueNode)
-}
-
-func (dec *tomlDecoder) decodeKeyValuesIntoMap(rootMap *CandidateNode, tomlNode *toml.Node) (bool, error) {
-	log.Debug("!! DECODE_KV_INTO_MAP -- processing first (current) entry")
-	if err := dec.processKeyValueIntoMap(rootMap, tomlNode); err != nil {
-		return false, err
-	}
-
-	for dec.parser.NextExpression() {
-		nextItem := dec.parser.Expression()
-		log.Debug("!! DECODE_KV_INTO_MAP -- next exp, its a %v", nextItem.Kind)
-
-		if nextItem.Kind == toml.KeyValue {
-			if err := dec.processKeyValueIntoMap(rootMap, nextItem); err != nil {
-				return false, err
-			}
-		} else {
-			// run out of key values
-			log.Debug("! DECODE_KV_INTO_MAP - ok we are done in decodeKeyValuesIntoMap, gota a %v", nextItem.Kind)
-			log.Debug("! DECODE_KV_INTO_MAP - processAgainstCurrentExp = true!")
-			return true, nil
-		}
-	}
-	log.Debug("! DECODE_KV_INTO_MAP - no more things to read in")
-	return false, nil
-}
-
-func (dec *tomlDecoder) createInlineTableMap(tomlNode *toml.Node) (*yaml.Node, error) {
-	content := make([]*yaml.Node, 0)
-	log.Debug("!! createInlineTableMap")
-
-	iterator := tomlNode.Children()
-	for iterator.Next() {
-		child := iterator.Node()
-		if child.Kind != toml.KeyValue {
-			return nil, fmt.Errorf("only keyvalue pairs are supported in inlinetables, got %v instead", child.Kind)
-		}
-
-		keyValues := &CandidateNode{
-			Node: &yaml.Node{
-				Kind: yaml.MappingNode,
-				Tag:  "!!map",
-			},
-		}
-
-		if err := dec.processKeyValueIntoMap(keyValues, child); err != nil {
-			return nil, err
-		}
-
-		content = append(content, keyValues.Node.Content...)
-	}
-
-	return &yaml.Node{
-		Kind:    yaml.MappingNode,
-		Tag:     "!!map",
-		Content: content,
-	}, nil
-}
-
-func (dec *tomlDecoder) createArray(tomlNode *toml.Node) (*yaml.Node, error) {
-	content := make([]*yaml.Node, 0)
-	iterator := tomlNode.Children()
-	for iterator.Next() {
-		child := iterator.Node()
-		yamlNode, err := dec.decodeNode(child)
-		if err != nil {
-			return nil, err
-		}
-		content = append(content, yamlNode)
-	}
-
-	return &yaml.Node{
-		Kind:    yaml.SequenceNode,
-		Tag:     "!!seq",
-		Content: content,
-	}, nil
-
-}
-
-func (dec *tomlDecoder) createStringScalar(tomlNode *toml.Node) (*yaml.Node, error) {
-	content := string(tomlNode.Data)
-	return createScalarNode(content, content), nil
-}
-
-func (dec *tomlDecoder) createBoolScalar(tomlNode *toml.Node) (*yaml.Node, error) {
-	content := string(tomlNode.Data)
-	return createScalarNode(content == "true", content), nil
-}
-
-func (dec *tomlDecoder) createIntegerScalar(tomlNode *toml.Node) (*yaml.Node, error) {
-	content := string(tomlNode.Data)
-	_, num, err := parseInt64(content)
-	return createScalarNode(num, content), err
-}
-
-func (dec *tomlDecoder) createDateTimeScalar(tomlNode *toml.Node) (*yaml.Node, error) {
-	content := string(tomlNode.Data)
-	val, err := parseDateTime(time.RFC3339, content)
-	return createScalarNode(val, content), err
-}
-
-func (dec *tomlDecoder) createFloatScalar(tomlNode *toml.Node) (*yaml.Node, error) {
-	content := string(tomlNode.Data)
-	num, err := strconv.ParseFloat(content, 64)
-	return createScalarNode(num, content), err
-}
-
-func (dec *tomlDecoder) decodeNode(tomlNode *toml.Node) (*yaml.Node, error) {
-	switch tomlNode.Kind {
-	case toml.Key, toml.String:
-		return dec.createStringScalar(tomlNode)
-	case toml.Bool:
-		return dec.createBoolScalar(tomlNode)
-	case toml.Integer:
-		return dec.createIntegerScalar(tomlNode)
-	case toml.DateTime:
-		return dec.createDateTimeScalar(tomlNode)
-	case toml.Float:
-		return dec.createFloatScalar(tomlNode)
-	case toml.Array:
-		return dec.createArray(tomlNode)
-	case toml.InlineTable:
-		return dec.createInlineTableMap(tomlNode)
-	default:
-		return nil, fmt.Errorf("unsupported type %v", tomlNode.Kind)
-	}
-
-}
-
-func (dec *tomlDecoder) Decode() (*CandidateNode, error) {
-	if dec.finished {
-		return nil, io.EOF
-	}
-	//
-	// toml library likes to panic
-	var deferredError error
-	defer func() { //catch or finally
-		if r := recover(); r != nil {
-			var ok bool
-			deferredError, ok = r.(error)
-			if !ok {
-				deferredError = fmt.Errorf("pkg: %v", r)
-			}
-		}
-	}()
-
-	log.Debug("ok here we go")
-	var runAgainstCurrentExp = false
-	var err error
-	for runAgainstCurrentExp || dec.parser.NextExpression() {
-
-		if runAgainstCurrentExp {
-			log.Debug("running against current exp")
-		}
-
-		currentNode := dec.parser.Expression()
-
-		log.Debug("currentNode: %v ", currentNode.Kind)
-		runAgainstCurrentExp, err = dec.processTopLevelNode(currentNode)
-		if err != nil {
-			return dec.rootMap, err
-		}
-
-	}
-
-	err = dec.parser.Error()
-	if err != nil {
-		return nil, err
-	}
-
-	// must have finished
-	dec.finished = true
-
-	if len(dec.rootMap.Node.Content) == 0 {
-		return nil, io.EOF
-	}
-
-	return &CandidateNode{
-		Node: &yaml.Node{
-			Kind:    yaml.DocumentNode,
-			Content: []*yaml.Node{dec.rootMap.Node},
-		},
-	}, deferredError
-
-}
-
-func (dec *tomlDecoder) processTopLevelNode(currentNode *toml.Node) (bool, error) {
-	var runAgainstCurrentExp bool
-	var err error
-	log.Debug("!!!!!!!!!!!!Going to process %v state is current %v", currentNode.Kind, NodeToString(dec.rootMap))
-	if currentNode.Kind == toml.Table {
-		runAgainstCurrentExp, err = dec.processTable((currentNode))
-	} else if currentNode.Kind == toml.ArrayTable {
-		runAgainstCurrentExp, err = dec.processArrayTable((currentNode))
-	} else {
-		runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(dec.rootMap, currentNode)
-	}
-
-	log.Debug("!!!!!!!!!!!!DONE Processing state is now %v", NodeToString(dec.rootMap))
-	return runAgainstCurrentExp, err
-}
-
-func (dec *tomlDecoder) processTable(currentNode *toml.Node) (bool, error) {
-	log.Debug("!!! processing table")
-	fullPath := dec.getFullPath(currentNode.Child())
-	log.Debug("!!!fullpath: %v", fullPath)
-
-	hasValue := dec.parser.NextExpression()
-	if !hasValue {
-		return false, fmt.Errorf("error retrieving table %v value: %w", fullPath, dec.parser.Error())
-	}
-
-	tableNodeValue := &CandidateNode{
-		Node: &yaml.Node{
-			Kind: yaml.MappingNode,
-			Tag:  "!!map",
-		},
-	}
-
-	tableValue := dec.parser.Expression()
-	if tableValue.Kind != toml.KeyValue {
-		log.Debug("got an empty table, returning")
-		return true, nil
-	}
-
-	runAgainstCurrentExp, err := dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)
-	log.Debugf("table node err: %w", err)
-	if err != nil && !errors.Is(io.EOF, err) {
-		return false, err
-	}
-	c := Context{}
-
-	c = c.SingleChildContext(dec.rootMap)
-	err = dec.d.DeeplyAssign(c, fullPath, tableNodeValue.Node)
-	if err != nil {
-		return false, err
-	}
-	return runAgainstCurrentExp, nil
-}
-
-func (dec *tomlDecoder) arrayAppend(context Context, path []interface{}, rhsNode *yaml.Node) error {
-	rhsCandidateNode := &CandidateNode{
-		Path: path,
-		Node: &yaml.Node{
-			Kind:    yaml.SequenceNode,
-			Tag:     "!!seq",
-			Content: []*yaml.Node{rhsNode},
-		},
-	}
-
-	assignmentOp := &Operation{OperationType: addAssignOpType}
-
-	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}
-
-	assignmentOpNode := &ExpressionNode{
-		Operation: assignmentOp,
-		LHS:       createTraversalTree(path, traversePreferences{}, false),
-		RHS:       &ExpressionNode{Operation: rhsOp},
-	}
-
-	_, err := dec.d.GetMatchingNodes(context, assignmentOpNode)
-	return err
-}
-
-func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error) {
-	log.Debug("!!! processing table")
-	fullPath := dec.getFullPath(currentNode.Child())
-	log.Debug("!!!fullpath: %v", fullPath)
-
-	// need to use the array append exp to add another entry to
-	// this array: fullpath += [ thing ]
-
-	hasValue := dec.parser.NextExpression()
-	if !hasValue {
-		return false, fmt.Errorf("error retrieving table %v value: %w", fullPath, dec.parser.Error())
-	}
-
-	tableNodeValue := &CandidateNode{
-		Node: &yaml.Node{
-			Kind: yaml.MappingNode,
-			Tag:  "!!map",
-		},
-	}
-
-	tableValue := dec.parser.Expression()
-	runAgainstCurrentExp, err := dec.decodeKeyValuesIntoMap(tableNodeValue, tableValue)
-	log.Debugf("table node err: %w", err)
-	if err != nil && !errors.Is(io.EOF, err) {
-		return false, err
-	}
-	c := Context{}
-
-	c = c.SingleChildContext(dec.rootMap)
-
-	// += function
-	err = dec.arrayAppend(c, fullPath, tableNodeValue.Node)
-
-	return runAgainstCurrentExp, err
-}
@@ -1,60 +0,0 @@
-package yqlib
-
-import (
-	"bytes"
-	"io"
-	"net/url"
-
-	yaml "gopkg.in/yaml.v3"
-)
-
-type uriDecoder struct {
-	reader       io.Reader
-	finished     bool
-	readAnything bool
-}
-
-func NewUriDecoder() Decoder {
-	return &uriDecoder{finished: false}
-}
-
-func (dec *uriDecoder) Init(reader io.Reader) error {
-	dec.reader = reader
-	dec.readAnything = false
-	dec.finished = false
-	return nil
-}
-
-func (dec *uriDecoder) Decode() (*CandidateNode, error) {
-	if dec.finished {
-		return nil, io.EOF
-	}
-
-	buf := new(bytes.Buffer)
-
-	if _, err := buf.ReadFrom(dec.reader); err != nil {
-		return nil, err
-	}
-	if buf.Len() == 0 {
-		dec.finished = true
-
-		// if we've read _only_ an empty string, lets return that
-		// otherwise if we've already read some bytes, and now we get
-		// an empty string, then we are done.
-		if dec.readAnything {
-			return nil, io.EOF
-		}
-	}
-	newValue, err := url.QueryUnescape(buf.String())
-	if err != nil {
-		return nil, err
-	}
-	dec.readAnything = true
-	return &CandidateNode{
-		Node: &yaml.Node{
-			Kind:  yaml.ScalarNode,
-			Tag:   "!!str",
-			Value: newValue,
-		},
-	}, nil
-}
@@ -1,13 +1,9 @@
-//go:build !yq_noxml
-
 package yqlib

 import (
 	"encoding/xml"
 	"errors"
-	"fmt"
 	"io"
-	"regexp"
 	"strings"
 	"unicode"

@@ -16,24 +12,34 @@ import (
 )

 type xmlDecoder struct {
-	reader       io.Reader
-	readAnything bool
-	finished     bool
-	prefs        XmlPreferences
+	reader          io.Reader
+	readAnything    bool
+	attributePrefix string
+	contentName     string
+	strictMode      bool
+	keepNamespace   bool
+	useRawToken     bool
+	finished        bool
 }

-func NewXMLDecoder(prefs XmlPreferences) Decoder {
+func NewXMLDecoder(attributePrefix string, contentName string, strictMode bool, keepNamespace bool, useRawToken bool) Decoder {
+	if contentName == "" {
+		contentName = "content"
+	}
 	return &xmlDecoder{
-		finished: false,
-		prefs:    prefs,
+		attributePrefix: attributePrefix,
+		contentName:     contentName,
+		finished:        false,
+		strictMode:      strictMode,
+		keepNamespace:   keepNamespace,
+		useRawToken:     useRawToken,
 	}
 }

-func (dec *xmlDecoder) Init(reader io.Reader) error {
+func (dec *xmlDecoder) Init(reader io.Reader) {
 	dec.reader = reader
 	dec.readAnything = false
 	dec.finished = false
-	return nil
 }

 func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yaml.Node, error) {
@@ -49,33 +55,23 @@ func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yaml.Node, error) {
 	return yamlNode, nil
 }

-var decoderCommentPrefix = regexp.MustCompile(`(^|\n)([[:alpha:]])`)
-
 func (dec *xmlDecoder) processComment(c string) string {
 	if c == "" {
 		return ""
 	}
-	//need to replace "cat " with "# cat"
-	// "\ncat\n" with "\n cat\n"
-	// ensure non-empty comments starting with newline have a space in front
-
-	replacement := decoderCommentPrefix.ReplaceAllString(c, "$1 $2")
-	replacement = "#" + strings.ReplaceAll(strings.TrimRight(replacement, " "), "\n", "\n#")
-	return replacement
+	return "#" + strings.TrimRight(c, " ")
 }

 func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
-	log.Debug("createMap: headC: %v, lineC: %v, footC: %v", n.HeadComment, n.LineComment, n.FootComment)
+	log.Debug("createMap: headC: %v, footC: %v", n.HeadComment, n.FootComment)
 	yamlNode := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}

 	if len(n.Data) > 0 {
-		log.Debugf("creating content node for map: %v", dec.prefs.ContentName)
-		label := dec.prefs.ContentName
+		label := dec.contentName
 		labelNode := createScalarNode(label, label)
 		labelNode.HeadComment = dec.processComment(n.HeadComment)
-		labelNode.LineComment = dec.processComment(n.LineComment)
 		labelNode.FootComment = dec.processComment(n.FootComment)
-		yamlNode.Content = append(yamlNode.Content, labelNode, dec.createValueNodeFromData(n.Data))
+		yamlNode.Content = append(yamlNode.Content, labelNode, createScalarNode(n.Data, n.Data))
 	}

 	for i, keyValuePair := range n.Children {
@@ -86,12 +82,13 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
 		var err error

 		if i == 0 {
-			log.Debugf("head comment here")
 			labelNode.HeadComment = dec.processComment(n.HeadComment)

 		}
-		log.Debugf("label=%v, i=%v, keyValuePair.FootComment: %v", label, i, keyValuePair.FootComment)
+
+		// if i == len(n.Children)-1 {
 		labelNode.FootComment = dec.processComment(keyValuePair.FootComment)
+		// }

 		log.Debug("len of children in %v is %v", label, len(children))
 		if len(children) > 1 {
@@ -104,16 +101,8 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
 			// if the value is a scalar, the head comment of the scalar needs to go on the key?
 			// add tests for <z/> as well as multiple <ds> of inputXmlWithComments > yaml
 			if len(children[0].Children) == 0 && children[0].HeadComment != "" {
-				if len(children[0].Data) > 0 {
-
-					log.Debug("scalar comment hack, currentlabel [%v]", labelNode.HeadComment)
-					labelNode.HeadComment = joinComments([]string{labelNode.HeadComment, strings.TrimSpace(children[0].HeadComment)}, "\n")
-					children[0].HeadComment = ""
-				} else {
-					// child is null, put the headComment as a linecomment for reasons
-					children[0].LineComment = children[0].HeadComment
-					children[0].HeadComment = ""
-				}
+				labelNode.HeadComment = labelNode.HeadComment + "\n" + strings.TrimSpace(children[0].HeadComment)
+				children[0].HeadComment = ""
 			}
 			valueNode, err = dec.convertToYamlNode(children[0])
 			if err != nil {
@@ -126,75 +115,48 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
 	return yamlNode, nil
 }

-func (dec *xmlDecoder) createValueNodeFromData(values []string) *yaml.Node {
-	switch len(values) {
-	case 0:
-		return createScalarNode(nil, "")
-	case 1:
-		return createScalarNode(values[0], values[0])
-	default:
-		content := make([]*yaml.Node, 0)
-		for _, value := range values {
-			content = append(content, createScalarNode(value, value))
-		}
-		return &yaml.Node{
-			Kind:    yaml.SequenceNode,
-			Tag:     "!!seq",
-			Content: content,
-		}
-	}
-}
-
 func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yaml.Node, error) {
 	if len(n.Children) > 0 {
 		return dec.createMap(n)
 	}
-
-	scalar := dec.createValueNodeFromData(n.Data)
-
-	log.Debug("scalar (%v), headC: %v, lineC: %v, footC: %v", scalar.Tag, n.HeadComment, n.LineComment, n.FootComment)
+	scalar := createScalarNode(n.Data, n.Data)
+	if n.Data == "" {
+		scalar = createScalarNode(nil, "")
+	}
+	log.Debug("scalar headC: %v, footC: %v", n.HeadComment, n.FootComment)
 	scalar.HeadComment = dec.processComment(n.HeadComment)
 	scalar.LineComment = dec.processComment(n.LineComment)
-	if scalar.Tag == "!!seq" {
-		scalar.Content[0].HeadComment = scalar.LineComment
-		scalar.LineComment = ""
-	}
-
 	scalar.FootComment = dec.processComment(n.FootComment)

 	return scalar, nil
 }

-func (dec *xmlDecoder) Decode() (*CandidateNode, error) {
+func (dec *xmlDecoder) Decode(rootYamlNode *yaml.Node) error {
 	if dec.finished {
-		return nil, io.EOF
+		return io.EOF
 	}
 	root := &xmlNode{}
 	// cant use xj - it doesn't keep map order.
 	err := dec.decodeXML(root)

 	if err != nil {
-		return nil, err
+		return err
 	}
 	firstNode, err := dec.convertToYamlNode(root)

 	if err != nil {
-		return nil, err
+		return err
 	} else if firstNode.Tag == "!!null" {
 		dec.finished = true
 		if dec.readAnything {
-			return nil, io.EOF
+			return io.EOF
 		}
 	}
 	dec.readAnything = true
+	rootYamlNode.Kind = yaml.DocumentNode
+	rootYamlNode.Content = []*yaml.Node{firstNode}
 	dec.finished = true
-
-	return &CandidateNode{
-		Node: &yaml.Node{
-			Kind:    yaml.DocumentNode,
-			Content: []*yaml.Node{firstNode},
-		},
-	}, nil
+	return nil
 }

 type xmlNode struct {
@@ -202,7 +164,7 @@ type xmlNode struct {
 	HeadComment string
 	FootComment string
 	LineComment string
-	Data        []string
+	Data        string
 }

 type xmlChildrenKv struct {
@@ -243,12 +205,10 @@ type element struct {
 // of the map keys.
 func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
 	xmlDec := xml.NewDecoder(dec.reader)
-	xmlDec.Strict = dec.prefs.StrictMode
+	xmlDec.Strict = dec.strictMode
 	// That will convert the charset if the provided XML is non-UTF-8
 	xmlDec.CharsetReader = charset.NewReaderLabel

-	started := false
-
 	// Create first element from the root node
 	elem := &element{
 		parent: nil,
@@ -256,7 +216,7 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
 	}

 	getToken := func() (xml.Token, error) {
-		if dec.prefs.UseRawToken {
+		if dec.useRawToken {
 			return xmlDec.RawToken()
 		}
 		return xmlDec.Token()
@@ -284,23 +244,17 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {

 			// Extract attributes as children
 			for _, a := range se.Attr {
-				if dec.prefs.KeepNamespace {
+				if dec.keepNamespace {
 					if a.Name.Space != "" {
 						a.Name.Local = a.Name.Space + ":" + a.Name.Local
 					}
 				}
-				elem.n.AddChild(dec.prefs.AttributePrefix+a.Name.Local, &xmlNode{Data: []string{a.Value}})
+				elem.n.AddChild(dec.attributePrefix+a.Name.Local, &xmlNode{Data: a.Value})
 			}
 		case xml.CharData:
-
 			// Extract XML data (if any)
-			newBit := trimNonGraphic(string(se))
-			if !started && len(newBit) > 0 {
-				return fmt.Errorf("invalid XML: Encountered chardata [%v] outside of XML node", newBit)
-			}
-
-			if len(newBit) > 0 {
-				elem.n.Data = append(elem.n.Data, newBit)
+			elem.n.Data = trimNonGraphic(string(se))
+			if elem.n.Data != "" {
 				elem.state = "chardata"
 				log.Debug("chardata [%v] for %v", elem.n.Data, elem.label)
 			}
@@ -322,22 +276,13 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {

 			} else if elem.state == "chardata" {
 				log.Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
-				elem.n.LineComment = joinComments([]string{elem.n.LineComment, commentStr}, " ")
+				elem.n.LineComment = joinFilter([]string{elem.n.LineComment, commentStr})
 			} else {
 				log.Debug("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
-				elem.n.HeadComment = joinComments([]string{elem.n.HeadComment, commentStr}, " ")
+				elem.n.HeadComment = joinFilter([]string{elem.n.HeadComment, commentStr})
 			}

-		case xml.ProcInst:
-			if !dec.prefs.SkipProcInst {
-				elem.n.AddChild(dec.prefs.ProcInstPrefix+se.Target, &xmlNode{Data: []string{string(se.Inst)}})
-			}
-		case xml.Directive:
-			if !dec.prefs.SkipDirectives {
-				elem.n.AddChild(dec.prefs.DirectiveName, &xmlNode{Data: []string{string(se)}})
-			}
 		}
-		started = true
 	}

 	return nil
@@ -349,28 +294,22 @@ func applyFootComment(elem *element, commentStr string) {
 	if len(elem.n.Children) > 0 {
 		lastChildIndex := len(elem.n.Children) - 1
 		childKv := elem.n.Children[lastChildIndex]
-		log.Debug("got a foot comment, putting on last child for %v: [%v]", childKv.K, commentStr)
-		// if it's an array of scalars, put the foot comment on the scalar itself
-		if len(childKv.V) > 0 && len(childKv.V[0].Children) == 0 {
-			nodeToUpdate := childKv.V[len(childKv.V)-1]
-			nodeToUpdate.FootComment = joinComments([]string{nodeToUpdate.FootComment, commentStr}, " ")
-		} else {
-			childKv.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ")
-		}
+		log.Debug("got a foot comment for %v: [%v]", childKv.K, commentStr)
+		childKv.FootComment = joinFilter([]string{elem.n.FootComment, commentStr})
 	} else {
 		log.Debug("got a foot comment for %v: [%v]", elem.label, commentStr)
-		elem.n.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ")
+		elem.n.FootComment = joinFilter([]string{elem.n.FootComment, commentStr})
 	}
 }

-func joinComments(rawStrings []string, joinStr string) string {
+func joinFilter(rawStrings []string) string {
 	stringsToJoin := make([]string, 0)
 	for _, str := range rawStrings {
 		if str != "" {
 			stringsToJoin = append(stringsToJoin, str)
 		}
 	}
-	return strings.Join(stringsToJoin, joinStr)
+	return strings.Join(stringsToJoin, " ")
 }

 // trimNonGraphic returns a slice of the string s, with all leading and trailing
@@ -1,143 +1,23 @@
 package yqlib

 import (
-	"bufio"
-	"bytes"
-	"errors"
 	"io"
-	"regexp"
-	"strings"

 	yaml "gopkg.in/yaml.v3"
 )

 type yamlDecoder struct {
 	decoder yaml.Decoder
-
-	prefs YamlPreferences
-
-	// work around of various parsing issues by yaml.v3 with document headers
-	leadingContent string
-	bufferRead     bytes.Buffer
-
-	readAnything bool
-	firstFile    bool
 }

-func NewYamlDecoder(prefs YamlPreferences) Decoder {
-	return &yamlDecoder{prefs: prefs, firstFile: true}
+func NewYamlDecoder() Decoder {
+	return &yamlDecoder{}
 }

-func (dec *yamlDecoder) processReadStream(reader *bufio.Reader) (io.Reader, string, error) {
-	var commentLineRegEx = regexp.MustCompile(`^\s*#`)
-	var yamlDirectiveLineRegEx = regexp.MustCompile(`^\s*%YA`)
-	var sb strings.Builder
-	for {
-		peekBytes, err := reader.Peek(3)
-		if errors.Is(err, io.EOF) {
-			// EOF are handled else where..
-			return reader, sb.String(), nil
-		} else if err != nil {
-			return reader, sb.String(), err
-		} else if string(peekBytes[0]) == "\n" {
-			_, err := reader.ReadString('\n')
-			sb.WriteString("\n")
-			if errors.Is(err, io.EOF) {
-				return reader, sb.String(), nil
-			} else if err != nil {
-				return reader, sb.String(), err
-			}
-		} else if string(peekBytes) == "---" {
-			_, err := reader.ReadString('\n')
-			sb.WriteString("$yqDocSeperator$\n")
-			if errors.Is(err, io.EOF) {
-				return reader, sb.String(), nil
-			} else if err != nil {
-				return reader, sb.String(), err
-			}
-		} else if commentLineRegEx.MatchString(string(peekBytes)) || yamlDirectiveLineRegEx.MatchString(string(peekBytes)) {
-			line, err := reader.ReadString('\n')
-			sb.WriteString(line)
-			if errors.Is(err, io.EOF) {
-				return reader, sb.String(), nil
-			} else if err != nil {
-				return reader, sb.String(), err
-			}
-		} else {
-			return reader, sb.String(), nil
-		}
-	}
+func (dec *yamlDecoder) Init(reader io.Reader) {
+	dec.decoder = *yaml.NewDecoder(reader)
 }

-func (dec *yamlDecoder) Init(reader io.Reader) error {
-	readerToUse := reader
-	leadingContent := ""
-	dec.bufferRead = bytes.Buffer{}
-	var err error
-	// if we 'evaluating together' - we only process the leading content
-	// of the first file - this ensures comments from subsequent files are
-	// merged together correctly.
-	if dec.prefs.LeadingContentPreProcessing && (!dec.prefs.EvaluateTogether || dec.firstFile) {
-		readerToUse, leadingContent, err = dec.processReadStream(bufio.NewReader(reader))
-		if err != nil {
-			return err
-		}
-	} else if !dec.prefs.LeadingContentPreProcessing {
-		// if we're not process the leading content
-		// keep a copy of what we've read. This is incase its a
-		// doc with only comments - the decoder will return nothing
-		// then we can read the comments from bufferRead
-		readerToUse = io.TeeReader(reader, &dec.bufferRead)
-	}
-	dec.leadingContent = leadingContent
-	dec.readAnything = false
-	dec.decoder = *yaml.NewDecoder(readerToUse)
-	dec.firstFile = false
-	return nil
-}
-
-func (dec *yamlDecoder) Decode() (*CandidateNode, error) {
-	var dataBucket yaml.Node
-	err := dec.decoder.Decode(&dataBucket)
-	if errors.Is(err, io.EOF) && dec.leadingContent != "" && !dec.readAnything {
-		// force returning an empty node with a comment.
-		dec.readAnything = true
-		return dec.blankNodeWithComment(), nil
-	} else if errors.Is(err, io.EOF) && !dec.prefs.LeadingContentPreProcessing && !dec.readAnything {
-		// didn't find any yaml,
-		// check the tee buffer, maybe there were comments
-		dec.readAnything = true
-		dec.leadingContent = dec.bufferRead.String()
-		if dec.leadingContent != "" {
-			return dec.blankNodeWithComment(), nil
-		}
-		return nil, err
-	} else if err != nil {
-		return nil, err
-	}
-
-	candidateNode := &CandidateNode{
-		Node: &dataBucket,
-	}
-
-	if dec.leadingContent != "" {
-		candidateNode.LeadingContent = dec.leadingContent
-		dec.leadingContent = ""
-	}
-	dec.readAnything = true
-	// move document comments into candidate node
-	// otherwise unwrap drops them.
-	candidateNode.TrailingContent = dataBucket.FootComment
-	dataBucket.FootComment = ""
-	return candidateNode, nil
-}
-
-func (dec *yamlDecoder) blankNodeWithComment() *CandidateNode {
-	return &CandidateNode{
-		Document:       0,
-		Filename:       "",
-		Node:           &yaml.Node{Kind: yaml.DocumentNode, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
-		FileIndex:      0,
-		LeadingContent: dec.leadingContent,
-	}
+func (dec *yamlDecoder) Decode(rootYamlNode *yaml.Node) error {
+	return dec.decoder.Decode(rootYamlNode)
 }
@@ -0,0 +1,6 @@
+
+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
@@ -9,6 +9,12 @@ Add behaves differently according to the type of the LHS:
 Use `+=` as a relative append assign for things like increment. Note that `.a += .x` is equivalent to running `.a = .a + .x`.


+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
 ## Concatenate arrays
 Given a sample.yml file of:
 ```yaml
@@ -86,23 +92,6 @@ will output
 a: ['dog', 'cat']
 ```

-## Prepend to existing array
-Given a sample.yml file of:
-```yaml
-a:
-  - dog
-```
-then
-```bash
-yq '.a = ["cat"] + .a' sample.yml
-```
-will output
-```yaml
-a:
-  - cat
-  - dog
-```
-
 ## Add new object to array
 Given a sample.yml file of:
 ```yaml
@@ -2,6 +2,12 @@

 This operator is used to provide alternative (or default) values when a particular expression is either null or false.

+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
 ## LHS is defined
 Given a sample.yml file of:
 ```yaml
@@ -73,36 +79,3 @@ will output
 cat
 ```

-## Update or create - entity exists
-This initialises `a` if it's not present
-
-Given a sample.yml file of:
-```yaml
-a: 1
-```
-then
-```bash
-yq '(.a // (.a = 0)) += 1' sample.yml
-```
-will output
-```yaml
-a: 2
-```
-
-## Update or create - entity does not exist
-This initialises `a` if it's not present
-
-Given a sample.yml file of:
-```yaml
-b: camel
-```
-then
-```bash
-yq '(.a // (.a = 0)) += 1' sample.yml
-```
-will output
-```yaml
-b: camel
-a: 1
-```
-
@@ -5,6 +5,12 @@ Use the `alias` and `anchor` operators to read and write yaml aliases and anchor
 `yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.


+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
 ## Merge one map
 see https://yaml.org/type/merge.html

@@ -302,7 +308,7 @@ foobar:
 ```

 ## Dereference and update a field
-Use explode with multiply to dereference an object
+`Use explode with multiply to dereference an object

 Given a sample.yml file of:
 ```yaml
@@ -1,28 +0,0 @@
-# Array to Map
-
-Use this operator to convert an array to..a map. The indices are used as map keys, null values in the array are skipped over.
-
-Behind the scenes, this is implemented using reduce:
-
-```
-(.[] | select(. != null) ) as $i ireduce({}; .[$i | key] = $i)
-```
-
-## Simple example
-Given a sample.yml file of:
-```yaml
-cool:
-  - null
-  - null
-  - hello
-```
-then
-```bash
-yq '.cool |= array_to_map' sample.yml
-```
-will output
-```yaml
-cool:
-  2: hello
-```
-
@@ -3,14 +3,15 @@
 This operator is used to update node values. It can be used in either the:

 ### plain form: `=`
-Which will set the LHS node values equal to the RHS node values. The RHS expression is run against the matching nodes in the pipeline.
+Which will assign the LHS node values to the RHS node values. The RHS expression is run against the matching nodes in the pipeline.

 ### relative form: `|=`
-This will do a similar thing to the plain form, but the RHS expression is run with _each LHS node as context_. This is useful for updating values based on old values, e.g. increment.
+This will do a similar thing to the plain form, however, the RHS expression is run against _the LHS nodes_. This is useful for updating values based on old values, e.g. increment.
+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;

-
-### Flags
- `c` clobber custom tags
+`yq e <exp> <file>`
+{% endhint %}

 ## Create yaml file
 Running
@@ -203,7 +204,7 @@ a:
 ```

 ## Update node value that has an anchor
-Anchor will remain
+Anchor will remaple

 Given a sample.yml file of:
 ```yaml
@@ -234,37 +235,3 @@ a:
    - bogs
 ```

-## Custom types are maintained by default
-Given a sample.yml file of:
-```yaml
-a: !cat meow
-b: !dog woof
-```
-then
-```bash
-yq '.a = .b' sample.yml
-```
-will output
-```yaml
-a: !cat woof
-b: !dog woof
-```
-
-## Custom types: clobber
-Use the `c` option to clobber custom tags
-
-Given a sample.yml file of:
-```yaml
-a: !cat meow
-b: !dog woof
-```
-then
-```bash
-yq '.a =c .b' sample.yml
-```
-will output
-```yaml
-a: !dog woof
-b: !dog woof
-```
-
@@ -16,6 +16,12 @@ These are most commonly used with the `select` operator to filter particular nod
 - comparison (`>=`, `<` etc) operators [here](https://mikefarah.gitbook.io/yq/operators/compare)
 - select operator [here](https://mikefarah.gitbook.io/yq/operators/select)

+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
 ## `or` example
 Running
 ```bash
@@ -26,24 +32,6 @@ will output
 true
 ```

-## "yes" and "no" are strings
-In the yaml 1.2 standard, support for yes/no as booleans was dropped - they are now considered strings. See '10.2.1.2. Boolean' in https://yaml.org/spec/1.2.2/
-
-Given a sample.yml file of:
-```yaml
- yes
- no
-```
-then
-```bash
-yq '.[] | tag' sample.yml
-```
-will output
-```yaml
-!!str
-!!str
-```
-
 ## `and` example
 Running
 ```bash
@@ -3,6 +3,12 @@
 This creates an array using the expression between the square brackets.


+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
 ## Collect empty
 Running
 ```bash
@@ -2,6 +2,12 @@

 Returns the column of the matching node. Starts from 1, 0 indicates there was no column data.

+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
 ## Returns column of _value_ node
 Given a sample.yml file of:
 ```yaml
@@ -1,18 +1,22 @@
 # Comment Operators

-Use these comment operators to set or retrieve comments. Note that line comments on maps/arrays are actually set on the _key_ node as opposed to the _value_ (map/array). See below for examples.
+Use these comment operators to set or retrieve comments.

 Like the `=` and `|=` assign operators, the same syntax applies when updating comments:

 ### plain form: `=`
-This will set the LHS nodes' comments equal to the expression on the RHS. The RHS is run against the matching nodes in the pipeline
+This will assign the LHS nodes comments to the expression on the RHS. The RHS is run against the matching nodes in the pipeline

 ### relative form: `|=` 
-This is similar to the plain form, but it evaluates the RHS with _each matching LHS node as context_. This is useful if you want to set the comments as a relative expression of the node, for instance its value or path.
+Similar to the plain form, however the RHS evaluates against each matching LHS node! This is useful if you want to set the comments as a relative expression of the node, for instance its value or path.
+
+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}

 ## Set line comment
-Set the comment on the key node for more reliability (see below).
-
 Given a sample.yml file of:
 ```yaml
 a: cat
@@ -26,24 +30,6 @@ will output
 a: cat # single
 ```

-## Set line comment of a maps/arrays
-For maps and arrays, you need to set the line comment on the _key_ node. This will also work for scalars.
-
-Given a sample.yml file of:
-```yaml
-a:
-  b: things
-```
-then
-```bash
-yq '(.a | key) line_comment="single"' sample.yml
-```
-will output
-```yaml
-a: # single
-  b: things
-```
-
 ## Use update assign to perform relative updates
 Given a sample.yml file of:
 ```yaml
@@ -60,121 +46,6 @@ a: cat # cat
 b: dog # dog
 ```

-## Where is the comment - map key example
-The underlying yaml parser can assign comments in a document to surprising nodes. Use an expression like this to find where you comment is. 'p' indicates the path, 'isKey' is if the node is a map key (as opposed to a map value).
-From this, you can see the 'hello-world-comment' is actually on the 'hello' key
-
-Given a sample.yml file of:
-```yaml
-hello: # hello-world-comment
-  message: world
-```
-then
-```bash
-yq '[... | {"p": path | join("."), "isKey": is_key, "hc": headComment, "lc": lineComment, "fc": footComment}]' sample.yml
-```
-will output
-```yaml
- p: ""
-  isKey: false
-  hc: ""
-  lc: ""
-  fc: ""
- p: hello
-  isKey: true
-  hc: ""
-  lc: hello-world-comment
-  fc: ""
- p: hello
-  isKey: false
-  hc: ""
-  lc: ""
-  fc: ""
- p: hello.message
-  isKey: true
-  hc: ""
-  lc: ""
-  fc: ""
- p: hello.message
-  isKey: false
-  hc: ""
-  lc: ""
-  fc: ""
-```
-
-## Retrieve comment - map key example
-From the previous example, we know that the comment is on the 'hello' _key_ as a lineComment
-
-Given a sample.yml file of:
-```yaml
-hello: # hello-world-comment
-  message: world
-```
-then
-```bash
-yq '.hello | key | line_comment' sample.yml
-```
-will output
-```yaml
-hello-world-comment
-```
-
-## Where is the comment - array example
-The underlying yaml parser can assign comments in a document to surprising nodes. Use an expression like this to find where you comment is. 'p' indicates the path, 'isKey' is if the node is a map key (as opposed to a map value).
-From this, you can see the 'under-name-comment' is actually on the first child
-
-Given a sample.yml file of:
-```yaml
-name:
-  # under-name-comment
-  - first-array-child
-```
-then
-```bash
-yq '[... | {"p": path | join("."), "isKey": is_key, "hc": headComment, "lc": lineComment, "fc": footComment}]' sample.yml
-```
-will output
-```yaml
- p: ""
-  isKey: false
-  hc: ""
-  lc: ""
-  fc: ""
- p: name
-  isKey: true
-  hc: ""
-  lc: ""
-  fc: ""
- p: name
-  isKey: false
-  hc: ""
-  lc: ""
-  fc: ""
- p: name.0
-  isKey: false
-  hc: under-name-comment
-  lc: ""
-  fc: ""
-```
-
-## Retrieve comment - array example
-From the previous example, we know that the comment is on the first child as a headComment
-
-Given a sample.yml file of:
-```yaml
-name:
-  # under-name-comment
-  - first-array-child
-```
-then
-```bash
-yq '.name[0] | headComment' sample.yml
-```
-will output
-```yaml
-under-name-comment
-```
-
 ## Set head comment
 Given a sample.yml file of:
 ```yaml
@@ -191,25 +62,6 @@ will output
 a: cat
 ```

-## Set head comment of a map entry
-Given a sample.yml file of:
-```yaml
-f: foo
-a:
-  b: cat
-```
-then
-```bash
-yq '(.a | key) head_comment="single"' sample.yml
-```
-will output
-```yaml
-f: foo
-# single
-a:
-  b: cat
-```
-
 ## Set foot comment, using an expression
 Given a sample.yml file of:
 ```yaml
@@ -246,8 +98,6 @@ Note the use of `...` to ensure key nodes are included.

 Given a sample.yml file of:
 ```yaml
-# hi
-
 a: cat # comment
 # great
 b: # key comment
@@ -265,8 +115,6 @@ b:
 ## Get line comment
 Given a sample.yml file of:
 ```yaml
-# welcome!
-
 a: cat # meow
 # have a great day
 ```
@@ -295,7 +143,6 @@ yq '. | head_comment' sample.yml
 will output
 ```yaml
 welcome!
-
 ```

 ## Head comment with document split
@@ -14,6 +14,12 @@ The following types are currently supported:
 - boolean operators (`and`, `or`, `any` etc) [here](https://mikefarah.gitbook.io/yq/operators/boolean-operators)
 - select operator [here](https://mikefarah.gitbook.io/yq/operators/select)

+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
 ## Compare numbers (>)
 Given a sample.yml file of:
 ```yaml
@@ -1,13 +1,11 @@
 # Contains

-This returns `true` if the context contains the passed in parameter, and false otherwise. For arrays, this will return true if the passed in array is contained within the array. For strings, it will return true if the string is a substring.
+This returns `true` if the context contains the passed in parameter, and false otherwise.

 {% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;

-_Note_ that, just like jq, when checking if an array of strings `contains` another, this will use `contains` and _not_ equals to check each string. This means an expression like `contains(["cat"])` will return true for an array `["cats"]`.
-
-See the "Array has a subset array" example below on how to check for a subset.
-
+`yq e <exp> <file>`
 {% endhint %}

 ## Array contains array
@@ -28,24 +26,6 @@ will output
 true
 ```

-## Array has a subset array
-Subtract the superset array from the subset, if there's anything left, it's not a subset
-
-Given a sample.yml file of:
-```yaml
- foobar
- foobaz
- blarp
-```
-then
-```bash
-yq '["baz", "bar"] - . | length == 0' sample.yml
-```
-will output
-```yaml
-false
-```
-
 ## Object included in array
 Given a sample.yml file of:
 ```yaml
@@ -2,6 +2,12 @@

 This is used to construct objects (or maps). This can be used against existing yaml, or to create fresh yaml documents.

+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
 ## Collect empty object
 Running
 ```bash
@@ -3,7 +3,7 @@
 Various operators for parsing and manipulating dates. 

 ## Date time formattings
-This uses Golang's built in time library for parsing and formatting date times.
+This uses the golangs built in time library for parsing and formatting date times.

 When not specified, the RFC3339 standard is assumed `2006-01-02T15:04:05Z07:00` for parsing.

@@ -13,17 +13,23 @@ To specify a custom parsing format, use the `with_dtf` operator. The first param
 yq 'with_dtf("myformat"; .a + "3h" | tz("Australia/Melbourne"))'
 ```

-See the [library docs](https://pkg.go.dev/time#pkg-constants) for examples of formatting options.
+See https://pkg.go.dev/time#pkg-constants for examples of formatting options.


 ## Timezones
-This uses Golang's built in LoadLocation function to parse timezones strings. See the [library docs](https://pkg.go.dev/time#LoadLocation) for more details.
+This uses golangs built in LoadLocation function to parse timezones strings. See https://pkg.go.dev/time#LoadLocation for more details.


 ## Durations
-Durations are parsed using Golang's built in [ParseDuration](https://pkg.go.dev/time#ParseDuration) function.
+Durations are parsed using golangs built in [ParseDuration](https://pkg.go.dev/time#ParseDuration) function.

-You can add durations to time using the `+` operator.
+You can durations to time using the `+` operator.
+
+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}

 ## Format: from standard RFC3339 format
 Providing a single parameter assumes a standard RFC3339 datetime format. If the target format is not a valid yaml datetime format, the result will be a string tagged node.
@@ -60,7 +66,7 @@ a: 2001-12-15
 ## Format: get the day of the week
 Given a sample.yml file of:
 ```yaml
-a: 2001-12-15
+a: 2001-12-15T02:59:43.1Z
 ```
 then
 ```bash
@@ -86,30 +92,6 @@ a: cool
 updated: 2021-05-19T01:02:03Z
 ```

-## From Unix
-Converts from unix time. Note, you don't have to pipe through the tz operator :)
-
-Running
-```bash
-yq --null-input '1675301929 | from_unix | tz("UTC")'
-```
-will output
-```yaml
-2023-02-02T01:38:49Z
-```
-
-## To Unix
-Converts to unix time
-
-Running
-```bash
-yq --null-input 'now | to_unix'
-```
-will output
-```yaml
-1621386123
-```
-
 ## Timezone: from standard RFC3339 format
 Returns a new datetime in the specified timezone. Specify standard IANA Time Zone format or 'utc', 'local'. When given a single parameter, this assumes the datetime is in RFC3339 format.

@@ -174,7 +156,7 @@ a: 2021-01-01T03:10:00Z
 ```

 ## Date subtraction
-You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/datetime#date-time-formattings) for more information.
+You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.

 Given a sample.yml file of:
 ```yaml
@@ -2,6 +2,12 @@

 Deletes matching entries in maps or arrays.

+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
 ## Delete entry in map
 Given a sample.yml file of:
 ```yaml
@@ -1,61 +0,0 @@
-# Divide
-
-Divide behaves differently according to the type of the LHS:
-* strings: split by the divider
-* number: arithmetic division
-
-## String split
-Given a sample.yml file of:
-```yaml
-a: cat_meow
-b: _
-```
-then
-```bash
-yq '.c = .a / .b' sample.yml
-```
-will output
-```yaml
-a: cat_meow
-b: _
-c:
-  - cat
-  - meow
-```
-
-## Number division
-The result during division is calculated as a float
-
-Given a sample.yml file of:
-```yaml
-a: 12
-b: 2.5
-```
-then
-```bash
-yq '.a = .a / .b' sample.yml
-```
-will output
-```yaml
-a: 4.8
-b: 2.5
-```
-
-## Number division by zero
-Dividing by zero results in +Inf or -Inf
-
-Given a sample.yml file of:
-```yaml
-a: 1
-b: -1
-```
-then
-```bash
-yq '.a = .a / 0 | .b = .b / 0' sample.yml
-```
-will output
-```yaml
-a: !!float +Inf
-b: !!float -Inf
-```
-
@@ -2,6 +2,12 @@

 Use the `documentIndex` operator (or the `di` shorthand) to select nodes of a particular document.

+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
 ## Retrieve a document index
 Given a sample.yml file of:
 ```yaml
@@ -9,15 +9,13 @@ These operators are useful to process yaml documents that have stringified embed

 | Format | Decode (from string) | Encode (to string) |
 | --- | -- | --|
-| Yaml | from_yaml/@yamld | to_yaml(i)/@yaml |
-| JSON | from_json/@jsond | to_json(i)/@json |
+| Yaml | from_yaml | to_yaml(i)/@yaml |
+| JSON | from_json | to_json(i)/@json |
 | Properties | from_props/@propsd  | to_props/@props |
 | CSV | from_csv/@csvd | to_csv/@csv |
 | TSV | from_tsv/@tsvd | to_tsv/@tsv |
-| XML | from_xml/@xmld | to_xml(i)/@xml |
+| XML | from_xml | to_xml(i)/@xml |
 | Base64 | @base64d | @base64 |
-| URI | @urid | @uri |
-| Shell |  | @sh |


 See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) for accepted formats.
@@ -25,7 +23,13 @@ See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) f
 XML uses the `--xml-attribute-prefix` and `xml-content-name` flags to identify attributes and content fields.


-Base64 assumes [rfc4648](https://rfc-editor.org/rfc/rfc4648.html) encoding. Encoding and decoding both assume that the content is a utf-8 string and not binary content.
+Base64 assumes [rfc4648](https://rfc-editor.org/rfc/rfc4648.html) encoding. Encoding and decoding both assume that the content is a string.
+
+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}

 ## Encode value as json string
 Given a sample.yml file of:
@@ -309,7 +313,7 @@ cat,"thing1,thing2",true,3.40
 dog,thing3,false,12
 ```

-## Encode array of arrays as tsv string
+## Encode array of array scalars as tsv string
 Scalars are strings, numbers and booleans.

 Given a sample.yml file of:
@@ -339,7 +343,7 @@ Given a sample.yml file of:
 a:
  cool:
    foo: bar
-    +@id: hi
+    +id: hi
 ```
 then
 ```bash
@@ -359,7 +363,7 @@ Given a sample.yml file of:
 a:
  cool:
    foo: bar
-    +@id: hi
+    +id: hi
 ```
 then
 ```bash
@@ -377,7 +381,7 @@ Given a sample.yml file of:
 a:
  cool:
    foo: bar
-    +@id: hi
+    +id: hi
 ```
 then
 ```bash
@@ -437,50 +441,6 @@ will output
 YTogYXBwbGUK
 ```

-## Encode a string to uri
-Given a sample.yml file of:
-```yaml
-coolData: this has & special () characters *
-```
-then
-```bash
-yq '.coolData | @uri' sample.yml
-```
-will output
-```yaml
-this+has+%26+special+%28%29+characters+%2A
-```
-
-## Decode a URI to a string
-Given a sample.yml file of:
-```yaml
-this+has+%26+special+%28%29+characters+%2A
-```
-then
-```bash
-yq '@urid' sample.yml
-```
-will output
-```yaml
-this has & special () characters *
-```
-
-## Encode a string to sh
-Sh/Bash friendly string
-
-Given a sample.yml file of:
-```yaml
-coolData: strings with spaces and a 'quote'
-```
-then
-```bash
-yq '.coolData | @sh' sample.yml
-```
-will output
-```yaml
-strings' with spaces and a '\'quote\'
-```
-
 ## Decode a base64 encoded string
 Decoded data is assumed to be a string.

@@ -2,6 +2,12 @@

 Similar to the same named functions in `jq` these functions convert to/from an object and an array of key-value pairs. This is most useful for performing operations on keys of maps.

+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
 ## to_entries Map
 Given a sample.yml file of:
 ```yaml
@@ -67,7 +73,7 @@ a: 1
 b: 2
 ```

-## from_entries with numeric key indices
+## from_entries with numeric key indexes
 from_entries always creates a map, even for numeric keys

 Given a sample.yml file of:
@@ -101,26 +107,6 @@ KEY_a: 1
 KEY_b: 2
 ```

-## Custom sort map keys
-Use to_entries to convert to an array of key/value pairs, sort the array using sort/sort_by/etc, and convert it back.
-
-Given a sample.yml file of:
-```yaml
-a: 1
-c: 3
-b: 2
-```
-then
-```bash
-yq 'to_entries | sort_by(.key) | reverse | from_entries' sample.yml
-```
-will output
-```yaml
-c: 3
-b: 2
-a: 1
-```
-
 ## Use with_entries to filter the map
 Given a sample.yml file of:
 ```yaml
@@ -30,6 +30,12 @@ yq '(.. | select(tag == "!!str")) |= envsubst' file.yaml
 ```


+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
 ## Read string environment variable
 Running
 ```bash
@@ -21,6 +21,12 @@ The not equals `!=` operator returns `false` if the LHS is equal to the RHS.
 - select operator [here](https://mikefarah.gitbook.io/yq/operators/select)


+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
 ## Match string
 Given a sample.yml file of:
 ```yaml
@@ -2,6 +2,12 @@

 Use this operation to short-circuit expressions. Useful for validation.

+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
 ## Validate a particular value
 Given a sample.yml file of:
 ```yaml
@@ -2,9 +2,15 @@

 Use `eval` to dynamically process an expression - for instance from an environment variable.

-`eval` takes a single argument, and evaluates that as a `yq` expression. Any valid expression can be used, be it a path `.a.b.c | select(. == "cat")`, or an update `.a.b.c = "gogo"`.
+`eval` takes a single argument, and evaluates that as a `yq` expression. Any valid expression can be used, beit a path `.a.b.c | select(. == "cat")`, or an update `.a.b.c = "gogo"`.

-Tip: This can be a useful way to parameterise complex scripts.
+Tip: This can be useful way parameterise complex scripts.
+
+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}

 ## Dynamically evaluate a path
 Given a sample.yml file of:
@@ -10,6 +10,12 @@ Note the use of eval-all to ensure all documents are loaded into memory.
 yq eval-all 'select(fi == 0) * select(filename == "file2.yaml")' file1.yaml file2.yaml
 ```

+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
 ## Get filename
 Given a sample.yml file of:
 ```yaml
@@ -1,42 +0,0 @@
-# Filter
-
-Filters an array (or map values) by the expression given. Equivalent to doing `map(select(exp))`.
-
-
-## Filter array
-Given a sample.yml file of:
-```yaml
- 1
- 2
- 3
-```
-then
-```bash
-yq 'filter(. < 3)' sample.yml
-```
-will output
-```yaml
- 1
- 2
-```
-
-## Filter map values
-Given a sample.yml file of:
-```yaml
-c:
-  things: cool
-  frog: yes
-d:
-  things: hot
-  frog: false
-```
-then
-```bash
-yq 'filter(.things == "cool")' sample.yml
-```
-will output
-```yaml
- things: cool
-  frog: yes
-```
-
@@ -1,6 +1,12 @@
 # Flatten
 This recursively flattens arrays.

+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
 ## Flatten
 Recursively flattens all arrays

@@ -2,6 +2,12 @@

 This is used to group items in an array by an expression.

+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}
+
 ## Group by field
 Given a sample.yml file of:
 ```yaml
@@ -1,6 +1,12 @@
 # Has

-This operation returns true if the key exists in a map (or index in an array), false otherwise.
+This is operation that returns true if the key exists in a map (or index in an array), false otherwise.
+
+{% hint style="warning" %}
+Note that versions prior to 4.18 require the 'eval/e' command to be specified.&#x20;
+
+`yq e <exp> <file>`
+{% endhint %}

 ## Has map key
 Given a sample.yml file of:
@@ -1,9 +0,0 @@
-# Array to Map
-
-Use this operator to convert an array to..a map. The indices are used as map keys, null values in the array are skipped over.
-
-Behind the scenes, this is implemented using reduce:
-
-```
-(.[] | select(. != null) ) as $i ireduce({}; .[$i | key] = $i)
-```
@@ -3,11 +3,7 @@
 This operator is used to update node values. It can be used in either the:

 ### plain form: `=`
-Which will set the LHS node values equal to the RHS node values. The RHS expression is run against the matching nodes in the pipeline.
+Which will assign the LHS node values to the RHS node values. The RHS expression is run against the matching nodes in the pipeline.

 ### relative form: `|=`
-This will do a similar thing to the plain form, but the RHS expression is run with _each LHS node as context_. This is useful for updating values based on old values, e.g. increment.
-
-
-### Flags
- `c` clobber custom tags
+This will do a similar thing to the plain form, however, the RHS expression is run against _the LHS nodes_. This is useful for updating values based on old values, e.g. increment.
@@ -1,11 +1,11 @@
 # Comment Operators

-Use these comment operators to set or retrieve comments. Note that line comments on maps/arrays are actually set on the _key_ node as opposed to the _value_ (map/array). See below for examples.
+Use these comment operators to set or retrieve comments.

 Like the `=` and `|=` assign operators, the same syntax applies when updating comments:

 ### plain form: `=`
-This will set the LHS nodes' comments equal to the expression on the RHS. The RHS is run against the matching nodes in the pipeline
+This will assign the LHS nodes comments to the expression on the RHS. The RHS is run against the matching nodes in the pipeline

 ### relative form: `|=` 
-This is similar to the plain form, but it evaluates the RHS with _each matching LHS node as context_. This is useful if you want to set the comments as a relative expression of the node, for instance its value or path.
+Similar to the plain form, however the RHS evaluates against each matching LHS node! This is useful if you want to set the comments as a relative expression of the node, for instance its value or path.
@@ -1,11 +1,3 @@
 # Contains

-This returns `true` if the context contains the passed in parameter, and false otherwise. For arrays, this will return true if the passed in array is contained within the array. For strings, it will return true if the string is a substring.
-
-{% hint style="warning" %}
-
-_Note_ that, just like jq, when checking if an array of strings `contains` another, this will use `contains` and _not_ equals to check each string. This means an expression like `contains(["cat"])` will return true for an array `["cats"]`.
-
-See the "Array has a subset array" example below on how to check for a subset.
-
-{% endhint %}
+This returns `true` if the context contains the passed in parameter, and false otherwise.
@@ -3,7 +3,7 @@
 Various operators for parsing and manipulating dates. 

 ## Date time formattings
-This uses Golang's built in time library for parsing and formatting date times.
+This uses the golangs built in time library for parsing and formatting date times.

 When not specified, the RFC3339 standard is assumed `2006-01-02T15:04:05Z07:00` for parsing.

@@ -13,14 +13,14 @@ To specify a custom parsing format, use the `with_dtf` operator. The first param
 yq 'with_dtf("myformat"; .a + "3h" | tz("Australia/Melbourne"))'
 ```

-See the [library docs](https://pkg.go.dev/time#pkg-constants) for examples of formatting options.
+See https://pkg.go.dev/time#pkg-constants for examples of formatting options.


 ## Timezones
-This uses Golang's built in LoadLocation function to parse timezones strings. See the [library docs](https://pkg.go.dev/time#LoadLocation) for more details.
+This uses golangs built in LoadLocation function to parse timezones strings. See https://pkg.go.dev/time#LoadLocation for more details.


 ## Durations
-Durations are parsed using Golang's built in [ParseDuration](https://pkg.go.dev/time#ParseDuration) function.
+Durations are parsed using golangs built in [ParseDuration](https://pkg.go.dev/time#ParseDuration) function.

-You can add durations to time using the `+` operator.
+You can durations to time using the `+` operator.
@@ -1,5 +0,0 @@
-# Divide
-
-Divide behaves differently according to the type of the LHS:
-* strings: split by the divider
-* number: arithmetic division
@@ -9,15 +9,13 @@ These operators are useful to process yaml documents that have stringified embed

 | Format | Decode (from string) | Encode (to string) |
 | --- | -- | --|
-| Yaml | from_yaml/@yamld | to_yaml(i)/@yaml |
-| JSON | from_json/@jsond | to_json(i)/@json |
+| Yaml | from_yaml | to_yaml(i)/@yaml |
+| JSON | from_json | to_json(i)/@json |
 | Properties | from_props/@propsd  | to_props/@props |
 | CSV | from_csv/@csvd | to_csv/@csv |
 | TSV | from_tsv/@tsvd | to_tsv/@tsv |
-| XML | from_xml/@xmld | to_xml(i)/@xml |
+| XML | from_xml | to_xml(i)/@xml |
 | Base64 | @base64d | @base64 |
-| URI | @urid | @uri |
-| Shell |  | @sh |


 See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) for accepted formats.
@@ -25,4 +23,4 @@ See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) f
 XML uses the `--xml-attribute-prefix` and `xml-content-name` flags to identify attributes and content fields.


-Base64 assumes [rfc4648](https://rfc-editor.org/rfc/rfc4648.html) encoding. Encoding and decoding both assume that the content is a utf-8 string and not binary content.
+Base64 assumes [rfc4648](https://rfc-editor.org/rfc/rfc4648.html) encoding. Encoding and decoding both assume that the content is a string.
@@ -2,6 +2,6 @@

 Use `eval` to dynamically process an expression - for instance from an environment variable.

-`eval` takes a single argument, and evaluates that as a `yq` expression. Any valid expression can be used, be it a path `.a.b.c | select(. == "cat")`, or an update `.a.b.c = "gogo"`.
+`eval` takes a single argument, and evaluates that as a `yq` expression. Any valid expression can be used, beit a path `.a.b.c | select(. == "cat")`, or an update `.a.b.c = "gogo"`.

-Tip: This can be a useful way to parameterise complex scripts.
+Tip: This can be useful way parameterise complex scripts.
@@ -1,4 +0,0 @@
-# Filter
-
-Filters an array (or map values) by the expression given. Equivalent to doing `map(select(exp))`.
-
@@ -1,3 +1,3 @@
 # Has

-This operation returns true if the key exists in a map (or index in an array), false otherwise.
+This is operation that returns true if the key exists in a map (or index in an array), false otherwise.
@@ -14,8 +14,6 @@ You can load files of the following supported types:
 | Plain String | load_str |
 | Base64 | load_base64 |

-Note that load_base64 only works for base64 encoded utf-8 strings.
-
 ## Samples files for tests:

 ### yaml
@@ -1,3 +0,0 @@
-# Modulo
-
-Arithmetic modulo operator, returns the remainder from dividing two numbers.
@@ -14,7 +14,6 @@ You can control how objects are merged by using one or more of the following fla
 - `d` deeply merge arrays
 - `?` only merge _existing_ fields
 - `n` only merge _new_ fields
- `c` clobber custom tags


 ### Merge two files together
@@ -1,8 +1,5 @@
 # Path

-The `path` operator can be used to get the traversal paths of matching nodes in an expression. The path is returned as an array, which if traversed in order will lead to the matching node.
+The path operator can be used to get the traversal paths of matching nodes in an expression. The path is returned as an array, which if traversed in order will lead to the matching node.

 You can get the key/index of matching nodes by using the `path` operator to return the path array then piping that through `.[-1]` to get the last element of that array, the key.
-
-Use `setpath` to set a value to the path array returned by `path`, and similarly `delpaths` for an array of path arrays.
-
--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Mike Farah	d8f8661567	Fixing from review	2022-08-01 10:16:59 +10:00
Mike Farah	9628aaf8c5	Added CSV roundtrip	2022-08-01 10:05:26 +10:00
Mike Farah	cb0bae128f	Adding CSV decoder	2022-07-31 17:35:36 +10:00
Mike Farah	183a42c249	WIP: adding CSV decoder	2022-07-31 17:31:27 +10:00