Add build tag to hcl_test.go to skip tests when HCL is disabled

Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com>

.github/ISSUE_TEMPLATE/bug_report_v3.md

@@ -1,51 +0,0 @@
----
-name: Bug report - V3
-about: Create a report to help us improve
-title: ''
-labels: bug, v3
-assignees: ''
-
----
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-Note that any how to questions should be posted in the discussion board and not raised as an issue.
-
-Version of yq: 3.X.X
-Operating system: mac/linux/windows/....
-Installed via: docker/binary release/homebrew/snap/...
-
-**Input Yaml**
-Concise yaml document(s) (as simple as possible to show the bug, please keep it to 10 lines or less)
-data1.yml:
-```yaml
-this: should really work
-```
-
-data2.yml:
-```yaml
-but: it strangely didn't
-```
-
-**Command**
-The command you ran:
-```
-yq merge data1.yml data2.yml
-```
-
-**Actual behavior**
-
-```yaml
-cat: meow
-```
-
-**Expected behavior**
-
-```yaml
-this: should really work
-but: it strangely didn't
-```
-
-**Additional context**
-Add any other context about the problem here.

.github/ISSUE_TEMPLATE/feature_request.md

@@ -26,7 +26,7 @@ country: Australia
 And we run a command:
 
 ```bash
-yq eval 'predictWeatherOf(.country)'
+yq 'predictWeatherOf(.country)'
 ```
 
 it could output

.github/dependabot.yml

@@ -0,0 +1,22 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: docker
+    directory: /
+    schedule:
+      day: thursday
+      interval: weekly
+  - package-ecosystem: github-actions
+    directory: /
+    schedule:
+      day: thursday
+      interval: weekly
+  - package-ecosystem: gomod
+    directory: /
+    schedule:
+      day: thursday
+      interval: weekly

.github/workflows/codeql.yml

@@ -0,0 +1,70 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master ]
+  schedule:
+    - cron: '24 3 * * 1'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v6
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v4
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v4
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v4

.github/workflows/docker-release.yml

@@ -0,0 +1,100 @@
+name: Release Docker
+
+on:
+  release:
+    types: [released]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  publishDocker:
+    environment: dockerhub
+    env:
+      IMAGE_NAME: mikefarah/yq
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: all
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          version: latest
+
+      - name: Available platforms
+        run: echo ${{ steps.buildx.outputs.platforms }} && docker version
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push image
+        run: |
+          echo "GithubRef: ${GITHUB_REF}"
+          VERSION=${GITHUB_REF##*/}
+          echo "VERSION: ${VERSION}"
+          IMAGE_VERSION=${VERSION:1}
+          echo "IMAGE_VERSION: ${IMAGE_VERSION}"
+
+          PLATFORMS="linux/amd64,linux/ppc64le,linux/arm64,linux/arm/v7,linux/s390x"
+
+          echo "Building and pushing version ${IMAGE_VERSION} of image ${IMAGE_NAME}"
+          docker buildx build \
+            --label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
+            --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
+            --label "org.opencontainers.image.description=yq is a portable command-line data file processor" \
+            --label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
+            --label "org.opencontainers.image.licenses=MIT" \
+            --label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \
+            --label "org.opencontainers.image.source=https://github.com/mikefarah/yq" \
+            --label "org.opencontainers.image.title=yq" \
+            --label "org.opencontainers.image.url=https://mikefarah.gitbook.io/yq/" \
+            --label "org.opencontainers.image.version=${IMAGE_VERSION}" \
+            --platform "${PLATFORMS}" \
+            --pull \
+            --push \
+            -t "${IMAGE_NAME}:${IMAGE_VERSION}" \
+            -t "${IMAGE_NAME}:4" \
+            -t "${IMAGE_NAME}:latest" \
+            -t "ghcr.io/${IMAGE_NAME}:${IMAGE_VERSION}" \
+            -t "ghcr.io/${IMAGE_NAME}:4" \
+            -t "ghcr.io/${IMAGE_NAME}:latest" \
+            .
+
+          cd github-action
+          docker buildx build \
+            --label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
+            --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
+            --label "org.opencontainers.image.description=yq is a portable command-line data file processor" \
+            --label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
+            --label "org.opencontainers.image.licenses=MIT" \
+            --label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \
+            --label "org.opencontainers.image.source=https://github.com/mikefarah/yq" \
+            --label "org.opencontainers.image.title=yq" \
+            --label "org.opencontainers.image.url=https://mikefarah.gitbook.io/yq/" \
+            --label "org.opencontainers.image.version=${IMAGE_VERSION}" \
+            --platform "${PLATFORMS}" \
+            --pull \
+            --push \
+            -t "${IMAGE_NAME}:${IMAGE_VERSION}-githubaction" \
+            -t "${IMAGE_NAME}:4-githubaction" \
+            -t "${IMAGE_NAME}:latest-githubaction" \
+            -t "ghcr.io/${IMAGE_NAME}:${IMAGE_VERSION}-githubaction" \
+            -t "ghcr.io/${IMAGE_NAME}:4-githubaction" \
+            -t "ghcr.io/${IMAGE_NAME}:latest-githubaction" \
+            .

.github/workflows/go.yml

@@ -1,5 +1,8 @@
 name: Build
-on: [push]
+on: [push, pull_request]
+permissions:
+  contents: read
+
 jobs:
 
   build:
@@ -7,14 +10,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.15
-      uses: actions/setup-go@v1
+    - name: Set up Go
+      uses: actions/setup-go@v5
       with:
-        go-version: 1.15
+        go-version: '^1.20'
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@v6
 
     - name: Get dependencies
       run: |
@@ -24,8 +27,9 @@ jobs:
             dep ensure
         fi
 
-    - name: Download deps
+    - name: Check the build
+      shell: bash -l {0}
       run: |
         export PATH=${PATH}:`go env GOPATH`/bin
         scripts/devtools.sh
-        make local build
+        make local build

.github/workflows/release.yml

@@ -2,26 +2,27 @@ name: Release YQ
 on:
   push:
     tags:
-      - 'v*'
+      - 'v4.*'
+      - 'draft-*'
 
 jobs:
   publishGitRelease:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/checkout@v6
+      - uses: actions/setup-go@v5
         with:
-          go-version: '^1.17.2'
+          go-version: '^1.20'
+          check-latest: true
       - name: Compile man page markup
         id: gen-man-page-md
         run: |
           ./scripts/generate-man-page-md.sh
-          echo "::set-output name=man-page-md::man.md"
 
       - name: Get the version
         id: get_version
-        run: echo ::set-output name=VERSION::${GITHUB_REF##*/}
-        
+        run: echo "VERSION=${GITHUB_REF##*/}" >> "${GITHUB_OUTPUT}"
+
       - name: Generate man page
         uses: docker://pandoc/core:2.14.2
         id: gen-man-page
@@ -34,63 +35,17 @@ jobs:
             --variable=header:"yq (https://github.com/mikefarah/yq/) version ${{ steps.get_version.outputs.VERSION }}"
             --variable=author:"Mike Farah"
             --output=yq.1
-            ${{ steps.gen-man-page-md.outputs.man-page-md }}
-        
+            man.md
+
       - name: Cross compile
         run: |
           sudo apt-get install rhash -y
-          go get github.com/mitchellh/gox
-          mkdir -p build
-          cp yq.1 build/yq.1
+          go install github.com/goreleaser/goreleaser/v2@latest
           ./scripts/xcompile.sh
 
-      - name: Create Release
-        id: create_release
-        uses: actions/create-release@v1.0.0
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Release
+        uses: softprops/action-gh-release@v1
         with:
-          tag_name: ${{ github.ref }}
-          release_name: ${{ github.ref }}
+          files: build/*
           draft: true
-          prerelease: false
-
-      - uses: shogo82148/actions-upload-release-asset@v1
-        with:
-          upload_url: ${{ steps.create_release.outputs.upload_url }}
-          asset_path: build/*
-
-  publishDocker:
-    environment: dockerhub
-    env:
-      IMAGE_NAME: mikefarah/yq
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-        with:
-          platforms: all
-  
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          version: latest
-
-      - name: Available platforms
-        run: echo ${{ steps.buildx.outputs.platforms }} && docker version
- 
-      - name: Build and push image
-        run: |
-          IMAGE_V_VERSION="$(git describe --tags --abbrev=0)"
-          IMAGE_VERSION=${IMAGE_V_VERSION:1}
-
-          SHORT_SHA1=$(git rev-parse --short HEAD)
-          PLATFORMS="linux/amd64,linux/ppc64le,linux/arm64"
-          echo "Building and pushing version ${IMAGE_VERSION} of image ${IMAGE_NAME}"
-          echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
-          docker buildx build --platform "${PLATFORMS}" -t "${IMAGE_NAME}:${IMAGE_VERSION}"  -t "${IMAGE_NAME}:latest" -t "${IMAGE_NAME}:4" \
-            --push .
-    
+          fail_on_unmatched_files: true

.github/workflows/snap-release.yml

@@ -0,0 +1,27 @@
+name: Release Snap
+
+on:
+  release:
+    types: [released]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+    buildSnap:
+      environment: snap
+      runs-on: ubuntu-latest
+      steps:
+        - uses: actions/checkout@v6
+        - uses: snapcore/action-build@v1
+          id: build
+          env:
+            SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.STORE_LOGIN }}
+          with:
+            snapcraft-args: "remote-build --launchpad-accept-public-upload"
+        - uses: snapcore/action-publish@v1
+          env:
+            SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.STORE_LOGIN }}
+          with:
+            snap: ${{ steps.build.outputs.snap }}
+            release: stable

.github/workflows/test-yq.yml

@@ -1,5 +1,11 @@
 name: Test Yq Action
-on: [push]
+on: 
+  push:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
 jobs:
 
   build:
@@ -7,13 +13,16 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v6
       - name: Get test
+        id: get_value
         uses: mikefarah/yq@master
         with:
-          cmd: yq eval '.a' examples/data1.yaml
+          cmd: yq '.test' examples/multiline-text.yaml
+      - name: Multiline test
+        run: echo "### It was [${{ steps.get_value.outputs.result }}]" >> $GITHUB_STEP_SUMMARY
       - name: Write inplace test
         id: lookupSdkVersion
         uses: mikefarah/yq@master
         with:
-          cmd: yq eval -i '.a.b = 5' examples/data1.yaml
+          cmd: yq -i '.b.c = 5' examples/sample.yaml

.gitignore

@@ -22,8 +22,10 @@ _cgo_gotypes.go
 _cgo_export.*
 
 _testmain.go
+cover.out
 coverage.out
 coverage.html
+coverage_sorted.txt
 *.exe
 *.test
 *.prof
@@ -38,10 +40,31 @@ parts/
 prime/
 .snapcraft/
 yq*.snap
+
 test.yml
-test2.yml
+test*.yml
+test*.xml
+test*.toml
+test*.yaml
+test_dir1/
+test_dir2/
+0.yml
+1.yml
+2.yml
 
 
 # man page
 man.md
-yq.1
+yq.1
+
+# debian pkg
+_build
+debian/files
+
+# intellij
+/.idea
+
+# vscode
+.vscode
+
+yq3

.golangci.bck.yml

@@ -0,0 +1,38 @@
+run:
+  timeout: 5m
+linters:
+  enable:
+    - asciicheck
+    - depguard
+    - errorlint
+    - gci
+    - gochecknoinits
+    - gofmt
+    - goimports
+    - gosec
+    - gosimple
+    - staticcheck
+    - unused
+    - misspell
+    - nakedret
+    - nolintlint
+    - predeclared
+    - revive
+    - unconvert
+    - unparam
+linters-settings:
+  depguard:
+    rules:
+      prevent_unmaintained_packages:
+        list-mode: lax
+        files:
+          - $all
+          - "!$test"
+        deny:
+          - pkg: io/ioutil
+            desc: "replaced by io and os packages since Go 1.16: https://tip.golang.org/doc/go1.16#ioutil"
+issues:
+  exclude-rules:
+  - linters:
+    - revive
+    text: "var-naming"

.golangci.yml

@@ -0,0 +1,52 @@
+version: "2"
+linters:
+  enable:
+    - asciicheck
+    - depguard
+    - errorlint
+    - gochecknoinits
+    - gosec
+    - misspell
+    - nakedret
+    - nolintlint
+    - predeclared
+    - revive
+    - unconvert
+    - unparam
+  settings:
+    depguard:
+      rules:
+        prevent_unmaintained_packages:
+          list-mode: lax
+          files:
+            - $all
+            - '!$test'
+          deny:
+            - pkg: io/ioutil
+              desc: 'replaced by io and os packages since Go 1.16: https://tip.golang.org/doc/go1.16#ioutil'
+  exclusions:
+    generated: lax
+    presets:
+      - comments
+      - common-false-positives
+      - legacy
+      - std-error-handling
+    rules:
+      - linters:
+          - revive
+        text: var-naming
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
+formatters:
+  enable:
+    - gci
+    - gofmt
+    - goimports
+  exclusions:
+    generated: lax
+    paths:
+      - third_party$
+      - builtin$
+      - examples$

.goreleaser.yaml

@@ -0,0 +1,49 @@
+version: 2
+
+dist: build
+
+builds:
+  - id: yq
+
+    binary: yq_{{ .Os }}_{{ .Arch }}
+
+    ldflags:
+      - -s -w
+
+    env:
+      - CGO_ENABLED=0
+
+    targets:
+      - darwin_amd64
+      - darwin_arm64
+      - freebsd_386
+      - freebsd_amd64
+      - freebsd_arm
+      - linux_386
+      - linux_amd64
+      - linux_arm
+      - linux_arm64
+      - linux_loong64
+      - linux_mips
+      - linux_mips64
+      - linux_mips64le
+      - linux_mipsle
+      - linux_ppc64
+      - linux_ppc64le
+      - linux_riscv64
+      - linux_s390x
+      - netbsd_386
+      - netbsd_amd64
+      - netbsd_arm
+      - openbsd_386
+      - openbsd_amd64
+      - windows_386
+      - windows_amd64
+      - windows_arm
+      - windows_arm64
+
+    no_unique_dist_dir: true
+
+release:
+  disable: true
+  skip_upload: true

CONTRIBUTING.md

@@ -1,8 +1,214 @@
-1. Install (golang)[https://golang.org/]
-1. Run  `scripts/devtools.sh` to install the required devtools
-2. Run `make [local] vendor` to install the vendor dependencies
-2. Run `make [local] test` to ensure you can run the existing tests
-3. Write unit tests - (see existing examples). Changes will not be accepted without corresponding unit tests.
-4. Make the code changes.
-5. `make [local] test` to lint code and run tests
-6. Profit! ok no profit, but raise a PR and get kudos :)
+# Before you begin 
+Not all new PRs will be merged in 
+
+It's recommended to check with the owner first (e.g. raise an issue) to discuss a new feature before developing, to ensure your hard efforts don't go to waste.
+
+PRs to fix bugs and issues are almost always welcome :pray: please ensure you write tests as well.
+
+The following types of PRs will _not_ be accepted:
+- **Significant refactors** take a lot of time to understand and can have all sorts of unintended side effects. If you think there's a better way to do things (that requires significant changes) raise an issue for discussion first :)
+- **Release pipeline PRs** are a security risk - it's too easy for a serious vulnerability to sneak in (either intended or not). If there is a new cool way of releasing things, raise an issue for discussion first - it will need to be gone over with a fine tooth comb.
+- **Version bumps** are handled by dependabot, the bot will auto-raise PRs and they will be regularly merged in. 
+- **New release platforms** At this stage, yq is not going to maintain any other release platforms other than GitHub and Docker - that said, I'm more than happy to put in other community maintained methods in the README for visibility :heart:
+
+
+# Development
+
+## Initial Setup
+
+1. Install [Golang](https://golang.org/) (version 1.24.0 or later)
+2. Run `scripts/devtools.sh` to install required development tools:
+   - golangci-lint for code linting
+   - gosec for security analysis
+3. Run `make [local] vendor` to install vendor dependencies
+4. Run `make [local] test` to ensure you can run the existing tests
+
+## Development Workflow
+
+1. **Write unit tests first** - Changes will not be accepted without corresponding unit tests (see Testing section below)
+2. **Make your code changes**
+3. **Run tests and linting**: `make [local] test` (this runs formatting, linting, security checks, and tests)
+4. **Create your PR** and get kudos! :)
+
+## Make Commands
+
+- Use `make [local] <command>` for local development (runs in Docker container)
+- Use `make <command>` for CI/CD environments
+- Common commands:
+  - `make [local] vendor` - Install dependencies
+  - `make [local] test` - Run all checks and tests
+  - `make [local] build` - Build the yq binary
+  - `make [local] format` - Format code
+  - `make [local] check` - Run linting and security checks
+
+# Code Quality
+
+## Linting and Formatting
+
+The project uses strict linting rules defined in `.golangci.yml`. All code must pass:
+
+- **Code formatting**: gofmt, goimports, gci
+- **Linting**: revive, errorlint, gosec, misspell, and others
+- **Security checks**: gosec security analysis
+- **Spelling checks**: misspell detection
+
+Run `make [local] check` to verify your code meets all quality standards.
+
+## Code Style Guidelines
+
+- Follow standard Go conventions
+- Use meaningful variable names
+- Add comments for public functions and complex logic
+- Keep functions focused and reasonably sized
+- Use the project's existing patterns and conventions
+
+# Testing
+
+## Test Structure
+
+Tests in yq use the `expressionScenario` pattern. Each test scenario includes:
+- `expression`: The yq expression to test
+- `document`: Input YAML/JSON (optional)
+- `expected`: Expected output
+- `skipDoc`: Whether to skip documentation generation
+
+## Writing Tests
+
+1. **Find the appropriate test file** (e.g., `operator_add_test.go` for addition operations)
+2. **Add your test scenario** to the `*OperatorScenarios` slice
+3. **Run the specific test**: `go test -run TestAddOperatorScenarios` (replace with appropriate test name)
+4. **Verify documentation generation** (see Documentation section)
+
+## Test Examples
+
+```go
+var addOperatorScenarios = []expressionScenario{
+    {
+        skipDoc:    true,
+        expression: `"foo" + "bar"`,
+        expected: []string{
+            "D0, P[], (!!str)::foobar\n",
+        },
+    },
+    {
+        document:   "apples: 3",
+        expression: `.apples + 3`,
+        expected: []string{
+            "D0, P[apples], (!!int)::6\n",
+        },
+    },
+}
+```
+
+## Running Tests
+
+- **All tests**: `make [local] test`
+- **Specific test**: `go test -run TestName`
+- **With coverage**: `make [local] cover`
+
+# Documentation
+
+## Documentation Generation
+
+The project uses a documentation system that combines static headers with dynamically generated content from tests.
+
+### How It Works
+
+1. **Static headers** are defined in `pkg/yqlib/doc/operators/headers/*.md`
+2. **Dynamic content** is generated from test scenarios in `*_test.go` files
+3. **Generated docs** are created in `pkg/yqlib/doc/*.md` by concatenating headers with test-generated content
+4. **Documentation is synced** to the gitbook branch for the website
+
+### Updating Operator Documentation
+
+#### For Test-Generated Documentation
+
+Most operator documentation is generated from tests. To update:
+
+1. **Find the test file** (e.g., `operator_add_test.go`)
+2. **Update test scenarios** - each `expressionScenario` with `skipDoc: false` becomes documentation
+3. **Run the test** to regenerate docs:
+   ```bash
+   cd pkg/yqlib
+   go test -run TestAddOperatorScenarios
+   ```
+4. **Verify the generated documentation** in `pkg/yqlib/doc/add.md`
+5. **Create a PR** with your changes
+
+#### For Header-Only Documentation
+
+If documentation exists only in `headers/*.md` files:
+1. **Update the header file directly** (e.g., `pkg/yqlib/doc/operators/headers/add.md`)
+2. **Create a PR** with your changes
+
+### Updating Static Documentation
+
+For documentation not in the master branch:
+
+1. **Check the gitbook branch** for additional pages
+2. **Update the `*.md` files** directly
+3. **Create a PR** to the gitbook branch
+
+### Documentation Best Practices
+
+- **Write clear, concise examples** in test scenarios
+- **Use meaningful variable names** in examples
+- **Include edge cases** and error conditions
+- **Test your documentation changes** by running the specific test
+- **Verify generated output** matches expectations
+
+Note: PRs with small changes (e.g. minor typos) may not be merged (see https://joel.net/how-one-guy-ruined-hacktoberfest2020-drama).
+
+# Troubleshooting
+
+## Common Setup Issues
+
+### Docker/Podman Issues
+- **Problem**: `make` commands fail with Docker errors
+- **Solution**: Ensure Docker or Podman is running and accessible
+- **Alternative**: Use `make local <command>` to run in containers
+
+### Go Version Issues
+- **Problem**: Build fails with Go version errors
+- **Solution**: Ensure you have Go 1.24.0 or later installed
+- **Check**: Run `go version` to verify
+
+### Vendor Dependencies
+- **Problem**: `make vendor` fails or dependencies are outdated
+- **Solution**: 
+  ```bash
+  go mod tidy
+  make [local] vendor
+  ```
+
+### Linting Failures
+- **Problem**: `make check` fails with linting errors
+- **Solution**: 
+  ```bash
+  make [local] format  # Auto-fix formatting
+  # Manually fix remaining linting issues
+  make [local] check   # Verify fixes
+  ```
+
+### Test Failures
+- **Problem**: Tests fail locally but pass in CI
+- **Solution**: 
+  ```bash
+  make [local] test    # Run in Docker container
+  ```
+
+### Documentation Generation Issues
+- **Problem**: Generated docs don't update after test changes
+- **Solution**: 
+  ```bash
+  cd pkg/yqlib
+  go test -run TestSpecificOperatorScenarios
+  # Check if generated file updated in pkg/yqlib/doc/
+  ```
+
+## Getting Help
+
+- **Check existing issues**: Search GitHub issues for similar problems
+- **Create an issue**: If you can't find a solution, create a detailed issue
+- **Ask questions**: Use GitHub Discussions for general questions
+- **Join the community**: Check the project's community channels

Dockerfile

@@ -1,30 +1,28 @@
-FROM golang:1.17 as builder
+FROM golang:1.25.5 AS builder
 
 WORKDIR /go/src/mikefarah/yq
 
-COPY . /go/src/mikefarah/yq
+COPY . .
 
-RUN CGO_ENABLED=0 go build .
-RUN ./scripts/test.sh
+RUN CGO_ENABLED=0 go build -ldflags "-s -w" .
+# RUN ./scripts/test.sh -- this too often times out in the github pipeline.
 RUN ./scripts/acceptance.sh
 
 # Choose alpine as a base image to make this useful for CI, as many
 # CI tools expect an interactive shell inside the container
-FROM alpine:3 as production
-
-RUN mkdir /home/yq/
-RUN addgroup -g 1000 yq && \
-    adduser -u 1000 -G yq -s /bin/bash -h /home/yq -D yq
-RUN chown -R yq:yq /home/yq/
+FROM alpine:3 AS production
+LABEL maintainer="Mike Farah <mikefarah@users.noreply.github.com>"
 
 COPY --from=builder /go/src/mikefarah/yq/yq /usr/bin/yq
-RUN chmod +x /usr/bin/yq
-
-ARG VERSION=none
-LABEL version=${VERSION}
-
-USER yq
 
 WORKDIR /workdir
 
+RUN set -eux; \
+  addgroup -g 1000 yq; \
+  adduser -u 1000 -G yq -s /bin/sh -h /home/yq -D yq
+
+RUN chown -R yq:yq /workdir
+
+USER yq
+
 ENTRYPOINT ["/usr/bin/yq"]

Dockerfile.dev

@@ -1,9 +1,14 @@
-FROM golang:1.17
+FROM golang:1.25.5
+
+RUN apt-get update && \
+    apt-get install -y npm && \
+    npm install -g npx cspell@latest
 
 COPY scripts/devtools.sh /opt/devtools.sh
 
-RUN set -e -x \
-    && /opt/devtools.sh
+RUN set -e -x && \
+    /opt/devtools.sh
+
 ENV PATH=/go/bin:$PATH
 
 ENV CGO_ENABLED 0

LICENSE

@@ -1,3 +1,5 @@
+The MIT License (MIT)
+
 Copyright (c) 2017 Mike Farah
 
 Permission is hereby granted, free of charge, to any person obtaining a copy

Makefile

@@ -2,6 +2,7 @@ MAKEFLAGS += --warn-undefined-variables
 SHELL := /bin/bash
 .SHELLFLAGS := -o pipefail -euc
 .DEFAULT_GOAL := install
+ENGINE := $(shell { (podman version > /dev/null 2>&1 && command -v podman) || command -v docker; } 2>/dev/null)
 
 include Makefile.variables
 
@@ -33,7 +34,7 @@ clean:
 
 ## prefix before other make targets to run in your local dev environment
 local: | quiet
-	@$(eval DOCKRUN= )
+	@$(eval ENGINERUN= )
 	@mkdir -p tmp
 	@touch tmp/dev_image_id
 quiet: # this is silly but shuts up 'Nothing to be done for `local`'
@@ -42,9 +43,9 @@ quiet: # this is silly but shuts up 'Nothing to be done for `local`'
 prepare: tmp/dev_image_id
 tmp/dev_image_id: Dockerfile.dev scripts/devtools.sh
 	@mkdir -p tmp
-	@docker rmi -f ${DEV_IMAGE} > /dev/null 2>&1 || true
-	@docker build -t ${DEV_IMAGE} -f Dockerfile.dev .
-	@docker inspect -f "{{ .ID }}" ${DEV_IMAGE} > tmp/dev_image_id
+	@${ENGINE} rmi -f ${DEV_IMAGE} > /dev/null 2>&1 || true
+	@${ENGINE} build -t ${DEV_IMAGE} -f Dockerfile.dev .
+	@${ENGINE} inspect -f "{{ .ID }}" ${DEV_IMAGE} > tmp/dev_image_id
 
 # ----------------------------------------------
 # build
@@ -54,57 +55,65 @@ build: build/dev
 .PHONY: build/dev
 build/dev: test *.go
 	@mkdir -p bin/
-	${DOCKRUN} go build --ldflags "$(LDFLAGS)"
-	${DOCKRUN} bash ./scripts/acceptance.sh
+	${ENGINERUN} go build --ldflags "$(LDFLAGS)"
+	${ENGINERUN} bash ./scripts/acceptance.sh
 
 ## Compile the project for multiple OS and Architectures.
 xcompile: check
 	@rm -rf build/
 	@mkdir -p build
-	${DOCKRUN} bash ./scripts/xcompile.sh
+	${ENGINERUN} bash ./scripts/xcompile.sh
 	@find build -type d -exec chmod 755 {} \; || :
 	@find build -type f -exec chmod 755 {} \; || :
 
 .PHONY: install
 install: build
-	${DOCKRUN} go install
+	${ENGINERUN} go install
 
 # Each of the fetch should be an entry within vendor.json; not currently included within project
 .PHONY: vendor
 vendor: tmp/dev_image_id
-	${DOCKRUN} go mod vendor
+	@mkdir -p vendor
+	${ENGINERUN} go mod vendor
 
 # ----------------------------------------------
 # develop and test
 
 .PHONY: format
 format: vendor
-	${DOCKRUN} bash ./scripts/format.sh
+	${ENGINERUN} bash ./scripts/format.sh
 
-.PHONY: check
-check: format
-	${DOCKRUN} bash ./scripts/check.sh
+
+.PHONY: spelling
+spelling: format
+	${ENGINERUN} bash ./scripts/spelling.sh
 
 .PHONY: secure
-secure: 
-	${DOCKRUN} bash ./scripts/secure.sh
+secure: spelling
+	${ENGINERUN} bash ./scripts/secure.sh
+
+.PHONY: check
+check: secure
+	${ENGINERUN} bash ./scripts/check.sh
+
+
 
 .PHONY: test
 test: check
-	${DOCKRUN} bash ./scripts/test.sh
+	${ENGINERUN} bash ./scripts/test.sh
 
 .PHONY: cover
 cover: check
 	@rm -rf cover/
 	@mkdir -p cover
-	${DOCKRUN} bash ./scripts/coverage.sh
+	${ENGINERUN} bash ./scripts/coverage.sh
 	@find cover -type d -exec chmod 755 {} \; || :
 	@find cover -type f -exec chmod 644 {} \; || :
 
 
 .PHONY: release
 release: xcompile
-	${DOCKRUN} bash ./scripts/publish.sh
+	${ENGINERUN} bash ./scripts/publish.sh
 
 # ----------------------------------------------
 # utilities

Makefile.variables

@@ -7,6 +7,7 @@ export GIT_DESCRIBE = $(shell git describe --tags --always)
 LDFLAGS :=
 LDFLAGS += -X main.GitCommit=${GIT_COMMIT}${GIT_DIRTY}
 LDFLAGS += -X main.GitDescribe=${GIT_DESCRIBE}
+LDFLAGS += -w -s
 
 GITHUB_TOKEN ?=
 
@@ -25,14 +26,15 @@ ifeq ($(CYG_CHECK),1)
 else
 	# all non-windows environments
 	ROOT := $(shell pwd)
+	SELINUX := $(shell which getenforce 2>&1 >/dev/null && echo :z)
 endif
 
 DEV_IMAGE := ${PROJECT}_dev
 
-DOCKRUN := docker run --rm \
+ENGINERUN := ${ENGINE} run --rm \
 	-e LDFLAGS="${LDFLAGS}" \
 	-e GITHUB_TOKEN="${GITHUB_TOKEN}" \
-	-v ${ROOT}/vendor:/go/src \
-	-v ${ROOT}:/${PROJECT}/src/${IMPORT_PATH} \
+	-v ${ROOT}/vendor:/go/src${SELINUX} \
+	-v ${ROOT}:/${PROJECT}/src/${IMPORT_PATH}${SELINUX} \
 	-w /${PROJECT}/src/${IMPORT_PATH} \
 	${DEV_IMAGE}

README.md

@@ -1,71 +1,107 @@
 # yq
 
-![Build](https://github.com/mikefarah/yq/workflows/Build/badge.svg)  ![Docker Pulls](https://img.shields.io/docker/pulls/mikefarah/yq.svg) ![Github Releases (by Release)](https://img.shields.io/github/downloads/mikefarah/yq/total.svg) ![Go Report](https://goreportcard.com/badge/github.com/mikefarah/yq)
+![Build](https://github.com/mikefarah/yq/workflows/Build/badge.svg)  ![Docker Pulls](https://img.shields.io/docker/pulls/mikefarah/yq.svg) ![Github Releases (by Release)](https://img.shields.io/github/downloads/mikefarah/yq/total.svg) ![Go Report](https://goreportcard.com/badge/github.com/mikefarah/yq) ![CodeQL](https://github.com/mikefarah/yq/workflows/CodeQL/badge.svg)
 
 
-a lightweight and portable command-line YAML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.
+A lightweight and portable command-line YAML, JSON, INI and XML processor. `yq` uses [jq](https://github.com/stedolan/jq) (a popular JSON processor) like syntax but works with yaml files as well as json, xml, ini, properties, csv and tsv. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.
 
-yq is written in go - so you can download a dependency free binary for your platform and you are good to go! If you prefer there are a variety of package managers that can be used as well as docker, all listed below.
+yq is written in Go - so you can download a dependency free binary for your platform and you are good to go! If you prefer there are a variety of package managers that can be used as well as Docker and Podman, all listed below.
 
 ## Quick Usage Guide
 
-Read a value:
+### Basic Operations
+
+**Read a value:**
 ```bash
-yq e '.a.b[0].c' file.yaml
+yq '.a.b[0].c' file.yaml
 ```
 
-Pipe from STDIN:
+**Pipe from STDIN:**
 ```bash
-cat file.yaml | yq e '.a.b[0].c' -
+yq '.a.b[0].c' < file.yaml
 ```
 
-Update a yaml file, inplace
+**Update a yaml file in place:**
 ```bash
-yq e -i '.a.b[0].c = "cool"' file.yaml
+yq -i '.a.b[0].c = "cool"' file.yaml
 ```
 
-Update using environment variables
+**Update using environment variables:**
 ```bash
-NAME=mike yq e -i '.a.b[0].c = strenv(NAME)' file.yaml
+NAME=mike yq -i '.a.b[0].c = strenv(NAME)' file.yaml
 ```
 
-Merge multiple files
-```
+### Advanced Operations
+
+**Merge multiple files:**
+```bash
+# merge two files
+yq -n 'load("file1.yaml") * load("file2.yaml")'
+
+# merge using globs (note: `ea` evaluates all files at once instead of in sequence)
 yq ea '. as $item ireduce ({}; . * $item )' path/to/*.yml
 ```
 
-Multiple updates to a yaml file
+**Multiple updates to a yaml file:**
 ```bash
-yq e -i '
+yq -i '
   .a.b[0].c = "cool" |
   .x.y.z = "foobar" |
   .person.name = strenv(NAME)
 ' file.yaml
 ```
 
-See the [documentation](https://mikefarah.gitbook.io/yq/) for more.
+**Find and update an item in an array:**
+```bash
+# Note: requires input file - add your file at the end
+yq -i '(.[] | select(.name == "foo") | .address) = "12 cat st"' data.yaml
+```
+
+**Convert between formats:**
+```bash
+# Convert JSON to YAML (pretty print)
+yq -Poy sample.json
+
+# Convert YAML to JSON
+yq -o json file.yaml
+
+# Convert XML to YAML
+yq -o yaml file.xml
+```
+
+See [recipes](https://mikefarah.gitbook.io/yq/recipes) for more examples and the [documentation](https://mikefarah.gitbook.io/yq/) for more information.
+
+Take a look at the discussions for [common questions](https://github.com/mikefarah/yq/discussions/categories/q-a), and [cool ideas](https://github.com/mikefarah/yq/discussions/categories/show-and-tell)
 
 ## Install
 
 ### [Download the latest binary](https://github.com/mikefarah/yq/releases/latest)
 
 ### wget
-Use wget to download the pre-compiled binaries:
+Use wget to download pre-compiled binaries. Choose your platform and architecture:
 
-#### Compressed via tar.gz
+**For Linux (example):**
 ```bash
-wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY}.tar.gz -O - |\
-  tar xz && mv ${BINARY} /usr/bin/yq
+# Set your platform variables (adjust as needed)
+VERSION=v4.2.0
+PLATFORM=linux_amd64
+
+# Download compressed binary
+wget https://github.com/mikefarah/yq/releases/download/${VERSION}/yq_${PLATFORM}.tar.gz -O - |\
+  tar xz && sudo mv yq_${PLATFORM} /usr/local/bin/yq
+
+# Or download plain binary
+wget https://github.com/mikefarah/yq/releases/download/${VERSION}/yq_${PLATFORM} -O /usr/local/bin/yq &&\
+    chmod +x /usr/local/bin/yq
 ```
 
-#### Plain binary
-
+**Latest version (Linux AMD64):**
 ```bash
-wget https://github.com/mikefarah/yq/releases/download/${VERSION}/${BINARY} -O /usr/bin/yq &&\
-    chmod +x /usr/bin/yq
+wget https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/local/bin/yq &&\
+    chmod +x /usr/local/bin/yq
 ```
 
-For instance, VERSION=v4.2.0 and BINARY=yq_linux_amd64
+**Available platforms:** `linux_amd64`, `linux_arm64`, `linux_arm`, `linux_386`, `darwin_amd64`, `darwin_arm64`, `windows_amd64`, `windows_386`, etc.
 
 ### MacOS / Linux via Homebrew:
 Using [Homebrew](https://brew.sh/)
@@ -73,57 +109,56 @@ Using [Homebrew](https://brew.sh/)
 brew install yq
 ```
 
-or, for the (deprecated) v3 version:
-
-```
-brew install yq@3
-```
-
-Note that for v3, as it is a versioned brew it will not add the `yq` command to your path automatically. Please follow the instructions given by brew upon installation.
-
 ### Linux via snap:
 ```
 snap install yq
 ```
 
-or, for the (deprecated) v3 version:
-
-```
-snap install yq --channel=v3/stable
-```
-
 #### Snap notes
 `yq` installs with [_strict confinement_](https://docs.snapcraft.io/snap-confinement/6233) in snap, this means it doesn't have direct access to root files. To read root files you can:
 
 ```
-sudo cat /etc/myfile | yq e '.a.path' - 
+sudo cat /etc/myfile | yq '.a.path'
 ```
 
 And to write to a root file you can either use [sponge](https://linux.die.net/man/1/sponge):
 ```
-sudo cat /etc/myfile | yq e '.a.path = "value"' - | sudo sponge /etc/myfile
+sudo cat /etc/myfile | yq '.a.path = "value"' | sudo sponge /etc/myfile
 ```
 or write to a temporary file:
 ```
-sudo cat /etc/myfile | yq e '.a.path = "value"' | sudo tee /etc/myfile.tmp
+sudo cat /etc/myfile | yq '.a.path = "value"' | sudo tee /etc/myfile.tmp
 sudo mv /etc/myfile.tmp /etc/myfile
 rm /etc/myfile.tmp
 ```
 
-### Run with Docker
-
-#### Oneshot use:
+### Run with Docker or Podman
 
+#### One-time use:
 ```bash
-docker run --rm -v "${PWD}":/workdir mikefarah/yq <command> [flags] [expression ]FILE...
+# Docker - process files in current directory
+docker run --rm -v "${PWD}":/workdir mikefarah/yq '.a.b[0].c' file.yaml
+
+# Podman - same usage as Docker
+podman run --rm -v "${PWD}":/workdir mikefarah/yq '.a.b[0].c' file.yaml
 ```
 
-#### Pipe in via STDIN:
+**Security note:** You can run `yq` in Docker with restricted privileges:
+```bash
+docker run --rm --security-opt=no-new-privileges --cap-drop all --network none \
+  -v "${PWD}":/workdir mikefarah/yq '.a.b[0].c' file.yaml
+```
 
-You'll need to pass the `-i\--interactive` flag to docker:
+#### Pipe data via STDIN:
+
+You'll need to pass the `-i --interactive` flag to Docker/Podman:
 
 ```bash
-cat myfile.yml | docker run -i --rm mikefarah/yq e . -
+# Process piped data
+docker run -i --rm mikefarah/yq '.this.thing' < myfile.yml
+
+# Same with Podman
+podman run -i --rm mikefarah/yq '.this.thing' < myfile.yml
 ```
 
 #### Run commands interactively:
@@ -132,6 +167,10 @@ cat myfile.yml | docker run -i --rm mikefarah/yq e . -
 docker run --rm -it -v "${PWD}":/workdir --entrypoint sh mikefarah/yq
 ```
 
+```bash
+podman run --rm -it -v "${PWD}":/workdir --entrypoint sh mikefarah/yq
+```
+
 It can be useful to have a bash function to avoid typing the whole docker command:
 
 ```bash
@@ -140,45 +179,99 @@ yq() {
 }
 ```
 
-
+```bash
+yq() {
+  podman run --rm -i -v "${PWD}":/workdir mikefarah/yq "$@"
+}
+```
 #### Running as root:
 
-`yq`'s docker image no longer runs under root (https://github.com/mikefarah/yq/pull/860). If you'd like to install more things in the docker image, or you're having permissions issues when attempting to read/write files you'll need to either:
+`yq`'s container image no longer runs under root (https://github.com/mikefarah/yq/pull/860). If you'd like to install more things in the container image, or you're having permissions issues when attempting to read/write files you'll need to either:
 
 
 ```
 docker run --user="root" -it --entrypoint sh mikefarah/yq
 ```
 
-Or, in your docker file:
+```
+podman run --user="root" -it --entrypoint sh mikefarah/yq
+```
+
+Or, in your Dockerfile:
 
 ```
 FROM mikefarah/yq
 
 USER root
-RUN apk add bash
+RUN apk add --no-cache bash
 USER yq
 ```
 
+#### Missing timezone data
+By default, the alpine image yq uses does not include timezone data. If you'd like to use the `tz` operator, you'll need to include this data:
+
+```
+FROM mikefarah/yq
+
+USER root
+RUN apk add --no-cache tzdata
+USER yq
+```
+
+#### Podman with SELinux
+
+If you are using podman with SELinux, you will need to set the shared volume flag `:z` on the volume mount:
+
+```
+-v "${PWD}":/workdir:z
+```
+
 ### GitHub Action
 ```
   - name: Set foobar to cool
     uses: mikefarah/yq@master
     with:
-      cmd: yq eval -i '.foo.bar = "cool"' 'config.yml'
+      cmd: yq -i '.foo.bar = "cool"' 'config.yml'
+  - name: Get an entry with a variable that might contain dots or spaces
+    id: get_username
+    uses: mikefarah/yq@master
+    with:
+      cmd: yq '.all.children.["${{ matrix.ip_address }}"].username' ops/inventories/production.yml
+  - name: Reuse a variable obtained in another step
+    run: echo ${{ steps.get_username.outputs.result }}
 ```
 
 See https://mikefarah.gitbook.io/yq/usage/github-action for more.
 
-### Go Get:
+### Go Install:
 ```
-GO111MODULE=on go get github.com/mikefarah/yq/v4
+go install github.com/mikefarah/yq/v4@latest
 ```
 
 ## Community Supported Installation methods
 As these are supported by the community :heart: - however, they may be out of date with the officially supported releases.
 
-# Webi
+_Please note that the Debian package (previously supported by @rmescandon) is no longer maintained. Please use an alternative installation method._
+
+
+### X-CMD
+Checkout `yq` on x-cmd: https://x-cmd.com/mod/yq
+
+- Instant Results: See the output of your yq filter in real-time.
+- Error Handling: Encounter a syntax error? It will display the error message and the results of the closest valid filter
+
+Thanks @edwinjhlee!
+
+### Nix
+
+```
+nix profile install nixpkgs#yq-go
+```
+
+See [here](https://search.nixos.org/packages?channel=unstable&show=yq-go&from=0&size=50&sort=relevance&type=packages&query=yq-go)
+
+
+### Webi
 
 ```
 webi yq
@@ -187,7 +280,16 @@ webi yq
 See [webi](https://webinstall.dev/)
 Supported by @adithyasunil26 (https://github.com/webinstall/webi-installers/tree/master/yq)
 
+### Arch Linux
+
+```
+pacman -S go-yq
+```
+
 ### Windows:
+
+Using [Chocolatey](https://chocolatey.org)
+
 [![Chocolatey](https://img.shields.io/chocolatey/v/yq.svg)](https://chocolatey.org/packages/yq)
 [![Chocolatey](https://img.shields.io/chocolatey/dt/yq.svg)](https://chocolatey.org/packages/yq)
 ```
@@ -195,7 +297,17 @@ choco install yq
 ```
 Supported by @chillum (https://chocolatey.org/packages/yq)
 
-### Mac:
+Using [scoop](https://scoop.sh/)
+```
+scoop install main/yq
+```
+
+Using [winget](https://learn.microsoft.com/en-us/windows/package-manager/)
+```
+winget install --id MikeFarah.yq
+```
+
+### MacPorts:
 Using [MacPorts](https://www.macports.org/)
 ```
 sudo port selfupdate
@@ -204,39 +316,55 @@ sudo port install yq
 Supported by @herbygillot (https://ports.macports.org/maintainer/github/herbygillot)
 
 ### Alpine Linux
-- Enable edge/community repo by adding ```$MIRROR/alpine/edge/community``` to ```/etc/apk/repositories```
-- Update database index with ```apk update```
-- Install yq with ```apk add yq```
 
-Supported by Tuan Hoang
-https://pkgs.alpinelinux.org/package/edge/community/x86/yq
+Alpine Linux v3.20+ (and Edge):
+```
+apk add yq-go
+```
 
+Alpine Linux up to v3.19:
+```
+apk add yq
+```
 
-### On Ubuntu 16.04 or higher from Debian package:
-```sh
-sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys CC86BB64
-sudo add-apt-repository ppa:rmescandon/yq
-sudo apt update
-sudo apt install yq -y
+Supported by Tuan Hoang (https://pkgs.alpinelinux.org/packages?name=yq-go)
+
+### Flox:
+
+Flox can be used to install yq on Linux, MacOS, and Windows through WSL.
+
+```
+flox install yq
+```
+
+
+### MacOS / Linux via gah:
+Using [gah](https://github.com/marverix/gah)
+
+```
+gah install yq
 ```
-Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
 
 ## Features
 - [Detailed documentation with many examples](https://mikefarah.gitbook.io/yq/)
 - Written in portable go, so you can download a lovely dependency free binary
-- Uses similar syntax as `jq` but works with YAML and JSON files
+- Uses similar syntax as `jq` but works with YAML, INI, [JSON](https://mikefarah.gitbook.io/yq/usage/convert) and [XML](https://mikefarah.gitbook.io/yq/usage/xml) files
 - Fully supports multi document yaml files
 - Supports yaml [front matter](https://mikefarah.gitbook.io/yq/usage/front-matter) blocks (e.g. jekyll/assemble)
 - Colorized yaml output
-- [Deeply traverse yaml](https://mikefarah.gitbook.io/yq/operators/traverse-read)
-- [Sort yaml by keys](https://mikefarah.gitbook.io/yq/operators/sort-keys)
+- [Date/Time manipulation and formatting with TZ](https://mikefarah.gitbook.io/yq/operators/datetime)
+- [Deep data structures](https://mikefarah.gitbook.io/yq/operators/traverse-read)
+- [Sort keys](https://mikefarah.gitbook.io/yq/operators/sort-keys)
 - Manipulate yaml [comments](https://mikefarah.gitbook.io/yq/operators/comment-operators), [styling](https://mikefarah.gitbook.io/yq/operators/style), [tags](https://mikefarah.gitbook.io/yq/operators/tag) and [anchors and aliases](https://mikefarah.gitbook.io/yq/operators/anchor-and-alias-operators).
-- [Update yaml inplace](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags)
+- [Update in place](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags)
 - [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/operators/select#select-and-update-matching-values-in-map)
 - Keeps yaml formatting and comments when updating (though there are issues with whitespace)
-- [Convert to/from json](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
-- [Convert to properties](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties)
-- [Pipe data in by using '-'](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate)
+- [Decode/Encode base64 data](https://mikefarah.gitbook.io/yq/operators/encode-decode)
+- [Load content from other files](https://mikefarah.gitbook.io/yq/operators/load)
+- [Convert to/from json/ndjson](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
+- [Convert to/from xml](https://mikefarah.gitbook.io/yq/v/v4.x/usage/xml)
+- [Convert to/from properties](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties)
+- [Convert to/from csv/tsv](https://mikefarah.gitbook.io/yq/usage/csv-tsv)
 - [General shell completion scripts (bash/zsh/fish/powershell)](https://mikefarah.gitbook.io/yq/v/v4.x/commands/shell-completion)
 - [Reduce](https://mikefarah.gitbook.io/yq/operators/reduce) to merge multiple files or sum an array or other fancy things.
 - [Github Action](https://mikefarah.gitbook.io/yq/usage/github-action) to use in your automated pipeline (thanks @devorbitus)
@@ -250,33 +378,101 @@ Usage:
   yq [flags]
   yq [command]
 
+Examples:
+
+# yq tries to auto-detect the file format based off the extension, and defaults to YAML if it's unknown (or piping through STDIN)
+# Use the '-p/--input-format' flag to specify a format type.
+cat file.xml | yq -p xml
+
+# read the "stuff" node from "myfile.yml"
+yq '.stuff' < myfile.yml
+
+# update myfile.yml in place
+yq -i '.stuff = "foo"' myfile.yml
+
+# print contents of sample.json as idiomatic YAML
+yq -P -oy sample.json
+
+
 Available Commands:
-  eval             Apply the expression to each document in each yaml file in sequence
-  eval-all         Loads _all_ yaml documents of _all_ yaml files and runs expression once
-  help             Help about any command
-  shell-completion Generate completion script
+  completion  Generate the autocompletion script for the specified shell
+  eval        (default) Apply the expression to each document in each yaml file in sequence
+  eval-all    Loads _all_ yaml documents of _all_ yaml files and runs expression once
+  help        Help about any command
 
 Flags:
-  -C, --colors                force print with colors
-  -e, --exit-status           set exit status if there are no matches or null or false is returned
-  -f, --front-matter string   (extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact
-      --header-preprocess     Slurp any header comments and seperators before processing expression. This is a workaround for go-yaml to persist header content (default true)
-  -h, --help                  help for yq
-  -I, --indent int            sets indent level for output (default 2)
-  -i, --inplace               update the yaml file inplace of first yaml file given.
-  -M, --no-colors             force print with no colors
-  -N, --no-doc                Don't print document separators (---)
-  -n, --null-input            Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.
-  -o, --output-format string  [yaml|y|json|j|props|p] output format type. (default "yaml")
-  -P, --prettyPrint           pretty print, shorthand for '... style = ""'
-      --unwrapScalar          unwrap scalar, print the value with no quotes, colors or comments (default true)
-  -v, --verbose               verbose mode
-  -V, --version               Print version information and quit
+  -C, --colors                          force print with colors
+      --csv-auto-parse                  parse CSV YAML/JSON values (default true)
+      --csv-separator char              CSV Separator character (default ,)
+      --debug-node-info                 debug node info
+  -e, --exit-status                     set exit status if there are no matches or null or false is returned
+      --expression string               forcibly set the expression argument. Useful when yq argument detection thinks your expression is a file.
+      --from-file string                Load expression from specified file.
+  -f, --front-matter string             (extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact
+      --header-preprocess               Slurp any header comments and separators before processing expression. (default true)
+  -h, --help                            help for yq
+  -I, --indent int                      sets indent level for output (default 2)
+  -i, --inplace                         update the file in place of first file given.
+  -p, --input-format string             [auto|a|yaml|y|json|j|props|p|csv|c|tsv|t|xml|x|base64|uri|toml|lua|l|ini|i] parse format for input. (default "auto")
+      --lua-globals                     output keys as top-level global variables
+      --lua-prefix string               prefix (default "return ")
+      --lua-suffix string               suffix (default ";\n")
+      --lua-unquoted                    output unquoted string keys (e.g. {foo="bar"})
+  -M, --no-colors                       force print with no colors
+  -N, --no-doc                          Don't print document separators (---)
+  -0, --nul-output                      Use NUL char to separate values. If unwrap scalar is also set, fail if unwrapped scalar contains NUL char.
+  -n, --null-input                      Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.
+  -o, --output-format string            [auto|a|yaml|y|json|j|props|p|csv|c|tsv|t|xml|x|base64|uri|toml|shell|s|lua|l|ini|i] output format type. (default "auto")
+  -P, --prettyPrint                     pretty print, shorthand for '... style = ""'
+      --properties-array-brackets       use [x] in array paths (e.g. for SpringBoot)
+      --properties-separator string     separator to use between keys and values (default " = ")
+      --security-disable-env-ops        Disable env related operations.
+      --security-disable-file-ops       Disable file related operations (e.g. load)
+      --shell-key-separator string      separator for shell variable key paths (default "_")
+  -s, --split-exp string                print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter. The necessary directories will be created.
+      --split-exp-file string           Use a file to specify the split-exp expression.
+      --string-interpolation            Toggles strings interpolation of \(exp) (default true)
+      --tsv-auto-parse                  parse TSV YAML/JSON values (default true)
+  -r, --unwrapScalar                    unwrap scalar, print the value with no quotes, colors or comments. Defaults to true for yaml (default true)
+  -v, --verbose                         verbose mode
+  -V, --version                         Print version information and quit
+      --xml-attribute-prefix string     prefix for xml attributes (default "+@")
+      --xml-content-name string         name for xml content (if no attribute name is present). (default "+content")
+      --xml-directive-name string       name for xml directives (e.g. <!DOCTYPE thing cat>) (default "+directive")
+      --xml-keep-namespace              enables keeping namespace after parsing attributes (default true)
+      --xml-proc-inst-prefix string     prefix for xml processing instructions (e.g. <?xml version="1"?>) (default "+p_")
+      --xml-raw-token                   enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details. (default true)
+      --xml-skip-directives             skip over directives (e.g. <!DOCTYPE thing cat>)
+      --xml-skip-proc-inst              skip over process instructions (e.g. <?xml version="1"?>)
+      --xml-strict-mode                 enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.
+      --yaml-fix-merge-anchor-to-spec   Fix merge anchor to match YAML spec. Will default to true in late 2025
 
 Use "yq [command] --help" for more information about a command.
 ```
+
+## Troubleshooting
+
+### Common Issues
+
+**PowerShell quoting issues:**
+```powershell
+# Use single quotes for expressions
+yq '.a.b[0].c' file.yaml
+
+# Or escape double quotes
+yq ".a.b[0].c = \"value\"" file.yaml
+```
+
+### Getting Help
+
+- **Check existing issues**: [GitHub Issues](https://github.com/mikefarah/yq/issues)
+- **Ask questions**: [GitHub Discussions](https://github.com/mikefarah/yq/discussions)
+- **Documentation**: [Complete documentation](https://mikefarah.gitbook.io/yq/)
+- **Examples**: [Recipes and examples](https://mikefarah.gitbook.io/yq/recipes)
+
 ## Known Issues / Missing Features
 - `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details)
-- Powershell has its own...opinions: https://mikefarah.gitbook.io/yq/usage/tips-and-tricks#quotes-in-windows-powershell
+- Powershell has its own...[opinions on quoting yq](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks#quotes-in-windows-powershell)
+- "yes", "no" were dropped as boolean values in the yaml 1.2 standard - which is the standard yq assumes.
 
-See [tips and tricks](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks) for more common problems and solutions.
+See [tips and tricks](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks) for more common problems and solutions.

acceptance_tests/bad_args.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+
+testWriteInPlacePipeIn() {
+  result=$(./yq e -i -n '.a' 2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: write in place flag only applicable when giving an expression and at least one file" "$result"
+}
+
+testWriteInPlacePipeInEvalall() {
+  result=$(./yq ea -i -n '.a' 2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: write in place flag only applicable when giving an expression and at least one file" "$result"
+}
+
+testWriteInPlaceWithSplit() {
+  result=$(./yq e -s "cat" -i '.a = "thing"' test.yml 2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: write in place cannot be used with split file" "$result"
+}
+
+testWriteInPlaceWithSplitEvalAll() {
+  result=$(./yq ea -s "cat" -i '.a = "thing"' test.yml 2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: write in place cannot be used with split file" "$result"
+}
+
+testNullWithFiles() {
+  result=$(./yq e -n '.a = "thing"' test.yml 2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: cannot pass files in when using null-input flag" "$result"
+}
+
+testNullWithFilesEvalAll() {
+  result=$(./yq ea -n '.a = "thing"' test.yml 2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: cannot pass files in when using null-input flag" "$result"
+}
+
+
+
+source ./scripts/shunit2

acceptance_tests/basic.sh

@@ -1,12 +1,236 @@
 #!/bin/bash
 
 setUp() {
-  rm -f test.yml
+  rm test*.yml 2>/dev/null || true
+  rm .xyz 2>/dev/null || true
+  rm instructions.txt 2>/dev/null || true
 }
 
 testBasicEvalRoundTrip() {
+  ./yq -n ".a = 123" > test.yml
+  X=$(./yq '.a' test.yml)
+  assertEquals 123 "$X"
+}
+
+testBasicTrailingContent() {
+  cat >test-trailing.yml <<EOL
+test:
+# this comment will be removed
+EOL
+
+  read -r -d '' expected << EOM
+test:
+# this comment will be removed
+EOM
+  X=$(./yq test-trailing.yml -P)
+  assertEquals "$expected" "$X"
+}
+
+testBasicTrailingContent() {
+  cat >test-trailing.yml <<EOL
+test:
+# this comment will be removed
+EOL
+
+  read -r -d '' expected << EOM
+test:
+# hi
+EOM
+  X=$(./yq '. footComment = "hi"' test-trailing.yml)
+  assertEquals "$expected" "$X"
+}
+
+testBasicTrailingContentEvalAll() {
+  cat >test-trailing.yml <<EOL
+test:
+# this comment will be removed
+EOL
+
+  read -r -d '' expected << EOM
+test:
+# this comment will be removed
+EOM
+  X=$(./yq ea test-trailing.yml -P)
+  assertEquals "$expected" "$X"
+}
+
+testBasicTrailingContentEvalAll() {
+  cat >test-trailing.yml <<EOL
+test:
+# this comment will be removed
+EOL
+
+  read -r -d '' expected << EOM
+test:
+# hi
+EOM
+  X=$(./yq ea '. footComment = "hi"' test-trailing.yml)
+  assertEquals "$expected" "$X"
+}
+
+testBasicPipeWithDot() {
+  ./yq -n ".a = 123" > test.yml
+  X=$(cat test.yml | ./yq '.')
+  assertEquals "a: 123" "$X"
+}
+
+testBasicExpressionMatchesFileName() {
+  ./yq -n ".xyz = 123" > test.yml
+  touch .xyz
+  
+  X=$(./yq --expression '.xyz' test.yml)
+  assertEquals "123" "$X"
+
+  X=$(./yq ea --expression '.xyz' test.yml)
+  assertEquals "123" "$X"
+}
+
+testBasicExpressionFromFile() {
+  ./yq -n ".xyz = 123" > test.yml
+  echo '.xyz = "meow" | .cool = "frog"' > instructions.txt
+  
+  X=$(./yq --from-file instructions.txt test.yml -o=j -I=0)
+  assertEquals '{"xyz":"meow","cool":"frog"}' "$X"
+
+  X=$(./yq ea --from-file instructions.txt test.yml -o=j -I=0)
+  assertEquals '{"xyz":"meow","cool":"frog"}' "$X"
+}
+
+testBasicExpressionFromFileDos() {
+  ./yq -n ".xyz = 123" > test.yml
+  echo '.xyz = "meow" | .cool = "frog"' | sed 's/$'"/`echo \\\r`/" > instructions.txt
+  
+  X=$(./yq --from-file instructions.txt test.yml -o=j -I=0)
+  assertEquals '{"xyz":"meow","cool":"frog"}' "$X"
+
+  X=$(./yq ea --from-file instructions.txt test.yml -o=j -I=0)
+  assertEquals '{"xyz":"meow","cool":"frog"}' "$X"
+}
+
+testBasicGitHubAction() {
+  ./yq -n ".a = 123" > test.yml
+  X=$(cat /dev/null | ./yq test.yml)
+  assertEquals "a: 123" "$X"
+
+  X=$(cat /dev/null | ./yq e test.yml)
+  assertEquals "a: 123" "$X"
+
+  X=$(cat /dev/null | ./yq ea test.yml)
+  assertEquals "a: 123" "$X"
+}
+
+testBasicGitHubActionWithExpression() {
+  ./yq -n ".a = 123" > test.yml
+  X=$(cat /dev/null | ./yq '.a' test.yml)
+  assertEquals "123" "$X"
+
+  X=$(cat /dev/null | ./yq e '.a' test.yml)
+  assertEquals "123" "$X"
+
+  X=$(cat /dev/null | ./yq ea '.a' test.yml)
+  assertEquals "123" "$X"
+}
+
+
+testBasicEvalAllAllFiles() {
+  ./yq -n ".a = 123" > test.yml
+  ./yq -n ".a = 124" > test2.yml
+  X=$(./yq ea test.yml test2.yml)
+  Y=$(./yq e '.' test.yml test2.yml)
+  assertEquals "$Y" "$X"
+}
+
+# when given a file, don't read STDIN
+# otherwise strange things start happening
+# in scripts
+# https://github.com/mikefarah/yq/issues/1115
+
+testBasicCatWithFilesNoDash() {
+  ./yq -n ".a = 123" > test.yml
+  ./yq -n ".a = 124" > test2.yml
+  X=$(cat test.yml | ./yq test2.yml)
+  Y=$(./yq e '.' test2.yml)
+  assertEquals "$Y" "$X"
+}
+
+# when the nullinput flag is used
+# don't automatically read STDIN (this breaks github actions)
+testBasicCreateFileGithubAction() {
+  cat /dev/null | ./yq -n ".a = 123" > test.yml
+}
+
+testBasicEvalAllCatWithFilesNoDash() {
+  ./yq -n ".a = 123" > test.yml
+  ./yq -n ".a = 124" > test2.yml
+  X=$(cat test.yml | ./yq ea test2.yml)
+  Y=$(./yq e '.' test2.yml)
+  assertEquals "$Y" "$X"
+}
+
+testBasicCatWithFilesNoDashWithExp() {
+  ./yq -n ".a = 123" > test.yml
+  ./yq -n ".a = 124" > test2.yml
+  X=$(cat test.yml | ./yq '.a' test2.yml)
+  Y=$(./yq e '.a' test2.yml)
+  assertEquals "$Y" "$X"
+}
+
+testBasicEvalAllCatWithFilesNoDashWithExp() {
+  ./yq -n ".a = 123" > test.yml
+  ./yq -n ".a = 124" > test2.yml
+  X=$(cat test.yml | ./yq ea '.a' test2.yml)
+  Y=$(./yq e '.a' test2.yml)
+  assertEquals "$Y" "$X"
+}
+
+
+testBasicStdInWithFiles() {
+  ./yq -n ".a = 123" > test.yml
+  ./yq -n ".a = 124" > test2.yml
+  X=$(cat test.yml | ./yq - test2.yml)
+  Y=$(./yq e '.' test.yml test2.yml)
+  assertEquals "$Y" "$X"
+}
+
+testBasicEvalAllStdInWithFiles() {
+  ./yq -n ".a = 123" > test.yml
+  ./yq -n ".a = 124" > test2.yml
+  X=$(cat test.yml | ./yq ea - test2.yml)
+  Y=$(./yq e '.' test.yml test2.yml)
+  assertEquals "$Y" "$X"
+}
+
+testBasicStdInWithFilesReverse() {
+  ./yq -n ".a = 123" > test.yml
+  ./yq -n ".a = 124" > test2.yml
+  X=$(cat test.yml | ./yq test2.yml -)
+  Y=$(./yq e '.' test2.yml test.yml)
+  assertEquals "$Y" "$X"
+}
+
+testBasicEvalAllStdInWithFilesReverse() {
+  ./yq -n ".a = 123" > test.yml
+  ./yq -n ".a = 124" > test2.yml
+  X=$(cat test.yml | ./yq ea test2.yml -)
+  Y=$(./yq e '.' test2.yml test.yml)
+  assertEquals "$Y" "$X"
+}
+
+testBasicEvalRoundTripNoEval() {
+  ./yq -n ".a = 123" > test.yml
+  X=$(./yq '.a' test.yml)
+  assertEquals 123 "$X"
+}
+
+testBasicStdInWithOneArg() {
   ./yq e -n ".a = 123" > test.yml
-  X=$(./yq e '.a' test.yml)
+  X=$(cat test.yml | ./yq e ".a")
+  assertEquals 123 "$X"
+
+  X=$(cat test.yml | ./yq ea ".a")
+  assertEquals 123 "$X"
+
+  X=$(cat test.yml | ./yq ".a")
   assertEquals 123 "$X"
 }
 
@@ -19,6 +243,15 @@ EOL
   assertEquals "10" "$X"
 }
 
+testBasicUpdateInPlaceSequenceNoEval() {
+  cat >test.yml <<EOL
+a: 0
+EOL
+  ./yq -i ".a = 10" test.yml
+  X=$(./yq '.a' test.yml)
+  assertEquals "10" "$X"
+}
+
 testBasicUpdateInPlaceSequenceEvalAll() {
   cat >test.yml <<EOL
 a: 0
@@ -28,6 +261,40 @@ EOL
   assertEquals "10" "$X"
 }
 
+testBasicUpdateInPlaceMultipleFilesNoExpressionEval() {
+  cat >test.yml <<EOL
+a: 0
+EOL
+  cat >test2.yml <<EOL
+a: 1
+EOL
+read -r -d '' expected << EOM
+0
+---
+1
+EOM
+  ./yq -i test.yml test2.yml
+  X=$(./yq e '.a' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testBasicUpdateInPlaceMultipleFilesNoExpressionEvalAll() {
+  cat >test.yml <<EOL
+a: 0
+EOL
+  cat >test2.yml <<EOL
+a: 1
+EOL
+read -r -d '' expected << EOM
+0
+---
+1
+EOM
+  ./yq -i ea test.yml test2.yml
+  X=$(./yq e '.a' test.yml)
+  assertEquals "$expected" "$X"
+}
+
 testBasicNoExitStatus() {
   echo "a: cat" > test.yml
   X=$(./yq e '.z' test.yml)
@@ -40,7 +307,13 @@ testBasicExitStatus() {
   assertEquals 1 "$?"
 }
 
-testBasicExtractFieldWithSeperator() {
+testBasicExitStatusNoEval() {
+  echo "a: cat" > test.yml
+  X=$(./yq -e '.z' test.yml 2&>/dev/null)
+  assertEquals 1 "$?"
+}
+
+testBasicExtractFieldWithSeparator() {
     cat >test.yml <<EOL
 ---
 name: chart-name
@@ -50,7 +323,7 @@ EOL
   assertEquals "chart-name" "$X"
 }
 
-testBasicExtractMultipleFieldWithSeperator() {
+testBasicExtractMultipleFieldWithSeparator() {
     cat >test.yml <<EOL
 ---
 name: chart-name
@@ -69,4 +342,32 @@ EOM
   assertEquals "$expected" "$X"
 }
 
+testBasicMultiplyAssignMultiDoc() {
+      cat >test.yml <<EOL
+a: 1
+---
+b: 2
+EOL
+
+read -r -d '' expected << EOM
+a: 1
+c: 3
+---
+b: 2
+c: 3
+EOM
+
+
+  X=$(./yq '. *= {"c":3}' test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testBasicClosedStdIn() {
+  cat >test.yml <<EOL
+a: 1
+EOL
+  X=$(./yq e '.a' test.yml <&-)
+  assertEquals "1" "$X"
+}
+
 source ./scripts/shunit2

acceptance_tests/completion.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+testCompletionRuns() {
+    result=$(./yq __complete "" 2>&1)
+    assertEquals 0 $?
+    assertContains "$result" "Completion ended with directive:"
+}
+
+source ./scripts/shunit2

acceptance_tests/empty.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 setUp() {
+  rm test*.yml || true
   cat >test.yml <<EOL
 # comment
 EOL
@@ -8,7 +9,7 @@ EOL
 
 testEmptyEval() {
   X=$(./yq e test.yml)
-  expected=$(cat test.yml)
+  expected="# comment"
   assertEquals 0 $?
   assertEquals "$expected" "$X"
 }

acceptance_tests/flags.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+setUp() {
+  rm test*.yml || true
+  cat >test.yml <<EOL
+# comment
+EOL
+}
+
+testStringInterpolation() {
+    X=$(./yq -n '"Mike \(3 + 4)"')
+    assertEquals "Mike 7" "$X"
+}
+
+testNoStringInterpolation() {
+    X=$(./yq --string-interpolation=f -n '"Mike \(3 + 4)"')
+    assertEquals "Mike \(3 + 4)" "$X"
+}
+
+
+source ./scripts/shunit2

acceptance_tests/front-matter.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 
 setUp() {
+  rm test*.yml || true
   cat >test.yml <<EOL
 ---
 a: apple

acceptance_tests/header-processing-off.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+setUp() {
+  rm test*.yml || true
+  
+}
+
+testLineCountFirstLineComment() {
+  cat >test.yml <<EOL
+#test123 
+abc: 123 
+test123: 123123 
+#comment 
+lalilu: lalilu
+EOL
+
+  X=$(./yq '.lalilu | line' --header-preprocess=false < test.yml)
+  assertEquals "5" "$X"
+}
+
+testArrayOfDocs() {
+  cat >test.yml <<EOL
+---
+# leading comment doc 1
+a: 1
+---
+# leading comment doc 2
+a: 2
+EOL
+
+read -r -d '' expected << EOM
+- # leading comment doc 1
+  a: 1
+- # leading comment doc 2
+  a: 2
+EOM
+
+  X=$(./yq ea '[.]' --header-preprocess=false < test.yml)
+  assertEquals "$expected" "$X"
+
+}
+
+source ./scripts/shunit2

acceptance_tests/inputs-format-auto.sh

@@ -0,0 +1,249 @@
+#!/bin/bash
+
+setUp() {
+  rm test*.yml 2>/dev/null || true
+  rm test*.toml 2>/dev/null || true
+  rm test*.tfstate 2>/dev/null || true
+  rm test*.json 2>/dev/null || true
+  rm test*.properties 2>/dev/null || true
+  rm test*.csv 2>/dev/null || true
+  rm test*.tsv 2>/dev/null || true
+  rm test*.xml 2>/dev/null || true
+}
+
+testInputJson() {
+  cat >test.json <<EOL
+{ "mike" : { "things": "cool" } }
+EOL
+
+  read -r -d '' expected << EOM
+{
+  "mike": {
+    "things": "cool"
+  }
+}
+EOM
+
+  X=$(./yq test.json)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea test.json)
+  assertEquals "$expected" "$X"
+}
+
+testInputToml() {
+  cat >test.toml <<EOL
+[owner]
+name = "Tom Preston-Werner"
+dob = 1979-05-27T07:32:00-08:00
+EOL
+
+  read -r -d '' expected << EOM
+owner:
+  name: Tom Preston-Werner
+  dob: 1979-05-27T07:32:00-08:00
+EOM
+
+  X=$(./yq -oy test.toml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -oy test.toml)
+  assertEquals "$expected" "$X"
+}
+
+testInputTfstate() {
+  cat >test.tfstate <<EOL
+{ "mike" : { "things": "cool" } }
+EOL
+
+  read -r -d '' expected << EOM
+{"mike": {"things": "cool"}}
+EOM
+
+  X=$(./yq test.tfstate)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea test.tfstate)
+  assertEquals "$expected" "$X"
+}
+
+testInputJsonOutputYaml() {
+  cat >test.json <<EOL
+{ "mike" : { "things": "cool" } }
+EOL
+
+  read -r -d '' expected << EOM
+mike:
+  things: cool
+EOM
+
+  X=$(./yq test.json -oy)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea test.json -oy)
+  assertEquals "$expected" "$X"
+}
+
+testInputProperties() {
+  cat >test.properties <<EOL
+mike.things = hello
+EOL
+
+  read -r -d '' expected << EOM
+mike.things = hello
+EOM
+
+  X=$(./yq e test.properties)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq test.properties)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea test.properties)
+  assertEquals "$expected" "$X"
+}
+
+testInputPropertiesGitHubAction() {
+  cat >test.properties <<EOL
+mike.things = hello
+EOL
+
+  read -r -d '' expected << EOM
+mike.things = hello
+EOM
+
+  X=$(cat /dev/null | ./yq e test.properties)
+  assertEquals "$expected" "$X"
+
+  X=$(cat /dev/null | ./yq ea test.properties)
+  assertEquals "$expected" "$X"
+}
+
+testInputCSV() {
+  cat >test.csv <<EOL
+fruit,yumLevel
+apple,5
+banana,4
+EOL
+
+  read -r -d '' expected << EOM
+fruit,yumLevel
+apple,5
+banana,4
+EOM
+
+  X=$(./yq e test.csv)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea test.csv)
+  assertEquals "$expected" "$X"
+}
+
+testInputCSVUTF8() {
+  read -r -d '' expected << EOM
+id,first,last
+1,john,smith
+1,jane,smith
+EOM
+
+  X=$(./yq utf8.csv)
+  assertEquals "$expected" "$X"
+}
+
+testInputTSV() {
+  cat >test.tsv <<EOL
+fruit	yumLevel
+apple	5
+banana	4
+EOL
+
+  read -r -d '' expected << EOM
+fruit	yumLevel
+apple	5
+banana	4
+EOM
+
+  X=$(./yq e test.tsv)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea test.tsv)
+  assertEquals "$expected" "$X"
+}
+
+
+
+
+testInputXml() {
+  cat >test.xml <<EOL
+<cat legs="4">BiBi</cat>
+EOL
+
+  read -r -d '' expected << EOM
+<cat legs="4">BiBi</cat>
+EOM
+
+  X=$(./yq e test.xml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea test.xml)
+  assertEquals "$expected" "$X"
+}
+
+testInputXmlNamespaces() {
+  cat >test.xml <<EOL
+<?xml version="1.0"?>
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
+</map>
+EOL
+
+  read -r -d '' expected << EOM
+<?xml version="1.0"?>
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url"></map>
+EOM
+
+  X=$(./yq e test.xml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea test.xml)
+  assertEquals "$expected" "$X"
+}
+
+
+
+testInputXmlStrict() {
+  cat >test.xml <<EOL
+<?xml version="1.0"?>
+<!DOCTYPE root [
+<!ENTITY writer "Catherine.">
+<!ENTITY copyright "(r) Great">
+]>
+<root>
+    <item>&writer;&copyright;</item>
+</root>
+EOL
+
+  X=$(./yq --xml-strict-mode test.xml  2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X"
+
+  X=$(./yq ea --xml-strict-mode test.xml  2>&1)
+  assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X"
+}
+
+testInputXmlGithubAction() {
+  cat >test.xml <<EOL
+<cat legs="4">BiBi</cat>
+EOL
+
+  read -r -d '' expected << EOM
+<cat legs="4">BiBi</cat>
+EOM
+
+  X=$(cat /dev/null | ./yq e test.xml)
+  assertEquals "$expected" "$X"
+
+  X=$(cat /dev/null | ./yq ea test.xml)
+  assertEquals "$expected" "$X"
+}
+
+source ./scripts/shunit2

acceptance_tests/inputs-format.sh

@@ -0,0 +1,258 @@
+#!/bin/bash
+
+setUp() {
+  rm test*.yml 2>/dev/null || true
+  rm test*.properties 2>/dev/null || true
+  rm test*.csv 2>/dev/null || true
+  rm test*.tsv 2>/dev/null || true
+  rm test*.xml 2>/dev/null || true
+}
+
+testInputProperties() {
+  cat >test.properties <<EOL
+mike.things = hello
+EOL
+
+  read -r -d '' expected << EOM
+mike:
+  things: hello
+EOM
+
+  X=$(./yq e -p=props test.properties)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -p=props test.properties)
+  assertEquals "$expected" "$X"
+}
+
+testInputPropertiesGitHubAction() {
+  cat >test.properties <<EOL
+mike.things = hello
+EOL
+
+  read -r -d '' expected << EOM
+mike:
+  things: hello
+EOM
+
+  X=$(cat /dev/null | ./yq e -p=props test.properties)
+  assertEquals "$expected" "$X"
+
+  X=$(cat /dev/null | ./yq ea -p=props test.properties)
+  assertEquals "$expected" "$X"
+}
+
+testInputCSV() {
+  cat >test.csv <<EOL
+fruit,yumLevel
+apple,5
+banana,4
+EOL
+
+  read -r -d '' expected << EOM
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOM
+
+  X=$(./yq e -p=csv test.csv)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -p=csv test.csv)
+  assertEquals "$expected" "$X"
+}
+
+testInputCSVCustomSeparator() {
+  cat >test.csv <<EOL
+fruit;yumLevel
+apple;5
+banana;4
+EOL
+
+  read -r -d '' expected << EOM
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOM
+
+  X=$(./yq -p=csv --csv-separator ";" test.csv)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -p=csv --csv-separator ";" test.csv)
+  assertEquals "$expected" "$X"
+}
+
+testInputCSVNoAuto() {
+  cat >test.csv <<EOL
+thing1
+name: cat
+EOL
+
+  read -r -d '' expected << EOM
+- thing1: 'name: cat'
+EOM
+
+  X=$(./yq --csv-auto-parse=f test.csv -oy)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --csv-auto-parse=f test.csv -oy)
+  assertEquals "$expected" "$X"
+}
+
+testInputTSVNoAuto() {
+  cat >test.tsv <<EOL
+thing1
+name: cat
+EOL
+
+  read -r -d '' expected << EOM
+- thing1: 'name: cat'
+EOM
+
+  X=$(./yq --tsv-auto-parse=f test.tsv -oy)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --tsv-auto-parse=f test.tsv -oy)
+  assertEquals "$expected" "$X"
+}
+
+testInputCSVUTF8() {
+  read -r -d '' expected << EOM
+- id: 1
+  first: john
+  last: smith
+- id: 1
+  first: jane
+  last: smith
+EOM
+
+  X=$(./yq -p=csv utf8.csv)
+  assertEquals "$expected" "$X"
+}
+
+testInputTSV() {
+  cat >test.tsv <<EOL
+fruit	yumLevel
+apple	5
+banana	4
+EOL
+
+  read -r -d '' expected << EOM
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOM
+
+  X=$(./yq e -p=t test.tsv)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -p=t test.tsv)
+  assertEquals "$expected" "$X"
+}
+
+
+
+
+testInputXml() {
+  cat >test.yml <<EOL
+<cat legs="4">BiBi</cat>
+EOL
+
+  read -r -d '' expected << EOM
+cat:
+  +content: BiBi
+  +@legs: "4"
+EOM
+
+  X=$(./yq e -p=xml test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -p=xml test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testInputXmlNamespaces() {
+  cat >test.xml <<EOL
+<?xml version="1.0"?>
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
+</map>
+EOL
+
+  read -r -d '' expected << EOM
++p_xml: version="1.0"
+map:
+  +@xmlns: some-namespace
+  +@xmlns:xsi: some-instance
+  +@xsi:schemaLocation: some-url
+EOM
+
+  X=$(./yq e -p=xml test.xml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -p=xml test.xml)
+  assertEquals "$expected" "$X"
+}
+
+testInputXmlRoundtrip() {
+  cat >test.yml <<EOL
+<?xml version="1.0"?>
+<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">Meow</map>
+EOL
+
+  read -r -d '' expected << EOM
+<?xml version="1.0"?>
+<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">Meow</map>
+EOM
+
+  X=$(./yq -p=xml -o=xml test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -p=xml -o=xml test.yml)
+  assertEquals "$expected" "$X"
+}
+
+
+testInputXmlStrict() {
+  cat >test.yml <<EOL
+<?xml version="1.0"?>
+<!DOCTYPE root [
+<!ENTITY writer "Catherine.">
+<!ENTITY copyright "(r) Great">
+]>
+<root>
+    <item>&writer;&copyright;</item>
+</root>
+EOL
+
+  X=$(./yq -p=xml --xml-strict-mode test.yml -o=xml 2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: bad file 'test.yml': XML syntax error on line 7: invalid character entity &writer;" "$X"
+
+  X=$(./yq ea -p=xml --xml-strict-mode test.yml -o=xml 2>&1)
+  assertEquals "Error: bad file 'test.yml': XML syntax error on line 7: invalid character entity &writer;" "$X"
+}
+
+testInputXmlGithubAction() {
+  cat >test.yml <<EOL
+<cat legs="4">BiBi</cat>
+EOL
+
+  read -r -d '' expected << EOM
+cat:
+  +content: BiBi
+  +@legs: "4"
+EOM
+
+  X=$(cat /dev/null | ./yq e -p=xml test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(cat /dev/null | ./yq ea -p=xml test.yml)
+  assertEquals "$expected" "$X"
+}
+
+source ./scripts/shunit2

acceptance_tests/leading-separator.sh

@@ -1,13 +1,17 @@
 #!/bin/bash
 
+
+# examples where header-preprocess is required
+
 setUp() {
+  rm test*.yml || true
   cat >test.yml <<EOL
 ---
 a: test
 EOL
 }
 
-testLeadingSeperatorWithDoc() {
+testLeadingSeparatorWithDoc() {
   cat >test.yml <<EOL
 # hi peeps
 # cool
@@ -30,18 +34,94 @@ EOM
   assertEquals "$expected" "$X"
 }
 
-testLeadingSeperatorPipeIntoEvalSeq() {
+
+testLeadingSeparatorWithNewlinesNewDoc() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+
+
+---
+a: test
+---
+b: cool
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+
+
+---
+a: thing
+---
+b: cool
+EOM
+
+  X=$(./yq e '(select(di == 0) | .a) = "thing"' - < test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLeadingSeparatorWithNewlinesMoreComments() {
+  cat >test.yml <<EOL
+# hi peeps
+# cool
+
+---
+# great
+
+a: test
+---
+b: cool
+EOL
+
+  read -r -d '' expected << EOM
+# hi peeps
+# cool
+
+---
+# great
+
+a: thing
+---
+b: cool
+EOM
+
+  X=$(./yq e '(select(di == 0) | .a) = "thing"' - < test.yml)
+  assertEquals "$expected" "$X"
+}
+
+
+testLeadingSeparatorWithDirective() {
+  cat >test.yml <<EOL
+%YAML 1.1
+---
+this: should really work
+EOL
+
+  read -r -d '' expected << EOM
+%YAML 1.1
+---
+this: should really work
+EOM
+
+  X=$(./yq < test.yml)
+  assertEquals "$expected" "$X"
+}
+
+
+testLeadingSeparatorPipeIntoEvalSeq() {
   X=$(./yq e - < test.yml)
   expected=$(cat test.yml)
   assertEquals "$expected" "$X"
 }
 
-testLeadingSeperatorExtractField() {
+testLeadingSeparatorExtractField() {
   X=$(./yq e '.a' - < test.yml)
   assertEquals "test" "$X"
 }
 
-testLeadingSeperatorExtractFieldWithCommentsAfterSep() {
+testLeadingSeparatorExtractFieldWithCommentsAfterSep() {
   cat >test.yml <<EOL
 ---
 # hi peeps
@@ -52,7 +132,7 @@ EOL
   assertEquals "test" "$X"
 }
 
-testLeadingSeperatorExtractFieldWithCommentsBeforeSep() {
+testLeadingSeparatorExtractFieldWithCommentsBeforeSep() {
   cat >test.yml <<EOL
 # hi peeps
 # cool
@@ -64,7 +144,7 @@ EOL
 }
 
 
-testLeadingSeperatorExtractFieldMultiDoc() {
+testLeadingSeparatorExtractFieldMultiDoc() {
   cat >test.yml <<EOL
 ---
 a: test
@@ -81,13 +161,13 @@ EOM
   assertEquals "$expected" "$X"
 }
 
-testLeadingSeperatorExtractFieldMultiDocWithComments() {
+testLeadingSeparatorExtractFieldMultiDocWithComments() {
   cat >test.yml <<EOL
 # here
 ---
 # there
 a: test
-# whereever
+# wherever
 ---
 # you are
 a: test2
@@ -104,26 +184,26 @@ EOM
 }
 
 
-testLeadingSeperatorEvalSeq() {
+testLeadingSeparatorEvalSeq() {
   X=$(./yq e test.yml)
   expected=$(cat test.yml)
   assertEquals "$expected" "$X"
 }
 
-testLeadingSeperatorPipeIntoEvalAll() {
+testLeadingSeparatorPipeIntoEvalAll() {
   X=$(./yq ea - < test.yml)
   expected=$(cat test.yml)
   assertEquals "$expected" "$X"
 }
 
 
-testLeadingSeperatorEvalAll() {
+testLeadingSeparatorEvalAll() {
   X=$(./yq ea test.yml)
   expected=$(cat test.yml)
   assertEquals "$expected" "$X"
 }
 
-testLeadingSeperatorMultiDocEvalSimple() {
+testLeadingSeparatorMultiDocEvalSimple() {
   read -r -d '' expected << EOM
 ---
 a: test
@@ -137,7 +217,7 @@ EOM
   assertEquals "$expected" "$X"
 }
 
-testLeadingSeperatorMultiDocInOneFile() {
+testLeadingSeparatorMultiDocInOneFile() {
   cat >test.yml <<EOL
 ---
 # hi peeps
@@ -151,7 +231,7 @@ EOL
   assertEquals "$expected" "$X"
 }
 
-testLeadingSeperatorMultiDocInOneFileEvalAll() {
+testLeadingSeparatorMultiDocInOneFileEvalAll() {
   cat >test.yml <<EOL
 ---
 # hi peeps
@@ -165,7 +245,7 @@ EOL
   assertEquals "$expected" "$X"
 }
 
-testLeadingSeperatorMultiDocEvalComments() {
+testLeadingSeparatorMultiDocEvalComments() {
   cat >test.yml <<EOL
 # hi peeps
 # cool
@@ -193,7 +273,7 @@ EOM
   assertEquals "$expected" "$X"
 }
 
-testLeadingSeperatorMultiDocEvalCommentsTrailingSep() {
+testLeadingSeparatorMultiDocEvalCommentsTrailingSep() {
   cat >test.yml <<EOL
 # hi peeps
 # cool
@@ -225,7 +305,7 @@ EOM
   assertEquals "$expected" "$X"
 }
 
-testLeadingSeperatorMultiMultiDocEvalCommentsTrailingSep() {
+testLeadingSeparatorMultiMultiDocEvalCommentsTrailingSep() {
   cat >test.yml <<EOL
 # hi peeps
 # cool
@@ -265,7 +345,7 @@ EOM
   assertEquals "$expected" "$X"
 }
 
-testLeadingSeperatorMultiDocEvalCommentsLeadingSep() {
+testLeadingSeparatorMultiDocEvalCommentsLeadingSep() {
   cat >test.yml <<EOL
 ---
 # hi peeps
@@ -325,7 +405,7 @@ EOM
   assertEquals "$expected" "$X"
 }
 
-testLeadingSeperatorMultiDocEvalCommentsStripComments() {
+testLeadingSeparatorMultiDocEvalCommentsStripComments() {
   cat >test.yml <<EOL
 ---
 # hi peeps
@@ -348,7 +428,7 @@ EOM
   assertEquals "$expected" "$X"
 }
 
-testLeadingSeperatorMultiDocEvalCommentsLeadingSepNoDocFlag() {
+testLeadingSeparatorMultiDocEvalCommentsLeadingSepNoDocFlag() {
   cat >test.yml <<EOL
 ---
 # hi peeps
@@ -374,7 +454,7 @@ EOM
   assertEquals "$expected" "$X"
 }
 
-testLeadingSeperatorMultiDocEvalJsonFlag() {
+testLeadingSeparatorMultiDocEvalJsonFlag() {
   cat >test.yml <<EOL
 ---
 # hi peeps
@@ -403,7 +483,7 @@ EOM
   assertEquals "$expected" "$X"
 }
 
-testLeadingSeperatorMultiDocEvalAllJsonFlag() {
+testLeadingSeparatorMultiDocEvalAllJsonFlag() {
   cat >test.yml <<EOL
 ---
 # hi peeps
@@ -432,7 +512,7 @@ EOM
   assertEquals "$expected" "$X"
 }
 
-testLeadingSeperatorMultiDocEvalAll() {
+testLeadingSeparatorMultiDocEvalAll() {
   read -r -d '' expected << EOM
 ---
 a: test

acceptance_tests/load-file.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+
+testLoadFileNotExist() {
+  result=$(./yq e -n 'load("cat.yml")' 2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: failed to load cat.yml: open cat.yml: no such file or directory" "$result"
+}
+
+testLoadFileExpNotExist() {
+  result=$(./yq e -n 'load(.a)' 2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: filename expression returned nil" "$result"
+}
+
+testStrLoadFileNotExist() {
+  result=$(./yq e -n 'strload("cat.yml")' 2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: failed to load cat.yml: open cat.yml: no such file or directory" "$result"
+}
+
+testStrLoadFileExpNotExist() {
+  result=$(./yq e -n 'strload(.a)' 2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: filename expression returned nil" "$result"
+}
+
+source ./scripts/shunit2

acceptance_tests/nul-separator.sh

@@ -0,0 +1,286 @@
+#!/bin/bash
+
+setUp() {
+    rm test*.yml || true
+}
+
+## Convenient bash shortcut to read records of NUL separated values
+## from stdin the safe way. See example usage in the next tests.
+read-0() {
+    local eof="" IFS=''
+    while [ "$1" ]; do
+        ## - The `-r` avoids bad surprise with '\n' and other interpreted
+        ##   sequences that can be read.
+        ## - The `-d ''` is the (strange?) way to refer to NUL delimiter.
+        ## - The `--` is how to avoid unpleasant surprises if your
+        ##   "$1" starts with "-" (minus) sign. This protection also
+        ##   will produce a readable error if you want to try to start
+        ##   your variable names with a "-".
+        read -r -d '' -- "$1" || eof=1
+        shift
+    done
+    [ -z "$eof" ] ## fail on EOF
+}
+
+## Convenient bash shortcut to be used with the next function `p-err`
+## to read NUL separated values the safe way AND catch any errors from
+## the process creating the stream of NUL separated data.  See example
+## usage in the tests.
+read-0-err() {
+    local ret="$1" eof="" idx=0 last=
+    read -r -- "${ret?}" <<<"0"
+    shift
+    while [ "$1" ]; do
+        last=$idx
+        read -r -d '' -- "$1" || {
+            ## Put this last value in ${!ret}
+            eof="$1"
+            read -r -- "$ret" <<<"${!eof}"
+            break
+        }
+        ((idx++))
+        shift
+    done
+    [ -z "$eof" ] || {
+        if [ "$last" != 0 ]; then
+            ## Uhoh, we have no idea if the errorlevel of the internal
+            ## command was properly delimited with a NUL char, and
+            ## anyway something went really wrong at least about the
+            ## number of fields separated by NUL char and the one
+            ## expected.
+            echo "Error: read-0-err couldn't fill all value $ret = '${!ret}', '$eof', '${!eof}'" >&2
+            read -r -- "$ret" <<<"not-enough-values"
+        else
+            if ! [[ "${!ret}" =~ ^[0-9]+$ && "${!ret}" -ge 0 && "${!ret}" -le 127 ]]; then
+                ## This could happen if you don't use `p-err` wrapper,
+                ## or used stdout in unexpected ways in your inner
+                ## command.
+                echo "Error: last value is not a number, did you finish with an errorlevel ?" >&2
+                read -r -- "$ret" <<<"last-value-not-a-number"
+            fi
+        fi
+        false
+    }
+}
+
+## Simply runs command given as argument and adds errorlevel in the
+## standard output. Is expected to be used in tandem with
+## `read-0-err`.
+p-err() {
+    local exp="$1"
+    "$@"
+    printf "%s" "$?"
+}
+
+wyq-r() {
+    local exp="$1"
+    ./yq e -0 -r=false "$1"
+    printf "%s" "$?"
+}
+
+testBasicUsageRaw() {
+  cat >test.yml <<EOL
+a: foo
+b: bar
+EOL
+
+  printf "foo\0bar\0" > expected.out
+
+  ## We need to compare binary content here. We have to filter the compared
+  ## content through a representation that gets rid of NUL chars but accurately
+  ## transcribe the content.
+  ## Also as it would be nice to have a pretty output in case the test fails,
+  ## we use here 'hd': a widely available shortcut to 'hexdump' that will
+  ## pretty-print any binary to it's hexadecimal representation.
+  ##
+  ## Note that the standard `assertEquals` compare its arguments
+  ## value, but they can't hold NUL characters (this comes from the
+  ## limitation of the C API of `exec*(..)` functions that requires
+  ## `const char *arv[]`). And these are NUL terminated strings.  As a
+  ## consequence, the NUL characters gets removed in bash arguments.
+  assertEquals "$(hd expected.out)" \
+               "$(./yq e -0 '.a, .b' test.yml | hd)"
+
+  rm expected.out
+}
+
+testBasicUsage() {
+  local a b
+  cat >test.yml <<EOL
+a: foo
+b: bar
+EOL
+
+  ## We provide 2 values, and ask to fill 2 variables.
+  read-0 a b < <(./yq e -0 '.a, .b' test.yml)
+  assertEquals "$?" "0"      ## Everything is fine
+  assertEquals "foo" "$a"    ## Values are correctly parsed
+  assertEquals "bar" "$b"
+
+  a=YYY ; b=XXX
+  ## Not enough values provided to fill `a` and `b`.
+  read-0 a b < <(./yq e -0 '.a' test.yml)
+  assertEquals "$?" "1"      ## An error was emitted
+  assertEquals "foo" "$a"    ## First value was correctly parsed
+  assertEquals "" "$b"       ## Second was still reset
+
+  ## Error from inner command are not catchable !. Use
+  ## `read-0-err`/`p-err` for that.
+  read-0 a < <(printf "\0"; ./yq e -0 'xxx' test.yml; )
+  assertEquals "$?" "0"
+
+}
+
+testBasicUsageJson() {
+  cat >test.yml <<EOL
+a:
+  x: foo
+b: bar
+EOL
+
+  read-0 a b < <(./yq e -0 -o=json '.a, .b' test.yml)
+
+  assertEquals '{
+  "x": "foo"
+}' "$a"
+  assertEquals '"bar"' "$b"
+
+}
+
+testFailWithValueContainingNUL() {
+  local a b c
+  ## Note that value of field 'a' actually contains a NUL char !
+  cat >test.yml <<EOL
+a: "foo\u0000bar"
+b: 1
+c: |
+  wiz
+  boom
+EOL
+
+  ## We are looking for trouble with asking to separated fields with NUL
+  ## char and requested value `.a` actually contains itself a NUL char !
+  read-0 a b c < <(./yq e -0 '.a, .b, .c' test.yml)
+  assertNotEquals "0" "$?"   ## read-0 failed to fill all values
+
+  ## But here, we can request for one value, even if `./yq` fails
+  read-0 b < <(./yq e -0 '.b, .a' test.yml)
+  assertEquals "0" "$?"   ## read-0 succeeds at feeding the first value
+  ## Note: to catch the failure of `yq`, see in the next tests the usage
+  ## of `read-0-err`.
+
+  ## using -r=false solves any NUL containing value issues, but keeps
+  ## all in YAML representation:
+  read-0 a b c < <(./yq e -0 -r=false '.a, .b, .c' test.yml)
+  assertEquals "0" "$?"    ## All goes well despite asking for `a` value
+
+  assertEquals '"foo\0bar"' "$a"   ## This is a YAML string representation
+  assertEquals '1' "$b"
+  assertEquals '|
+  wiz
+  boom' "$c"
+}
+
+testStandardLoop() {
+    local E a b res
+
+    ## Here everything is normal: 4 values, that will be paired
+    ## in key/values.
+    cat >test.yml <<EOL
+- yay
+- wiz
+- hop
+- pow
+EOL
+
+    res=""
+    while read-0-err E a b; do
+        res+="$a: $b;"
+    done < <(p-err ./yq -0 '.[]' test.yml)
+
+    assertEquals "0" "$E"                     ## errorlevel of internal command
+    assertEquals "yay: wiz;hop: pow;" "$res"  ## expected result
+}
+
+testStandardLoopWithoutEnoughValues() {
+    local E a b res
+
+    ## Here 5 values, there will be a missing value when reading
+    ## pairs of value.
+    cat >test.yml <<EOL
+- yay
+- wiz
+- hop
+- pow
+- kwak
+EOL
+
+    res=""
+    ## The loop will succeed 2 times then fail
+    while read-0-err E a b; do
+        res+="$a: $b;"
+    done < <(p-err ./yq -0 '.[]' test.yml)
+
+    assertEquals "not-enough-values" "$E"     ## Not enough value error
+    assertEquals "yay: wiz;hop: pow;" "$res"  ## the 2 full key/value pairs
+
+}
+
+testStandardLoopWithInternalCmdError() {
+    local E a b res
+
+    ## Note the third value contains a NUL char !
+    cat >test.yml <<EOL
+- yay
+- wiz
+- "foo\0bar"
+- hop
+- pow
+EOL
+
+    res=""
+    ## It should be only upon the second pass in the loop that
+    ## read-0-err will catch the fact that there is an error !
+    while read-0-err E a b; do
+        res+="$a: $b;"
+    done < <(p-err ./yq -0 '.[]' test.yml)
+    assertEquals "1" "$E"            ## Internal command errorlevel (from `./yq`)
+    assertEquals "yay: wiz;" "$res"  ## first 2 values were ok at least
+
+}
+
+testStandardLoopNotEnoughErrorEatsCmdError() {
+    local E a b res
+
+    ## Because of possible edge cases where the internal errorlevel
+    ## reported by `p-err` in the standard output might be mangled
+    ## with the unfinished record, `read-0-err E ...` will NOT report
+    ## the internal command error in the variable E and instead will
+    ## store the value 'not-enough-values'. In real world, anyway, you
+    ## will want to react the same if the internal command failed
+    ## and/or you didn't get as much values as expected while
+    ## reading. Keep in mind also that standard error is not
+    ## swallowed, so you can read reports from the inner command AND
+    ## from `read-0-err`.
+
+    ## Here, note that the fourth value contains a NUL char !
+    cat >test.yml <<EOL
+- yay
+- wiz
+- hop
+- "foo\0bar"
+- pow
+EOL
+
+    res=""
+    ## It should be only upon the second loop that read-0-err will catch
+    ## the fact that there are not enough data to fill the requested variables
+    while read-0-err E a b; do
+        res+="$a: $b;"
+    done < <(p-err ./yq -0 '.[]' test.yml)
+    assertEquals "not-enough-values" "$E"          ## Not enough values error eats internal error !
+    assertEquals "yay: wiz;" "$res"  ## first 2 values were ok at least
+}
+
+
+source ./scripts/shunit2

acceptance_tests/output-format.sh

@@ -1,5 +1,9 @@
 #!/bin/bash
 
+setUp() {
+  rm test*.yml || true
+}
+
 testOutputJsonDeprecated() {
   cat >test.yml <<EOL
 a: {b: ["cat"]}
@@ -44,6 +48,64 @@ EOM
   assertEquals "$expected" "$X"
 }
 
+testOutputYamlRawDefault() {
+  cat >test.yml <<EOL
+a: "cat"
+EOL
+
+  X=$(./yq e  '.a' test.yml)
+  assertEquals "cat" "$X"
+
+  X=$(./yq ea '.a' test.yml)
+  assertEquals "cat" "$X"
+}
+
+testOutputYamlRawOff() {
+  cat >test.yml <<EOL
+a: "cat"
+EOL
+
+  X=$(./yq e -r=false '.a' test.yml)
+  assertEquals "\"cat\"" "$X"
+
+  X=$(./yq ea -r=false '.a' test.yml)
+  assertEquals "\"cat\"" "$X"
+}
+
+testOutputYamlRawOnRoot() {
+  cat >test.yml <<EOL
+'a'
+EOL
+
+  X=$(./yq e -r '.' test.yml)
+  assertEquals "a" "$X"
+}
+
+testOutputJsonRaw() {
+  cat >test.yml <<EOL
+a: cat
+EOL
+
+  X=$(./yq e -r --output-format=json '.a' test.yml)
+  assertEquals "cat" "$X"
+
+  X=$(./yq ea -r --output-format=json '.a' test.yml)
+  assertEquals "cat" "$X"
+}
+
+testOutputJsonDefault() {
+  cat >test.yml <<EOL
+a: cat
+EOL
+
+  X=$(./yq e --output-format=json '.a' test.yml)
+  assertEquals "\"cat\"" "$X"
+
+  X=$(./yq ea --output-format=json '.a' test.yml)
+  assertEquals "\"cat\"" "$X"
+}
+
+
 testOutputJsonShort() {
   cat >test.yml <<EOL
 a: {b: ["cat"]}
@@ -68,11 +130,11 @@ EOM
 
 testOutputProperties() {
   cat >test.yml <<EOL
-a: {b: {c: ["cat"]}}
+a: {b: {c: ["cat cat"]}}
 EOL
 
   read -r -d '' expected << EOM
-a.b.c.0 = cat
+a.b.c.0 = cat cat
 EOM
 
   X=$(./yq e --output-format=props test.yml)
@@ -82,13 +144,30 @@ EOM
   assertEquals "$expected" "$X"
 }
 
-testOutputPropertiesShort() {
+testOutputPropertiesDontUnwrap() {
   cat >test.yml <<EOL
-a: {b: {c: ["cat"]}}
+a: {b: {c: ["cat cat"]}}
 EOL
 
   read -r -d '' expected << EOM
-a.b.c.0 = cat
+a.b.c.0 = "cat cat"
+EOM
+
+  X=$(./yq e -r=false --output-format=props test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -r=false --output-format=props test.yml)
+  assertEquals "$expected" "$X"
+}
+
+
+testOutputPropertiesShort() {
+  cat >test.yml <<EOL
+a: {b: {c: ["cat cat"]}}
+EOL
+
+  read -r -d '' expected << EOM
+a.b.c.0 = cat cat
 EOM
 
   X=$(./yq e -o=p test.yml)
@@ -98,4 +177,167 @@ EOM
   assertEquals "$expected" "$X"
 }
 
-source ./scripts/shunit2
+testOutputCSV() {
+  cat >test.yml <<EOL
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOL
+
+  read -r -d '' expected << EOM
+fruit,yumLevel
+apple,5
+banana,4
+EOM
+
+  X=$(./yq -o=c test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -o=csv test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testOutputCSVCustomSeparator() {
+  cat >test.yml <<EOL
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOL
+
+  read -r -d '' expected << EOM
+fruit;yumLevel
+apple;5
+banana;4
+EOM
+
+  X=$(./yq -oc --csv-separator ";" test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -o=csv --csv-separator ";" test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testOutputTSV() {
+  cat >test.yml <<EOL
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOL
+
+  read -r -d '' expected << EOM
+fruit	yumLevel
+apple	5
+banana	4
+EOM
+
+  X=$(./yq -o=t test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -o=tsv test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testOutputXml() {
+  cat >test.yml <<EOL
+a: {b: {c: ["cat"]}}
+EOL
+
+  read -r -d '' expected << EOM
+<a>
+  <b>
+    <c>cat</c>
+  </b>
+</a>
+EOM
+
+  X=$(./yq e --output-format=xml test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --output-format=xml test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testOutputXmlShort() {
+  cat >test.yml <<EOL
+a: {b: {c: ["cat"]}}
+EOL
+
+  read -r -d '' expected << EOM
+<a>
+  <b>
+    <c>cat</c>
+  </b>
+</a>
+EOM
+
+  X=$(./yq e --output-format=x test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --output-format=x test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testOutputXmComplex() {
+  cat >test.yml <<EOL
+a: {b: {c: ["cat", "dog"], +@f: meow}}
+EOL
+
+  read -r -d '' expected << EOM
+<a>
+  <b f="meow">
+    <c>cat</c>
+    <c>dog</c>
+  </b>
+</a>
+EOM
+
+  X=$(./yq e --output-format=x test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea --output-format=x test.yml)
+  assertEquals "$expected" "$X"
+}
+
+testLuaOutputPretty() {
+  cat >test.yml <<EOL
+animals:
+  cat: meow
+EOL
+
+  read -r -d '' expected << EOM
+return {
+	["animals"] = {
+		["cat"] = "meow";
+	};
+};
+EOM
+
+  X=$(./yq e --output-format=lua test.yml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq e --output-format=lua --prettyPrint test.yml)
+  assertEquals "$expected" "$X"
+
+}
+
+testLuaOutputSubset() {
+  cat >test.yml <<EOL
+animals:
+  cat: meow
+EOL
+
+  read -r -d '' expected << EOM
+return {
+	["cat"] = "meow";
+};
+EOM
+
+  X=$(./yq e --output-format=lua '.animals' test.yml)
+  assertEquals "$expected" "$X"
+
+}
+
+source ./scripts/shunit2

acceptance_tests/pipe.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+setUp() {
+  rm test*.yml || true
+  cat >test.yml <<EOL
+a: frog
+EOL
+}
+
+testPipeViaCatWithParam() {
+  X=$(cat test.yml | ./yq '.a')
+  assertEquals "frog" "$X"
+}
+
+testPipeViaCatWithParamEval() {
+  X=$(cat test.yml | ./yq e '.a')
+  assertEquals "frog" "$X"
+}
+
+testPipeViaCatWithParamEvalAll() {
+  X=$(cat test.yml | ./yq ea '.a')
+  assertEquals "frog" "$X"
+}
+
+testPipeViaCatNoParam() {
+  X=$(cat test.yml | ./yq)
+  assertEquals "a: frog" "$X"
+}
+
+testPipeViaCatNoParamEval() {
+  X=$(cat test.yml | ./yq e)
+  assertEquals "a: frog" "$X"
+}
+
+testPipeViaCatNoParamEvalAll() {
+  X=$(cat test.yml | ./yq ea)
+  assertEquals "a: frog" "$X"
+}
+
+testPipeViaFileishWithParam() {
+  X=$(./yq '.a' < test.yml)
+  assertEquals "frog" "$X"
+}
+
+testPipeViaFileishWithParamEval() {
+  X=$(./yq e '.a' < test.yml)
+  assertEquals "frog" "$X"
+}
+
+testPipeViaFileishWithParamEvalAll() {
+  X=$(./yq ea '.a' < test.yml)
+  assertEquals "frog" "$X"
+}
+
+testPipeViaFileishNoParam() {
+  X=$(./yq < test.yml)
+  assertEquals "a: frog" "$X"
+}
+
+testPipeViaFileishNoParamEval() {
+  X=$(./yq e < test.yml)
+  assertEquals "a: frog" "$X"
+}
+
+testPipeViaFileishNoParamEvalAll() {
+  X=$(./yq ea < test.yml)
+  assertEquals "a: frog" "$X"
+}
+
+source ./scripts/shunit2

acceptance_tests/pretty-print.sh

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+setUp() {
+  rm test*.yml || true
+}
 
 testPrettyPrintWithBooleans() {
   cat >test.yml <<EOL

acceptance_tests/shebang.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+setUp() {
+  rm test*.yq || true
+  cat >test.yq <<EOL
+#!./yq
+.a.b
+EOL
+chmod +x test.yq
+
+rm test*.yml || true
+  cat >test.yml <<EOL
+a: {b: apple}
+EOL
+}
+
+testCanExecYqFile() {
+  read -r -d '' expected << EOM
+apple
+EOM
+   X=$(./test.yq test.yml)
+  assertEquals "$expected" "$X"
+}
+
+source ./scripts/shunit2
+

acceptance_tests/split-printer.sh

@@ -0,0 +1,227 @@
+#!/bin/bash
+
+setUp() {
+  rm test*.yml || true
+  rm -rf test_dir* || true
+}
+
+testBasicSplitWithName() {
+  cat >test.yml <<EOL
+a: test_doc1
+--- 
+a: test_doc2
+EOL
+
+  ./yq e test.yml -s ".a"
+
+  doc1=$(cat test_doc1.yml)
+  
+  assertEquals "a: test_doc1" "$doc1"
+
+  doc2=$(cat test_doc2.yml)
+  read -r -d '' expectedDoc2 << EOM
+---
+a: test_doc2
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+}
+
+testBasicSplitWithNameCustomExtension() {
+  rm test*.yaml || true
+  cat >test.yml <<EOL
+a: test_doc1
+--- 
+a: test_doc2
+EOL
+
+  ./yq e test.yml -s '.a + ".yaml"'
+
+  doc1=$(cat test_doc1.yaml)
+  
+  assertEquals "a: test_doc1" "$doc1"
+
+  doc2=$(cat test_doc2.yaml)
+  read -r -d '' expectedDoc2 << EOM
+---
+a: test_doc2
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+}
+
+
+
+
+testSplitFromFile() {
+  cat >test.yml <<EOL
+a: test_doc1
+--- 
+a: test_doc2
+EOL
+
+cat >test_splitExp.yml <<EOL
+.a
+EOL
+
+  ./yq test.yml --split-exp-file test_splitExp.yml
+
+  doc1=$(cat test_doc1.yml)
+  
+  assertEquals "a: test_doc1" "$doc1"
+
+  doc2=$(cat test_doc2.yml)
+  read -r -d '' expectedDoc2 << EOM
+---
+a: test_doc2
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+}
+
+testBasicSplitWithNameEvalAll() {
+  cat >test.yml <<EOL
+a: test_doc1
+--- 
+a: test_doc2
+EOL
+
+  ./yq ea test.yml -s ".a"
+
+  doc1=$(cat test_doc1.yml)
+  
+  assertEquals "a: test_doc1" "$doc1"
+
+  doc2=$(cat test_doc2.yml)
+  read -r -d '' expectedDoc2 << EOM
+---
+a: test_doc2
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+}
+
+testBasicSplitWithIndex() {
+  cat >test.yml <<EOL
+a: test_doc1
+--- 
+a: test_doc2
+EOL
+
+  ./yq e test.yml -s '"test_" + $index'
+
+  doc1=$(cat test_0.yml)
+  
+  assertEquals "a: test_doc1" "$doc1"
+
+  doc2=$(cat test_1.yml)
+  read -r -d '' expectedDoc2 << EOM
+---
+a: test_doc2
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+}
+
+testBasicSplitWithIndexEvalAll() {
+  cat >test.yml <<EOL
+a: test_doc1
+--- 
+a: test_doc2
+EOL
+
+  ./yq ea test.yml -s '"test_" + $index'
+
+  doc1=$(cat test_0.yml)
+  
+  assertEquals "a: test_doc1" "$doc1"
+
+  doc2=$(cat test_1.yml)
+  read -r -d '' expectedDoc2 << EOM
+---
+a: test_doc2
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+}
+
+
+testArraySplitWithNameNoSeparators() {
+  cat >test.yml <<EOL
+- name: test_fred
+  age: 35
+- name: test_catherine
+  age: 37
+EOL
+
+  ./yq e --no-doc -s ".name"  ".[]" test.yml 
+
+  doc1=$(cat test_fred.yml)
+  read -r -d '' expectedDoc1 << EOM
+name: test_fred
+age: 35
+EOM
+
+  assertEquals "$expectedDoc1" "$doc1"
+
+  doc2=$(cat test_catherine.yml)
+  read -r -d '' expectedDoc2 << EOM
+name: test_catherine
+age: 37
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+}
+
+testArraySplitWithNameNoSeparatorsEvalAll() {
+  cat >test.yml <<EOL
+- name: test_fred
+  age: 35
+- name: test_catherine
+  age: 37
+EOL
+
+cat >test2.yml <<EOL
+- name: test_mike
+  age: 564
+EOL
+
+  ./yq ea --no-doc -s ".name"  ".[]" test.yml test2.yml
+
+  doc1=$(cat test_fred.yml)
+  read -r -d '' expectedDoc1 << EOM
+name: test_fred
+age: 35
+EOM
+
+  assertEquals "$expectedDoc1" "$doc1"
+
+  doc2=$(cat test_catherine.yml)
+  read -r -d '' expectedDoc2 << EOM
+name: test_catherine
+age: 37
+EOM
+  assertEquals "$expectedDoc2" "$doc2"
+
+
+  doc3=$(cat test_mike.yml)
+  read -r -d '' expectedDoc3 << EOM
+name: test_mike
+age: 564
+EOM
+  assertEquals "$expectedDoc3" "$doc3"
+}
+
+testSplitWithDirectories() {
+  cat >test.yml <<EOL
+f: test_dir1/test_file1
+---
+f: test_dir2/dir22/test_file2
+---
+f: test_file3
+EOL
+
+  ./yq e --no-doc -s ".f" test.yml
+
+  doc1=$(cat test_dir1/test_file1.yml)
+  assertEquals "f: test_dir1/test_file1" "$doc1"
+  doc2=$(cat test_dir2/dir22/test_file2.yml)
+  assertEquals "f: test_dir2/dir22/test_file2" "$doc2"
+  doc3=$(cat test_file3.yml)
+  assertEquals "f: test_file3" "$doc3"
+}
+
+source ./scripts/shunit2

action.yml

@@ -12,6 +12,6 @@ outputs:
     description: "The complete result from the yq command being run"
 runs:
   using: 'docker'
-  image: 'github-action/Dockerfile'
+  image: 'docker://mikefarah/yq:4-githubaction'
   args:
     - ${{ inputs.cmd }}

agents.md

@@ -0,0 +1,413 @@
+# Adding a New Encoder/Decoder
+
+This guide explains how to add support for a new format (encoder/decoder) to yq without modifying `candidate_node.go`.
+
+## Overview
+
+The encoder/decoder architecture in yq is based on two main interfaces:
+
+- **Encoder**: Converts a `CandidateNode` to output in a specific format
+- **Decoder**: Reads input in a specific format and creates a `CandidateNode`
+
+Each format is registered in `pkg/yqlib/format.go` and made available through factory functions.
+
+## Architecture
+
+### Key Files
+
+- `pkg/yqlib/encoder.go` - Defines the `Encoder` interface
+- `pkg/yqlib/decoder.go` - Defines the `Decoder` interface
+- `pkg/yqlib/format.go` - Format registry and factory functions
+- `pkg/yqlib/operator_encoder_decoder.go` - Encode/decode operators
+- `pkg/yqlib/encoder_*.go` - Encoder implementations
+- `pkg/yqlib/decoder_*.go` - Decoder implementations
+
+### Interfaces
+
+**Encoder Interface:**
+```go
+type Encoder interface {
+    Encode(writer io.Writer, node *CandidateNode) error
+    PrintDocumentSeparator(writer io.Writer) error
+    PrintLeadingContent(writer io.Writer, content string) error
+    CanHandleAliases() bool
+}
+```
+
+**Decoder Interface:**
+```go
+type Decoder interface {
+    Init(reader io.Reader) error
+    Decode() (*CandidateNode, error)
+}
+```
+
+## Step-by-Step: Adding a New Encoder/Decoder
+
+### Step 1: Create the Encoder File
+
+Create `pkg/yqlib/encoder_<format>.go` implementing the `Encoder` interface:
+- `Encode()` - Convert a `CandidateNode` to your format and write to the output writer
+- `PrintDocumentSeparator()` - Handle document separators if your format requires them
+- `PrintLeadingContent()` - Handle leading content/comments if supported
+- `CanHandleAliases()` - Return whether your format supports YAML aliases
+
+See `encoder_json.go` or `encoder_base64.go` for examples.
+
+### Step 2: Create the Decoder File
+
+Create `pkg/yqlib/decoder_<format>.go` implementing the `Decoder` interface:
+- `Init()` - Initialize the decoder with the input reader and set up any needed state
+- `Decode()` - Decode one document from the input and return a `CandidateNode`, or `io.EOF` when finished
+
+See `decoder_json.go` or `decoder_base64.go` for examples.
+
+### Step 3: Create Tests (Mandatory)
+
+Create a test file `pkg/yqlib/<format>_test.go` using the `formatScenario` pattern:
+- Define test scenarios as `formatScenario` structs with fields: `description`, `input`, `expected`, `scenarioType`
+- `scenarioType` can be `"decode"` (test decoding to YAML) or `"roundtrip"` (encode/decode preservation)
+- Create a helper function `test<Format>Scenario()` that switches on `scenarioType`
+- Create main test function `Test<Format>FormatScenarios()` that iterates over scenarios
+
+Test coverage must include:
+- Basic data types (scalars, arrays, objects/maps)
+- Nested structures
+- Edge cases (empty inputs, special characters, escape sequences)
+- Format-specific features or syntax
+- Round-trip tests: decode → encode → decode should preserve data
+
+See `hcl_test.go` for a complete example.
+
+### Step 4: Register the Format in format.go
+
+Edit `pkg/yqlib/format.go`:
+
+1. Add a new format variable:
+   - `"<format>"` is the formal name (e.g., "json", "yaml")
+   - `[]string{...}` contains short aliases (can be empty)
+   - The first function creates an encoder (can be nil for encode-only formats)
+   - The second function creates a decoder (can be nil for decode-only formats)
+
+2. Add the format to the `Formats` slice in the same file
+
+See existing formats in `format.go` for the exact structure.
+
+### Step 5: Handle Encoder Configuration (if needed)
+
+If your format has preferences/configuration options:
+
+1. Create a preferences struct with your configuration fields
+2. Update the encoder to accept preferences in its factory function
+3. Update `format.go` to pass the configured preferences
+4. Update `operator_encoder_decoder.go` if special indent handling is needed (see existing formats like JSON and YAML for the pattern)
+
+This pattern is optional and only needed if your format has user-configurable options.
+
+## Build Tags
+
+Use build tags to allow optional compilation of formats:
+- Add `//go:build !yq_no<format>` at the top of your encoder and decoder files
+- Create a no-build version in `pkg/yqlib/no_<format>.go` that returns nil for encoder/decoder factories
+
+This allows users to compile yq without certain formats using: `go build -tags yq_no<format>`
+
+## Working with CandidateNode
+
+The `CandidateNode` struct represents a YAML node with:
+- `Kind`: The node type (ScalarNode, SequenceNode, MappingNode)
+- `Tag`: The YAML tag (e.g., "!!str", "!!int", "!!map")
+- `Value`: The scalar value (for ScalarNode only)
+- `Content`: Child nodes (for SequenceNode and MappingNode)
+
+Key methods:
+- `node.guessTagFromCustomType()` - Infer the tag from Go type
+- `node.AsList()` - Convert to a list for processing
+- `node.CreateReplacement()` - Create a new replacement node
+- `NewCandidate()` - Create a new CandidateNode
+
+## Key Points
+
+✅ **DO:**
+- Implement only the `Encoder` and `Decoder` interfaces
+- Register your format in `format.go` only
+- Keep format-specific logic in your encoder/decoder files
+- Use the candidate_node style attribute to store style information for round-trip. Ask if this needs to be updated with new styles.
+- Use build tags for optional compilation
+- Add comprehensive tests
+- Handle errors gracefully
+- Add the no build directive, like the xml encoder and decoder, that enables a minimal yq builds. e.g.  `//go:build !yq_<format>`. Be sure to also update the build_small-yq.sh and build-tinygo-yq.sh to not include the new format.
+
+❌ **DON'T:**
+- Modify `candidate_node.go` to add format-specific logic
+- Add format-specific fields to `CandidateNode`
+- Create special cases in core navigation or evaluation logic
+- Bypass the encoder/decoder interfaces
+- Use candidate_node tag attribute for anything other than indicate the data type
+
+## Examples
+
+Refer to existing format implementations for patterns:
+
+- **Simple encoder/decoder**: `encoder_json.go`, `decoder_json.go`
+- **Complex with preferences**: `encoder_yaml.go`, `decoder_yaml.go`
+- **Encoder-only**: `encoder_sh.go` (ShFormat has nil decoder)
+- **String-only operations**: `encoder_base64.go`, `decoder_base64.go`
+
+## Testing Your Implementation (Mandatory)
+
+Tests must be implemented in `<format>_test.go` following the `formatScenario` pattern:
+
+1. **Create test scenarios** using the `formatScenario` struct with fields:
+   - `description`: Brief description of what's being tested
+   - `input`: Sample input in your format
+   - `expected`: Expected output (typically in YAML for decode tests)
+   - `scenarioType`: Either `"decode"` or `"roundtrip"`
+
+2. **Test coverage must include:**
+   - Basic data types (scalars, arrays, objects/maps)
+   - Nested structures
+   - Edge cases (empty inputs, special characters, escape sequences)
+   - Format-specific features or syntax
+   - Round-trip tests: decode → encode → decode should preserve data
+
+3. **Test function pattern:**
+   - `test<Format>Scenario()`: Helper function that switches on `scenarioType`
+   - `Test<Format>FormatScenarios()`: Main test function that iterates over scenarios
+
+4. **Example from existing formats:**
+   - See `hcl_test.go` for a complete example
+   - See `yaml_test.go` for YAML-specific patterns
+   - See `json_test.go` for more complex scenarios
+
+## Common Patterns
+
+### Scalar-Only Formats
+Some formats only work with scalars (like base64, uri):
+```go
+if node.guessTagFromCustomType() != "!!str" {
+    return fmt.Errorf("cannot encode %v as <format>, can only operate on strings", node.Tag)
+}
+```
+
+### Format with Indentation
+Use preferences to control output formatting:
+```go
+type <format>Preferences struct {
+    Indent int
+}
+
+func (prefs *<format>Preferences) Copy() <format>Preferences {
+    return *prefs
+}
+```
+
+### Multiple Documents
+Decoders should support reading multiple documents:
+```go
+func (dec *<format>Decoder) Decode() (*CandidateNode, error) {
+    if dec.finished {
+        return nil, io.EOF
+    }
+    // ... decode next document ...
+    if noMoreDocuments {
+        dec.finished = true
+    }
+    return candidate, nil
+}
+```
+
+---
+
+# Adding a New Operator
+
+This guide explains how to add a new operator to yq. Operators are the core of yq's expression language and process `CandidateNode` objects without requiring modifications to `candidate_node.go` itself.
+
+## Overview
+
+Operators transform data by implementing a handler function that processes a `Context` containing `CandidateNode` objects. Each operator is:
+
+1. Defined as an `operationType` in `operation.go`
+2. Registered in the lexer in `lexer_participle.go`
+3. Implemented in its own `operator_<type>.go` file
+4. Tested in `operator_<type>_test.go`
+5. Documented in `pkg/yqlib/doc/operators/headers/<type>.md`
+
+## Architecture
+
+### Key Files
+
+- `pkg/yqlib/operation.go` - Defines `operationType` and operator registry
+- `pkg/yqlib/lexer_participle.go` - Registers operators with their syntax patterns
+- `pkg/yqlib/operator_<type>.go` - Operator implementation
+- `pkg/yqlib/operator_<type>_test.go` - Operator tests using `expressionScenario`
+- `pkg/yqlib/doc/operators/headers/<type>.md` - Documentation header
+
+### Core Types
+
+**operationType:**
+```go
+type operationType struct {
+    Type                 string          // Unique operator name (e.g., "REVERSE")
+    NumArgs              uint            // Number of arguments (0 for no args)
+    Precedence           uint            // Operator precedence (higher = higher precedence)
+    Handler              operatorHandler // The function that executes the operator
+    CheckForPostTraverse bool            // Whether to apply post-traversal logic
+    ToString             func(*Operation) string // Custom string representation
+}
+```
+
+**operatorHandler signature:**
+```go
+type operatorHandler func(*dataTreeNavigator, Context, *ExpressionNode) (Context, error)
+```
+
+**expressionScenario for tests:**
+```go
+type expressionScenario struct {
+    description      string
+    subdescription   string
+    document         string
+    expression       string
+    expected         []string
+    skipDoc          bool
+    expectedError    string
+}
+```
+
+## Step-by-Step: Adding a New Operator
+
+### Step 1: Create the Operator Implementation File
+
+Create `pkg/yqlib/operator_<type>.go` implementing the operator handler function:
+- Implement the `operatorHandler` function signature
+- Process nodes from `context.MatchingNodes`
+- Return a new `Context` with results using `context.ChildContext()`
+- Use `candidate.CreateReplacement()` or `candidate.CreateReplacementWithComments()` to create new nodes
+- Handle errors gracefully with meaningful error messages
+
+See `operator_reverse.go` or `operator_keys.go` for examples.
+
+### Step 2: Register the Operator in operation.go
+
+Add the operator type definition to `pkg/yqlib/operation.go`:
+
+```go
+var <type>OpType = &operationType{
+    Type:       "<TYPE>",          // All caps, matches pattern in lexer
+    NumArgs:    0,                 // 0 for no args, 1+ for args
+    Precedence: 50,                // Typical range: 40-55
+    Handler:    <type>Operator,    // Reference to handler function
+}
+```
+
+**Precedence guidelines:**
+- 10-20: Logical operators (OR, AND, UNION)
+- 30: Pipe operator
+- 40: Assignment and comparison operators
+- 42: Arithmetic operators (ADD, SUBTRACT, MULTIPLY, DIVIDE)
+- 50-52: Most other operators
+- 55: High precedence (e.g., GET_VARIABLE)
+
+**Optional fields:**
+- `CheckForPostTraverse: true` - If your operator can have another directly after it without the pipe character. Most of the time this is false.
+- `ToString: customToString` - Custom string representation (rarely needed)
+
+### Step 3: Register the Operator in lexer_participle.go
+
+Edit `pkg/yqlib/lexer_participle.go` to add the operator to the lexer rules:
+- Use `simpleOp()` for simple keyword patterns
+- Use object syntax for regex patterns or complex syntax
+- Support optional characters with `_?` and aliases with `|`
+
+See existing operators in `lexer_participle.go` for pattern examples.
+
+### Step 4: Create Tests (Mandatory)
+
+Create `pkg/yqlib/operator_<type>_test.go` using the `expressionScenario` pattern:
+- Define test scenarios with `description`, `document`, `expression`, and `expected` fields
+- `expected` is a slice of strings showing output format: `"D<doc>, P[<path>], (<tag>)::<value>\n"`
+- Set `skipDoc: true` for edge cases you don't want in generated documentation
+- Include `subdescription` for longer test names
+- Set `expectedError` if testing error cases
+- Create main test function that iterates over scenarios
+
+Test coverage must include:
+- Basic data types and nested structures
+- Edge cases (empty inputs, special characters, type errors)
+- Multiple outputs if applicable
+- Format-specific features
+
+See `operator_reverse_test.go` for a simple example and `operator_keys_test.go` for complex cases.
+
+### Step 5: Create Documentation Header
+
+Create `pkg/yqlib/doc/operators/headers/<type>.md`:
+- Use the exact operator name as the title
+- Include a concise 1-2 sentence summary
+- Add additional context or examples if the operator is complex
+
+See existing headers in `doc/operators/headers/` for examples.
+
+## Working with Context and CandidateNode
+
+### Context Management
+- `context.ChildContext(results)` - Create child context with results
+- `context.GetVariable("varName")` - Get variables stored in context
+- `context.SetVariable("varName", value)` - Set variables in context
+
+### CandidateNode Operations
+- `candidate.CreateReplacement(ScalarNode, "!!str", stringValue)` - Create a replacement node
+- `candidate.CreateReplacementWithComments(SequenceNode, "!!seq", candidate.Style)` - With style preserved
+- `candidate.Kind` - The node type (ScalarNode, SequenceNode, MappingNode)
+- `candidate.Tag` - The YAML tag (!!str, !!int, etc.)
+- `candidate.Value` - The scalar value (for ScalarNode only)
+- `candidate.Content` - Child nodes (for SequenceNode and MappingNode)
+- `candidate.guessTagFromCustomType()` - Infer the tag from Go type
+- `candidate.AsList()` - Convert to a list representation
+
+## Key Points
+
+✅ **DO:**
+- Implement the operator handler with the correct signature
+- Register in `operation.go` with appropriate precedence
+- Add the lexer pattern in `lexer_participle.go`
+- Write comprehensive tests covering normal and edge cases
+- Create a documentation header in `doc/operators/headers/`
+- Use `Context.ChildContext()` for proper context threading
+- Handle all node types gracefully
+- Return meaningful error messages
+
+❌ **DON'T:**
+- Modify `candidate_node.go` (operators shouldn't need this)
+- Modify core navigation or evaluation logic
+- Bypass the handler function pattern
+- Add format-specific or operator-specific fields to `CandidateNode`
+- Skip tests or documentation
+
+## Examples
+
+Refer to existing operator implementations for patterns:
+
+- **No-argument operator**: `operator_reverse.go` - Processes arrays/sequences
+- **Single-argument operator**: `operator_map.go` - Takes an expression argument
+- **Complex multi-output**: `operator_keys.go` - Produces multiple results
+- **With preferences**: `operator_to_number.go` - Configuration options
+- **Error handling**: `operator_error.go` - Control flow with errors
+- **String operations**: `operator_strings.go` - Multiple related operators
+
+## Testing Patterns
+
+Refer to existing test files for specific patterns:
+- Basic expression tests in `operator_reverse_test.go`
+- Multi-output tests in `operator_keys_test.go`
+- Error handling tests in `operator_error_test.go`
+- Tests with `skipDoc` flag to exclude from generated documentation
+
+## Common Patterns
+
+Refer to existing operator implementations for these patterns:
+- Simple transformation: see `operator_reverse.go`
+- Type checking: see `operator_error.go`
+- Working with arguments: see `operator_map.go`
+- Post-traversal operators: see `operator_with.go`

cmd/commands_test.go

@@ -1,83 +0,0 @@
-package cmd
-
-// import (
-// 	"strings"
-// 	"testing"
-
-// 	"github.com/mikefarah/yq/v3/test"
-// 	"github.com/spf13/cobra"
-// )
-
-// func getRootCommand() *cobra.Command {
-// 	return New()
-// }
-
-// func TestRootCmd(t *testing.T) {
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, "")
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-
-// 	if !strings.Contains(result.Output, "Usage:") {
-// 		t.Error("Expected usage message to be printed out, but the usage message was not found.")
-// 	}
-// }
-
-// func TestRootCmd_Help(t *testing.T) {
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, "--help")
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-
-// 	if !strings.Contains(result.Output, "yq is a lightweight and portable command-line YAML processor. It aims to be the jq or sed of yaml files.") {
-// 		t.Error("Expected usage message to be printed out, but the usage message was not found.")
-// 	}
-// }
-
-// func TestRootCmd_VerboseLong(t *testing.T) {
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, "--verbose")
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-
-// 	if !verbose {
-// 		t.Error("Expected verbose to be true")
-// 	}
-// }
-
-// func TestRootCmd_VerboseShort(t *testing.T) {
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, "-v")
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-
-// 	if !verbose {
-// 		t.Error("Expected verbose to be true")
-// 	}
-// }
-
-// func TestRootCmd_VersionShort(t *testing.T) {
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, "-V")
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	if !strings.Contains(result.Output, "yq version") {
-// 		t.Error("expected version message to be printed out, but the message was not found.")
-// 	}
-// }
-
-// func TestRootCmd_VersionLong(t *testing.T) {
-// 	cmd := getRootCommand()
-// 	result := test.RunCmd(cmd, "--version")
-// 	if result.Error != nil {
-// 		t.Error(result.Error)
-// 	}
-// 	if !strings.Contains(result.Output, "yq version") {
-// 		t.Error("expected version message to be printed out, but the message was not found.")
-// 	}
-// }

cmd/completion.go

@@ -7,19 +7,20 @@ import (
 )
 
 var completionCmd = &cobra.Command{
-	Use:   "shell-completion [bash|zsh|fish|powershell]",
-	Short: "Generate completion script",
+	Use:     "completion [bash|zsh|fish|powershell]",
+	Aliases: []string{"shell-completion"},
+	Short:   "Generate the autocompletion script for the specified shell",
 	Long: `To load completions:
 
 Bash:
 
-$ source <(yq shell-completion bash)
+$ source <(yq completion bash)
 
 # To load completions for each session, execute once:
 Linux:
-  $ yq shell-completion bash > /etc/bash_completion.d/yq
+  $ yq completion bash > /etc/bash_completion.d/yq
 MacOS:
-  $ yq shell-completion bash > /usr/local/etc/bash_completion.d/yq
+  $ yq completion bash > /usr/local/etc/bash_completion.d/yq
 
 Zsh:
 
@@ -29,25 +30,25 @@ Zsh:
 $ echo "autoload -U compinit; compinit" >> ~/.zshrc
 
 # To load completions for each session, execute once:
-$ yq shell-completion zsh > "${fpath[1]}/_yq"
+$ yq completion zsh > "${fpath[1]}/_yq"
 
 # You will need to start a new shell for this setup to take effect.
 
 Fish:
 
-$ yq shell-completion fish | source
+$ yq completion fish | source
 
 # To load completions for each session, execute once:
-$ yq shell-completion fish > ~/.config/fish/completions/yq.fish
+$ yq completion fish > ~/.config/fish/completions/yq.fish
 `,
 	DisableFlagsInUseLine: true,
 	ValidArgs:             []string{"bash", "zsh", "fish", "powershell"},
-	Args:                  cobra.ExactValidArgs(1),
+	Args:                  cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs),
 	RunE: func(cmd *cobra.Command, args []string) error {
 		var err error = nil
 		switch args[0] {
 		case "bash":
-			err = cmd.Root().GenBashCompletion(os.Stdout)
+			err = cmd.Root().GenBashCompletionV2(os.Stdout, true)
 		case "zsh":
 			err = cmd.Root().GenZshCompletion(os.Stdout)
 		case "fish":

cmd/constant.go

@@ -1,24 +1,39 @@
 package cmd
 
-var leadingContentPreProcessing = true
-var unwrapScalar = true
+var unwrapScalarFlag = newUnwrapFlag()
+
+var printNodeInfo = false
+
+var unwrapScalar = false
 
 var writeInplace = false
 var outputToJSON = false
-var outputFormat = "yaml"
+
+var outputFormat = ""
+
+var inputFormat = ""
 
 var exitStatus = false
-var forceColor = false
-var forceNoColor = false
-var colorsEnabled = false
 var indent = 2
 var noDocSeparators = false
 var nullInput = false
+var nulSepOutput = false
 var verbose = false
 var version = false
 var prettyPrint = false
 
+var forceColor = false
+var forceNoColor = false
+var colorsEnabled = false
+
 // can be either "" (off), "extract" or "process"
 var frontMatter = ""
 
+var splitFileExp = ""
+var splitFileExpFile = ""
+
 var completedSuccessfully = false
+
+var forceExpression = ""
+
+var expressionFile = ""

cmd/evaluate_all_command.go

@@ -2,8 +2,6 @@ package cmd
 
 import (
 	"errors"
-	"fmt"
-	"os"
 
 	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"github.com/spf13/cobra"
@@ -14,8 +12,14 @@ func createEvaluateAllCommand() *cobra.Command {
 		Use:     "eval-all [expression] [yaml_file1]...",
 		Aliases: []string{"ea"},
 		Short:   "Loads _all_ yaml documents of _all_ yaml files and runs expression once",
+		ValidArgsFunction: func(_ *cobra.Command, args []string, _ string) ([]string, cobra.ShellCompDirective) {
+			if len(args) == 0 {
+				return nil, cobra.ShellCompDirectiveNoFileComp
+			}
+			return nil, cobra.ShellCompDirectiveDefault
+		},
 		Example: `
-# Merge f2.yml into f1.yml (inplace)
+# Merge f2.yml into f1.yml (in place)
 yq eval-all --inplace 'select(fileIndex == 0) * select(fileIndex == 1)' f1.yml f2.yml
 ## the same command and expression using shortened names:
 yq ea -i 'select(fi == 0) * select(fi == 1)' f1.yml f2.yml
@@ -28,7 +32,7 @@ yq ea '. as $item ireduce ({}; . * $item )' file1.yml file2.yml ...
 ## use '-' as a filename to pipe from STDIN
 cat file2.yml | yq ea '.a.b' file1.yml - file3.yml
 `,
-		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
+		Long: `yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) 
 See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.
 
 ## Evaluate All ##
@@ -40,71 +44,69 @@ Note that it consumes more memory than eval.
 	}
 	return cmdEvalAll
 }
-func evaluateAll(cmd *cobra.Command, args []string) error {
-	cmd.SilenceUsage = true
+func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 	// 0 args, read std in
 	// 1 arg, null input, process expression
 	// 1 arg, read file in sequence
 	// 2+ args, [0] = expression, file the rest
 
 	var err error
-	stat, _ := os.Stdin.Stat()
-	pipingStdIn := (stat.Mode() & os.ModeCharDevice) == 0
+
+	expression, args, err := initCommand(cmd, args)
+	if err != nil {
+		return err
+	}
 
 	out := cmd.OutOrStdout()
 
-	fileInfo, _ := os.Stdout.Stat()
-
-	if forceColor || (!forceNoColor && (fileInfo.Mode()&os.ModeCharDevice) != 0) {
-		colorsEnabled = true
-	}
-
-	firstFileIndex := -1
-	if !nullInput && len(args) == 1 {
-		firstFileIndex = 0
-	} else if len(args) > 1 {
-		firstFileIndex = 1
-	}
-
-	if writeInplace && (firstFileIndex == -1) {
-		return fmt.Errorf("Write inplace flag only applicable when giving an expression and at least one file")
-	}
-
 	if writeInplace {
 		// only use colors if its forced
 		colorsEnabled = forceColor
-		writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[firstFileIndex])
+		writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[0])
 		out, err = writeInPlaceHandler.CreateTempFile()
 		if err != nil {
 			return err
 		}
 		// need to indirectly call the function so  that completedSuccessfully is
 		// passed when we finish execution as opposed to now
-		defer func() { writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully) }()
+		defer func() {
+			if cmdError == nil {
+				cmdError = writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully)
+			}
+		}()
 	}
 
-	if nullInput && len(args) > 1 {
-		return errors.New("Cannot pass files in when using null-input flag")
-	}
-	// backwards compatibilty
-	if outputToJSON {
-		outputFormat = "json"
-	}
-
-	format, err := yqlib.OutputFormatFromString(outputFormat)
+	format, err := yqlib.FormatFromString(outputFormat)
 	if err != nil {
 		return err
 	}
 
-	printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
+	decoder, err := configureDecoder(true)
+	if err != nil {
+		return err
+	}
+
+	printerWriter, err := configurePrinterWriter(format, out)
+	if err != nil {
+		return err
+	}
+	encoder, err := configureEncoder()
+	if err != nil {
+		return err
+	}
+
+	printer := yqlib.NewPrinter(encoder, printerWriter)
+	if nulSepOutput {
+		printer.SetNulSepOutput(true)
+	}
 
 	if frontMatter != "" {
-		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])
+		frontMatterHandler := yqlib.NewFrontMatterHandler(args[0])
 		err = frontMatterHandler.Split()
 		if err != nil {
 			return err
 		}
-		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
+		args[0] = frontMatterHandler.GetYamlFrontMatterFilename()
 
 		if frontMatter == "process" {
 			reader := frontMatterHandler.GetContentReader()
@@ -115,22 +117,17 @@ func evaluateAll(cmd *cobra.Command, args []string) error {
 	}
 
 	allAtOnceEvaluator := yqlib.NewAllAtOnceEvaluator()
+
 	switch len(args) {
 	case 0:
-		if pipingStdIn {
-			err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer, leadingContentPreProcessing)
+		if nullInput {
+			err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(expression), printer)
 		} else {
 			cmd.Println(cmd.UsageString())
 			return nil
 		}
-	case 1:
-		if nullInput {
-			err = yqlib.NewStreamEvaluator().EvaluateNew(processExpression(args[0]), printer, "")
-		} else {
-			err = allAtOnceEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer, leadingContentPreProcessing)
-		}
 	default:
-		err = allAtOnceEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing)
+		err = allAtOnceEvaluator.EvaluateFiles(processExpression(expression), args, printer, decoder)
 	}
 
 	completedSuccessfully = err == nil

cmd/evaluate_all_command_test.go

@@ -0,0 +1,328 @@
+package cmd
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestCreateEvaluateAllCommand(t *testing.T) {
+	cmd := createEvaluateAllCommand()
+
+	if cmd == nil {
+		t.Fatal("createEvaluateAllCommand returned nil")
+	}
+
+	// Test basic command properties
+	if cmd.Use != "eval-all [expression] [yaml_file1]..." {
+		t.Errorf("Expected Use to be 'eval-all [expression] [yaml_file1]...', got %q", cmd.Use)
+	}
+
+	if cmd.Short == "" {
+		t.Error("Expected Short description to be non-empty")
+	}
+
+	if cmd.Long == "" {
+		t.Error("Expected Long description to be non-empty")
+	}
+
+	// Test aliases
+	expectedAliases := []string{"ea"}
+	if len(cmd.Aliases) != len(expectedAliases) {
+		t.Errorf("Expected %d aliases, got %d", len(expectedAliases), len(cmd.Aliases))
+	}
+
+	for i, expected := range expectedAliases {
+		if i >= len(cmd.Aliases) || cmd.Aliases[i] != expected {
+			t.Errorf("Expected alias %d to be %q, got %q", i, expected, cmd.Aliases[i])
+		}
+	}
+}
+
+func TestEvaluateAll_NoArgs(t *testing.T) {
+	// Create a temporary command
+	cmd := createEvaluateAllCommand()
+
+	// Set up command to capture output
+	var output bytes.Buffer
+	cmd.SetOut(&output)
+
+	// Test with no arguments and no null input
+	nullInput = false
+	defer func() { nullInput = false }()
+
+	err := evaluateAll(cmd, []string{})
+
+	// Should not error, but should print usage
+	if err != nil {
+		t.Errorf("evaluateAll with no args should not error, got: %v", err)
+	}
+
+	// Should have printed usage information
+	if output.Len() == 0 {
+		t.Error("Expected usage information to be printed")
+	}
+}
+
+func TestEvaluateAll_NullInput(t *testing.T) {
+	// Create a temporary command
+	cmd := createEvaluateAllCommand()
+
+	// Set up command to capture output
+	var output bytes.Buffer
+	cmd.SetOut(&output)
+
+	// Test with null input
+	nullInput = true
+	defer func() { nullInput = false }()
+
+	err := evaluateAll(cmd, []string{})
+
+	// Should not error when using null input
+	if err != nil {
+		t.Errorf("evaluateAll with null input should not error, got: %v", err)
+	}
+}
+
+func TestEvaluateAll_WithSingleFile(t *testing.T) {
+	// Create a temporary YAML file
+	tempDir := t.TempDir()
+	yamlFile := filepath.Join(tempDir, "test.yaml")
+	yamlContent := []byte("name: test\nage: 25\n")
+	err := os.WriteFile(yamlFile, yamlContent, 0600)
+	if err != nil {
+		t.Fatalf("Failed to create test YAML file: %v", err)
+	}
+
+	// Create a temporary command
+	cmd := createEvaluateAllCommand()
+
+	// Set up command to capture output
+	var output bytes.Buffer
+	cmd.SetOut(&output)
+
+	// Test with a single file
+	err = evaluateAll(cmd, []string{yamlFile})
+
+	// Should not error
+	if err != nil {
+		t.Errorf("evaluateAll with single file should not error, got: %v", err)
+	}
+
+	// Should have some output
+	if output.Len() == 0 {
+		t.Error("Expected output from evaluateAll with single file")
+	}
+}
+
+func TestEvaluateAll_WithMultipleFiles(t *testing.T) {
+	// Create temporary YAML files
+	tempDir := t.TempDir()
+
+	yamlFile1 := filepath.Join(tempDir, "test1.yaml")
+	yamlContent1 := []byte("name: test1\nage: 25\n")
+	err := os.WriteFile(yamlFile1, yamlContent1, 0600)
+	if err != nil {
+		t.Fatalf("Failed to create test YAML file 1: %v", err)
+	}
+
+	yamlFile2 := filepath.Join(tempDir, "test2.yaml")
+	yamlContent2 := []byte("name: test2\nage: 30\n")
+	err = os.WriteFile(yamlFile2, yamlContent2, 0600)
+	if err != nil {
+		t.Fatalf("Failed to create test YAML file 2: %v", err)
+	}
+
+	// Create a temporary command
+	cmd := createEvaluateAllCommand()
+
+	// Set up command to capture output
+	var output bytes.Buffer
+	cmd.SetOut(&output)
+
+	// Test with multiple files
+	err = evaluateAll(cmd, []string{yamlFile1, yamlFile2})
+
+	// Should not error
+	if err != nil {
+		t.Errorf("evaluateAll with multiple files should not error, got: %v", err)
+	}
+
+	// Should have output
+	if output.Len() == 0 {
+		t.Error("Expected output from evaluateAll with multiple files")
+	}
+}
+
+func TestEvaluateAll_WithExpression(t *testing.T) {
+	// Create a temporary YAML file
+	tempDir := t.TempDir()
+	yamlFile := filepath.Join(tempDir, "test.yaml")
+	yamlContent := []byte("name: test\nage: 25\n")
+	err := os.WriteFile(yamlFile, yamlContent, 0600)
+	if err != nil {
+		t.Fatalf("Failed to create test YAML file: %v", err)
+	}
+
+	// Create a temporary command
+	cmd := createEvaluateAllCommand()
+
+	// Set up command to capture output
+	var output bytes.Buffer
+	cmd.SetOut(&output)
+
+	// Test with expression
+	err = evaluateAll(cmd, []string{".name", yamlFile})
+
+	// Should not error
+	if err != nil {
+		t.Errorf("evaluateAll with expression should not error, got: %v", err)
+	}
+
+	// Should have output
+	if output.Len() == 0 {
+		t.Error("Expected output from evaluateAll with expression")
+	}
+}
+
+func TestEvaluateAll_WriteInPlace(t *testing.T) {
+	// Create a temporary YAML file
+	tempDir := t.TempDir()
+	yamlFile := filepath.Join(tempDir, "test.yaml")
+	yamlContent := []byte("name: test\nage: 25\n")
+	err := os.WriteFile(yamlFile, yamlContent, 0600)
+	if err != nil {
+		t.Fatalf("Failed to create test YAML file: %v", err)
+	}
+
+	// Create a temporary command
+	cmd := createEvaluateAllCommand()
+
+	// Set up command to capture output
+	var output bytes.Buffer
+	cmd.SetOut(&output)
+
+	// Enable write in place
+	originalWriteInplace := writeInplace
+	writeInplace = true
+	defer func() { writeInplace = originalWriteInplace }()
+
+	// Test with write in place
+	err = evaluateAll(cmd, []string{".name = \"updated\"", yamlFile})
+
+	// Should not error
+	if err != nil {
+		t.Errorf("evaluateAll with write in place should not error, got: %v", err)
+	}
+
+	// Verify the file was updated
+	updatedContent, err := os.ReadFile(yamlFile)
+	if err != nil {
+		t.Fatalf("Failed to read updated file: %v", err)
+	}
+
+	// Should contain the updated content
+	if !strings.Contains(string(updatedContent), "updated") {
+		t.Errorf("Expected file to contain 'updated', got: %s", string(updatedContent))
+	}
+}
+
+func TestEvaluateAll_ExitStatus(t *testing.T) {
+	// Create a temporary YAML file
+	tempDir := t.TempDir()
+	yamlFile := filepath.Join(tempDir, "test.yaml")
+	yamlContent := []byte("name: test\nage: 25\n")
+	err := os.WriteFile(yamlFile, yamlContent, 0600)
+	if err != nil {
+		t.Fatalf("Failed to create test YAML file: %v", err)
+	}
+
+	// Create a temporary command
+	cmd := createEvaluateAllCommand()
+
+	// Set up command to capture output
+	var output bytes.Buffer
+	cmd.SetOut(&output)
+
+	// Enable exit status
+	originalExitStatus := exitStatus
+	exitStatus = true
+	defer func() { exitStatus = originalExitStatus }()
+
+	// Test with expression that should find no matches
+	err = evaluateAll(cmd, []string{".nonexistent", yamlFile})
+
+	// Should error when no matches found and exit status is enabled
+	if err == nil {
+		t.Error("Expected error when no matches found and exit status is enabled")
+	}
+}
+
+func TestEvaluateAll_WithMultipleDocuments(t *testing.T) {
+	// Create a temporary YAML file with multiple documents
+	tempDir := t.TempDir()
+	yamlFile := filepath.Join(tempDir, "test.yaml")
+	yamlContent := []byte("---\nname: doc1\nage: 25\n---\nname: doc2\nage: 30\n")
+	err := os.WriteFile(yamlFile, yamlContent, 0600)
+	if err != nil {
+		t.Fatalf("Failed to create test YAML file: %v", err)
+	}
+
+	// Create a temporary command
+	cmd := createEvaluateAllCommand()
+
+	// Set up command to capture output
+	var output bytes.Buffer
+	cmd.SetOut(&output)
+
+	// Test with multiple documents
+	err = evaluateAll(cmd, []string{".", yamlFile})
+
+	// Should not error
+	if err != nil {
+		t.Errorf("evaluateAll with multiple documents should not error, got: %v", err)
+	}
+
+	// Should have output
+	if output.Len() == 0 {
+		t.Error("Expected output from evaluateAll with multiple documents")
+	}
+}
+
+func TestEvaluateAll_NulSepOutput(t *testing.T) {
+	// Create a temporary YAML file
+	tempDir := t.TempDir()
+	yamlFile := filepath.Join(tempDir, "test.yaml")
+	yamlContent := []byte("name: test\nage: 25\n")
+	err := os.WriteFile(yamlFile, yamlContent, 0600)
+	if err != nil {
+		t.Fatalf("Failed to create test YAML file: %v", err)
+	}
+
+	// Create a temporary command
+	cmd := createEvaluateAllCommand()
+
+	// Set up command to capture output
+	var output bytes.Buffer
+	cmd.SetOut(&output)
+
+	// Enable nul separator output
+	originalNulSepOutput := nulSepOutput
+	nulSepOutput = true
+	defer func() { nulSepOutput = originalNulSepOutput }()
+
+	// Test with nul separator output
+	err = evaluateAll(cmd, []string{".name", yamlFile})
+
+	// Should not error
+	if err != nil {
+		t.Errorf("evaluateAll with nul separator output should not error, got: %v", err)
+	}
+
+	// Should have output
+	if output.Len() == 0 {
+		t.Error("Expected output from evaluateAll with nul separator output")
+	}
+}

cmd/evaluate_sequence_command.go

@@ -3,7 +3,6 @@ package cmd
 import (
 	"errors"
 	"fmt"
-	"os"
 
 	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"github.com/spf13/cobra"
@@ -13,7 +12,13 @@ func createEvaluateSequenceCommand() *cobra.Command {
 	var cmdEvalSequence = &cobra.Command{
 		Use:     "eval [expression] [yaml_file1]...",
 		Aliases: []string{"e"},
-		Short:   "Apply the expression to each document in each yaml file in sequence",
+		Short:   "(default) Apply the expression to each document in each yaml file in sequence",
+		ValidArgsFunction: func(_ *cobra.Command, args []string, _ string) ([]string, cobra.ShellCompDirective) {
+			if len(args) == 0 {
+				return nil, cobra.ShellCompDirectiveNoFileComp
+			}
+			return nil, cobra.ShellCompDirectiveDefault
+		},
 		Example: `
 # Reads field under the given path for each file
 yq e '.a.b' f1.yml f2.yml 
@@ -29,10 +34,10 @@ cat file2.yml | yq e '.a.b' file1.yml - file3.yml
 ## Note that editing an empty file does not work.
 yq e -n '.a.b.c = "cat"' 
 
-# Update a file inplace
+# Update a file in place
 yq e '.a.b = "cool"' -i file.yaml 
 `,
-		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
+		Long: `yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) 
 See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.
 
 ## Evaluate Sequence ##
@@ -44,83 +49,85 @@ expression and prints the result in sequence.`,
 }
 
 func processExpression(expression string) string {
-	var prettyPrintExp = `(... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not))  ) style=""`
+
 	if prettyPrint && expression == "" {
-		return prettyPrintExp
+		return yqlib.PrettyPrintExp
 	} else if prettyPrint {
-		return fmt.Sprintf("%v | %v", expression, prettyPrintExp)
+		return fmt.Sprintf("%v | %v", expression, yqlib.PrettyPrintExp)
 	}
 	return expression
 }
 
-func evaluateSequence(cmd *cobra.Command, args []string) error {
-	cmd.SilenceUsage = true
+func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 	// 0 args, read std in
 	// 1 arg, null input, process expression
 	// 1 arg, read file in sequence
 	// 2+ args, [0] = expression, file the rest
 
-	var err error
-	stat, _ := os.Stdin.Stat()
-	pipingStdIn := (stat.Mode() & os.ModeCharDevice) == 0
-
 	out := cmd.OutOrStdout()
 
-	fileInfo, _ := os.Stdout.Stat()
+	var err error
 
-	if forceColor || (!forceNoColor && (fileInfo.Mode()&os.ModeCharDevice) != 0) {
-		colorsEnabled = true
-	}
-
-	firstFileIndex := -1
-	if !nullInput && len(args) == 1 {
-		firstFileIndex = 0
-	} else if len(args) > 1 {
-		firstFileIndex = 1
-	}
-
-	if writeInplace && (firstFileIndex == -1) {
-		return fmt.Errorf("Write inplace flag only applicable when giving an expression and at least one file")
+	expression, args, err := initCommand(cmd, args)
+	if err != nil {
+		return err
 	}
 
 	if writeInplace {
 		// only use colors if its forced
 		colorsEnabled = forceColor
-		writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[firstFileIndex])
+		writeInPlaceHandler := yqlib.NewWriteInPlaceHandler(args[0])
 		out, err = writeInPlaceHandler.CreateTempFile()
 		if err != nil {
 			return err
 		}
 		// need to indirectly call the function so  that completedSuccessfully is
 		// passed when we finish execution as opposed to now
-		defer func() { writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully) }()
+		defer func() {
+			if cmdError == nil {
+				cmdError = writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully)
+			}
+		}()
 	}
 
-	// backwards compatibilty
-	if outputToJSON {
-		outputFormat = "json"
-	}
-
-	format, err := yqlib.OutputFormatFromString(outputFormat)
+	format, err := yqlib.FormatFromString(outputFormat)
 	if err != nil {
 		return err
 	}
 
-	printer := yqlib.NewPrinter(out, format, unwrapScalar, colorsEnabled, indent, !noDocSeparators)
-
-	streamEvaluator := yqlib.NewStreamEvaluator()
-
-	if nullInput && len(args) > 1 {
-		return errors.New("Cannot pass files in when using null-input flag")
+	printerWriter, err := configurePrinterWriter(format, out)
+	if err != nil {
+		return err
+	}
+	encoder, err := configureEncoder()
+	if err != nil {
+		return err
 	}
 
+	printer := yqlib.NewPrinter(encoder, printerWriter)
+
+	if printNodeInfo {
+		printer = yqlib.NewNodeInfoPrinter(printerWriter)
+	}
+
+	if nulSepOutput {
+		printer.SetNulSepOutput(true)
+	}
+
+	decoder, err := configureDecoder(false)
+	if err != nil {
+		return err
+	}
+	streamEvaluator := yqlib.NewStreamEvaluator()
+
 	if frontMatter != "" {
-		frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex])
+		yqlib.GetLogger().Debug("using front matter handler")
+		frontMatterHandler := yqlib.NewFrontMatterHandler(args[0])
 		err = frontMatterHandler.Split()
 		if err != nil {
 			return err
 		}
-		args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename()
+		args[0] = frontMatterHandler.GetYamlFrontMatterFilename()
 
 		if frontMatter == "process" {
 			reader := frontMatterHandler.GetContentReader()
@@ -132,20 +139,14 @@ func evaluateSequence(cmd *cobra.Command, args []string) error {
 
 	switch len(args) {
 	case 0:
-		if pipingStdIn {
-			err = streamEvaluator.EvaluateFiles(processExpression(""), []string{"-"}, printer, leadingContentPreProcessing)
+		if nullInput {
+			err = streamEvaluator.EvaluateNew(processExpression(expression), printer)
 		} else {
 			cmd.Println(cmd.UsageString())
 			return nil
 		}
-	case 1:
-		if nullInput {
-			err = streamEvaluator.EvaluateNew(processExpression(args[0]), printer, "")
-		} else {
-			err = streamEvaluator.EvaluateFiles(processExpression(""), []string{args[0]}, printer, leadingContentPreProcessing)
-		}
 	default:
-		err = streamEvaluator.EvaluateFiles(processExpression(args[0]), args[1:], printer, leadingContentPreProcessing)
+		err = streamEvaluator.EvaluateFiles(processExpression(expression), args, printer, decoder)
 	}
 	completedSuccessfully = err == nil
 

cmd/evaluate_sequence_command_test.go

@@ -0,0 +1,276 @@
+package cmd
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+)
+
+func TestCreateEvaluateSequenceCommand(t *testing.T) {
+	cmd := createEvaluateSequenceCommand()
+
+	if cmd == nil {
+		t.Fatal("createEvaluateSequenceCommand returned nil")
+	}
+
+	// Test basic command properties
+	if cmd.Use != "eval [expression] [yaml_file1]..." {
+		t.Errorf("Expected Use to be 'eval [expression] [yaml_file1]...', got %q", cmd.Use)
+	}
+
+	if cmd.Short == "" {
+		t.Error("Expected Short description to be non-empty")
+	}
+
+	if cmd.Long == "" {
+		t.Error("Expected Long description to be non-empty")
+	}
+
+	// Test aliases
+	expectedAliases := []string{"e"}
+	if len(cmd.Aliases) != len(expectedAliases) {
+		t.Errorf("Expected %d aliases, got %d", len(expectedAliases), len(cmd.Aliases))
+	}
+
+	for i, expected := range expectedAliases {
+		if i >= len(cmd.Aliases) || cmd.Aliases[i] != expected {
+			t.Errorf("Expected alias %d to be %q, got %q", i, expected, cmd.Aliases[i])
+		}
+	}
+}
+
+func TestProcessExpression(t *testing.T) {
+	// Reset global variables
+	originalPrettyPrint := prettyPrint
+	defer func() { prettyPrint = originalPrettyPrint }()
+
+	tests := []struct {
+		name        string
+		prettyPrint bool
+		expression  string
+		expected    string
+	}{
+		{
+			name:        "empty expression without pretty print",
+			prettyPrint: false,
+			expression:  "",
+			expected:    "",
+		},
+		{
+			name:        "empty expression with pretty print",
+			prettyPrint: true,
+			expression:  "",
+			expected:    `(... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not))  ) style=""`,
+		},
+		{
+			name:        "simple expression without pretty print",
+			prettyPrint: false,
+			expression:  ".a.b",
+			expected:    ".a.b",
+		},
+		{
+			name:        "simple expression with pretty print",
+			prettyPrint: true,
+			expression:  ".a.b",
+			expected:    `.a.b | (... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not))  ) style=""`,
+		},
+		{
+			name:        "complex expression with pretty print",
+			prettyPrint: true,
+			expression:  ".items[] | select(.active == true)",
+			expected:    `.items[] | select(.active == true) | (... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not))  ) style=""`,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			prettyPrint = tt.prettyPrint
+			result := processExpression(tt.expression)
+			if result != tt.expected {
+				t.Errorf("processExpression(%q) = %q, want %q", tt.expression, result, tt.expected)
+			}
+		})
+	}
+}
+
+func TestEvaluateSequence_NoArgs(t *testing.T) {
+	// Create a temporary command
+	cmd := createEvaluateSequenceCommand()
+
+	// Set up command to capture output
+	var output bytes.Buffer
+	cmd.SetOut(&output)
+
+	// Test with no arguments and no null input
+	nullInput = false
+	defer func() { nullInput = false }()
+
+	err := evaluateSequence(cmd, []string{})
+
+	// Should not error, but should print usage
+	if err != nil {
+		t.Errorf("evaluateSequence with no args should not error, got: %v", err)
+	}
+
+	// Should have printed usage information
+	if output.Len() == 0 {
+		t.Error("Expected usage information to be printed")
+	}
+}
+
+func TestEvaluateSequence_NullInput(t *testing.T) {
+	// Create a temporary command
+	cmd := createEvaluateSequenceCommand()
+
+	// Set up command to capture output
+	var output bytes.Buffer
+	cmd.SetOut(&output)
+
+	// Test with null input
+	nullInput = true
+	defer func() { nullInput = false }()
+
+	err := evaluateSequence(cmd, []string{})
+
+	// Should not error when using null input
+	if err != nil {
+		t.Errorf("evaluateSequence with null input should not error, got: %v", err)
+	}
+}
+
+func TestEvaluateSequence_WithFile(t *testing.T) {
+	// Create a temporary YAML file
+	tempDir := t.TempDir()
+	yamlFile := filepath.Join(tempDir, "test.yaml")
+	yamlContent := []byte("name: test\nage: 25\n")
+	err := os.WriteFile(yamlFile, yamlContent, 0600)
+	if err != nil {
+		t.Fatalf("Failed to create test YAML file: %v", err)
+	}
+
+	// Create a temporary command
+	cmd := createEvaluateSequenceCommand()
+
+	// Set up command to capture output
+	var output bytes.Buffer
+	cmd.SetOut(&output)
+
+	// Test with a file
+	err = evaluateSequence(cmd, []string{yamlFile})
+
+	// Should not error
+	if err != nil {
+		t.Errorf("evaluateSequence with file should not error, got: %v", err)
+	}
+
+	// Should have some output
+	if output.Len() == 0 {
+		t.Error("Expected output from evaluateSequence with file")
+	}
+}
+
+func TestEvaluateSequence_WithExpressionAndFile(t *testing.T) {
+	// Create a temporary YAML file
+	tempDir := t.TempDir()
+	yamlFile := filepath.Join(tempDir, "test.yaml")
+	yamlContent := []byte("name: test\nage: 25\n")
+	err := os.WriteFile(yamlFile, yamlContent, 0600)
+	if err != nil {
+		t.Fatalf("Failed to create test YAML file: %v", err)
+	}
+
+	// Create a temporary command
+	cmd := createEvaluateSequenceCommand()
+
+	// Set up command to capture output
+	var output bytes.Buffer
+	cmd.SetOut(&output)
+
+	// Test with expression and file
+	err = evaluateSequence(cmd, []string{".name", yamlFile})
+
+	// Should not error
+	if err != nil {
+		t.Errorf("evaluateSequence with expression and file should not error, got: %v", err)
+	}
+
+	// Should have output
+	if output.Len() == 0 {
+		t.Error("Expected output from evaluateSequence with expression and file")
+	}
+}
+
+func TestEvaluateSequence_WriteInPlace(t *testing.T) {
+	// Create a temporary YAML file
+	tempDir := t.TempDir()
+	yamlFile := filepath.Join(tempDir, "test.yaml")
+	yamlContent := []byte("name: test\nage: 25\n")
+	err := os.WriteFile(yamlFile, yamlContent, 0600)
+	if err != nil {
+		t.Fatalf("Failed to create test YAML file: %v", err)
+	}
+
+	// Create a temporary command
+	cmd := createEvaluateSequenceCommand()
+
+	// Set up command to capture output
+	var output bytes.Buffer
+	cmd.SetOut(&output)
+
+	// Enable write in place
+	originalWriteInplace := writeInplace
+	writeInplace = true
+	defer func() { writeInplace = originalWriteInplace }()
+
+	// Test with write in place
+	err = evaluateSequence(cmd, []string{".name = \"updated\"", yamlFile})
+
+	// Should not error
+	if err != nil {
+		t.Errorf("evaluateSequence with write in place should not error, got: %v", err)
+	}
+
+	// Verify the file was updated
+	updatedContent, err := os.ReadFile(yamlFile)
+	if err != nil {
+		t.Fatalf("Failed to read updated file: %v", err)
+	}
+
+	// Should contain the updated content
+	if !strings.Contains(string(updatedContent), "updated") {
+		t.Errorf("Expected file to contain 'updated', got: %s", string(updatedContent))
+	}
+}
+
+func TestEvaluateSequence_ExitStatus(t *testing.T) {
+	// Create a temporary YAML file
+	tempDir := t.TempDir()
+	yamlFile := filepath.Join(tempDir, "test.yaml")
+	yamlContent := []byte("name: test\nage: 25\n")
+	err := os.WriteFile(yamlFile, yamlContent, 0600)
+	if err != nil {
+		t.Fatalf("Failed to create test YAML file: %v", err)
+	}
+
+	// Create a temporary command
+	cmd := createEvaluateSequenceCommand()
+
+	// Set up command to capture output
+	var output bytes.Buffer
+	cmd.SetOut(&output)
+
+	// Enable exit status
+	originalExitStatus := exitStatus
+	exitStatus = true
+	defer func() { exitStatus = originalExitStatus }()
+
+	// Test with expression that should find no matches
+	err = evaluateSequence(cmd, []string{".nonexistent", yamlFile})
+
+	// Should error when no matches found and exit status is enabled
+	if err == nil {
+		t.Error("Expected error when no matches found and exit status is enabled")
+	}
+}

cmd/root.go

@@ -1,46 +1,105 @@
 package cmd
 
 import (
+	"fmt"
 	"os"
+	"strings"
 
+	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"github.com/spf13/cobra"
 	logging "gopkg.in/op/go-logging.v1"
 )
 
+type runeValue rune
+
+func newRuneVar(p *rune) *runeValue {
+	return (*runeValue)(p)
+}
+
+func (r *runeValue) String() string {
+	return string(*r)
+}
+
+func (r *runeValue) Set(rawVal string) error {
+	val := strings.ReplaceAll(rawVal, "\\n", "\n")
+	val = strings.ReplaceAll(val, "\\t", "\t")
+	val = strings.ReplaceAll(val, "\\r", "\r")
+	val = strings.ReplaceAll(val, "\\f", "\f")
+	val = strings.ReplaceAll(val, "\\v", "\v")
+	if len(val) != 1 {
+		return fmt.Errorf("[%v] is not a valid character. Must be length 1 was %v", val, len(val))
+	}
+	*r = runeValue(rune(val[0]))
+	return nil
+}
+
+func (r *runeValue) Type() string {
+	return "char"
+}
+
 func New() *cobra.Command {
 	var rootCmd = &cobra.Command{
 		Use:   "yq",
-		Short: "yq is a lightweight and portable command-line YAML processor.",
-		Long: `yq is a portable command-line YAML processor (https://github.com/mikefarah/yq/) 
+		Short: "yq is a lightweight and portable command-line data file processor.",
+		Long: `yq is a portable command-line data file processor (https://github.com/mikefarah/yq/) 
 See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
+		Example: `
+# yq tries to auto-detect the file format based off the extension, and defaults to YAML if it's unknown (or piping through STDIN)
+# Use the '-p/--input-format' flag to specify a format type.
+cat file.xml | yq -p xml
+
+# read the "stuff" node from "myfile.yml"
+yq '.stuff' < myfile.yml
+
+# update myfile.yml in place
+yq -i '.stuff = "foo"' myfile.yml
+
+# print contents of sample.json as idiomatic YAML
+yq -P -oy sample.json
+`,
+
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if version {
 				cmd.Print(GetVersionDisplay())
 				return nil
 			}
-			cmd.Println(cmd.UsageString())
-			return nil
+			return evaluateSequence(cmd, args)
 
 		},
-		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+		PersistentPreRunE: func(cmd *cobra.Command, _ []string) error {
 			cmd.SetOut(cmd.OutOrStdout())
-			var format = logging.MustStringFormatter(
-				`%{color}%{time:15:04:05} %{shortfunc} [%{level:.4s}]%{color:reset} %{message}`,
-			)
+			level := logging.WARNING
+			stringFormat := `[%{level}] %{color}%{time:15:04:05}%{color:reset} %{message}`
+
+			// when NO_COLOR environment variable presents and not an empty string the coloured output should be disabled;
+			// refer to no-color.org
+			forceNoColor = forceNoColor || os.Getenv("NO_COLOR") != ""
+
+			if verbose && forceNoColor {
+				level = logging.DEBUG
+				stringFormat = `[%{level:5.5s}] %{time:15:04:05} %{shortfile:-33s} %{shortfunc:-25s} %{message}`
+			} else if verbose {
+				level = logging.DEBUG
+				stringFormat = `[%{level:5.5s}] %{color}%{time:15:04:05}%{color:bold} %{shortfile:-33s} %{shortfunc:-25s}%{color:reset} %{message}`
+			} else if forceNoColor {
+				stringFormat = `[%{level}] %{time:15:04:05} %{message}`
+			}
+
+			var format = logging.MustStringFormatter(stringFormat)
 			var backend = logging.AddModuleLevel(
 				logging.NewBackendFormatter(logging.NewLogBackend(os.Stderr, "", 0), format))
 
-			if verbose {
-				backend.SetLevel(logging.DEBUG, "")
-			} else {
-				backend.SetLevel(logging.ERROR, "")
-			}
+			backend.SetLevel(level, "")
 
 			logging.SetBackend(backend)
+			yqlib.InitExpressionParser()
+
+			return nil
 		},
 	}
 
 	rootCmd.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "verbose mode")
+	rootCmd.PersistentFlags().BoolVarP(&printNodeInfo, "debug-node-info", "", false, "debug node info")
 
 	rootCmd.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "(deprecated) output as json. Set indent to 0 to print json in one line.")
 	err := rootCmd.PersistentFlags().MarkDeprecated("tojson", "please use -o=json instead")
@@ -48,21 +107,120 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`,
 		panic(err)
 	}
 
-	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "yaml", "[yaml|y|json|j|props|p] output format type.")
-	rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating yaml docs from scratch.")
+	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", fmt.Sprintf("[auto|a|%v] output format type.", yqlib.GetAvailableOutputFormatString()))
+	var outputCompletions = []string{"auto"}
+	for _, formats := range yqlib.GetAvailableOutputFormats() {
+		outputCompletions = append(outputCompletions, formats.FormalName)
+	}
+
+	if err = rootCmd.RegisterFlagCompletionFunc("output-format", cobra.FixedCompletions(outputCompletions, cobra.ShellCompDirectiveNoFileComp)); err != nil {
+		panic(err)
+	}
+	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", fmt.Sprintf("[auto|a|%v] parse format for input.", yqlib.GetAvailableInputFormatString()))
+
+	var inputCompletions = []string{"auto"}
+	for _, formats := range yqlib.GetAvailableInputFormats() {
+		inputCompletions = append(inputCompletions, formats.FormalName)
+	}
+
+	if err = rootCmd.RegisterFlagCompletionFunc("input-format", cobra.FixedCompletions(inputCompletions, cobra.ShellCompDirectiveNoFileComp)); err != nil {
+		panic(err)
+	}
+
+	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
+	if err = rootCmd.RegisterFlagCompletionFunc("xml-attribute-prefix", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
+	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")
+	if err = rootCmd.RegisterFlagCompletionFunc("xml-content-name", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.StrictMode, "xml-strict-mode", yqlib.ConfiguredXMLPreferences.StrictMode, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.KeepNamespace, "xml-keep-namespace", yqlib.ConfiguredXMLPreferences.KeepNamespace, "enables keeping namespace after parsing attributes")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.UseRawToken, "xml-raw-token", yqlib.ConfiguredXMLPreferences.UseRawToken, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")
+	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ProcInstPrefix, "xml-proc-inst-prefix", yqlib.ConfiguredXMLPreferences.ProcInstPrefix, "prefix for xml processing instructions (e.g. <?xml version=\"1\"?>)")
+	if err = rootCmd.RegisterFlagCompletionFunc("xml-proc-inst-prefix", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
+	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.DirectiveName, "xml-directive-name", yqlib.ConfiguredXMLPreferences.DirectiveName, "name for xml directives (e.g. <!DOCTYPE thing cat>)")
+	if err = rootCmd.RegisterFlagCompletionFunc("xml-directive-name", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", yqlib.ConfiguredXMLPreferences.SkipProcInst, "skip over process instructions (e.g. <?xml version=\"1\"?>)")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", yqlib.ConfiguredXMLPreferences.SkipDirectives, "skip over directives (e.g. <!DOCTYPE thing cat>)")
+
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredCsvPreferences.AutoParse, "csv-auto-parse", yqlib.ConfiguredCsvPreferences.AutoParse, "parse CSV YAML/JSON values")
+	rootCmd.PersistentFlags().Var(newRuneVar(&yqlib.ConfiguredCsvPreferences.Separator), "csv-separator", "CSV Separator character")
+
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredTsvPreferences.AutoParse, "tsv-auto-parse", yqlib.ConfiguredTsvPreferences.AutoParse, "parse TSV YAML/JSON values")
+
+	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocPrefix, "lua-prefix", yqlib.ConfiguredLuaPreferences.DocPrefix, "prefix")
+	if err = rootCmd.RegisterFlagCompletionFunc("lua-prefix", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
+	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocSuffix, "lua-suffix", yqlib.ConfiguredLuaPreferences.DocSuffix, "suffix")
+	if err = rootCmd.RegisterFlagCompletionFunc("lua-suffix", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.UnquotedKeys, "lua-unquoted", yqlib.ConfiguredLuaPreferences.UnquotedKeys, "output unquoted string keys (e.g. {foo=\"bar\"})")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.Globals, "lua-globals", yqlib.ConfiguredLuaPreferences.Globals, "output keys as top-level global variables")
+
+	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, "properties-separator", yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, "separator to use between keys and values")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredPropertiesPreferences.UseArrayBrackets, "properties-array-brackets", yqlib.ConfiguredPropertiesPreferences.UseArrayBrackets, "use [x] in array paths (e.g. for SpringBoot)")
+
+	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredShellVariablesPreferences.KeySeparator, "shell-key-separator", yqlib.ConfiguredShellVariablesPreferences.KeySeparator, "separator for shell variable key paths")
+	if err = rootCmd.RegisterFlagCompletionFunc("shell-key-separator", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
+
+	rootCmd.PersistentFlags().BoolVar(&yqlib.StringInterpolationEnabled, "string-interpolation", yqlib.StringInterpolationEnabled, "Toggles strings interpolation of \\(exp)")
+
+	rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.")
 	rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)")
 
 	rootCmd.PersistentFlags().IntVarP(&indent, "indent", "I", 2, "sets indent level for output")
+	if err = rootCmd.RegisterFlagCompletionFunc("indent", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
 	rootCmd.Flags().BoolVarP(&version, "version", "V", false, "Print version information and quit")
-	rootCmd.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace of first yaml file given.")
-	rootCmd.PersistentFlags().BoolVarP(&unwrapScalar, "unwrapScalar", "", true, "unwrap scalar, print the value with no quotes, colors or comments")
+	rootCmd.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the file in place of first file given.")
+	rootCmd.PersistentFlags().VarP(unwrapScalarFlag, "unwrapScalar", "r", "unwrap scalar, print the value with no quotes, colors or comments. Defaults to true for yaml")
+	rootCmd.PersistentFlags().Lookup("unwrapScalar").NoOptDefVal = "true"
+	rootCmd.PersistentFlags().BoolVarP(&nulSepOutput, "nul-output", "0", false, "Use NUL char to separate values. If unwrap scalar is also set, fail if unwrapped scalar contains NUL char.")
+
 	rootCmd.PersistentFlags().BoolVarP(&prettyPrint, "prettyPrint", "P", false, "pretty print, shorthand for '... style = \"\"'")
 	rootCmd.PersistentFlags().BoolVarP(&exitStatus, "exit-status", "e", false, "set exit status if there are no matches or null or false is returned")
 
 	rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors")
-	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors")
+	rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", forceNoColor, "force print with no colors")
 	rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data intact")
-	rootCmd.PersistentFlags().BoolVarP(&leadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression. This is a workaround for go-yaml to persist header content properly.")
+	if err = rootCmd.RegisterFlagCompletionFunc("front-matter", cobra.FixedCompletions([]string{"extract", "process"}, cobra.ShellCompDirectiveNoFileComp)); err != nil {
+		panic(err)
+	}
+	rootCmd.PersistentFlags().StringVarP(&forceExpression, "expression", "", "", "forcibly set the expression argument. Useful when yq argument detection thinks your expression is a file.")
+	if err = rootCmd.RegisterFlagCompletionFunc("expression", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
+	rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredYamlPreferences.LeadingContentPreProcessing, "header-preprocess", "", true, "Slurp any header comments and separators before processing expression.")
+	rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredYamlPreferences.FixMergeAnchorToSpec, "yaml-fix-merge-anchor-to-spec", "", false, "Fix merge anchor to match YAML spec. Will default to true in late 2025")
+
+	rootCmd.PersistentFlags().StringVarP(&splitFileExp, "split-exp", "s", "", "print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter. The necessary directories will be created.")
+	if err = rootCmd.RegisterFlagCompletionFunc("split-exp", cobra.NoFileCompletions); err != nil {
+		panic(err)
+	}
+	rootCmd.PersistentFlags().StringVarP(&splitFileExpFile, "split-exp-file", "", "", "Use a file to specify the split-exp expression.")
+	if err = rootCmd.MarkPersistentFlagFilename("split-exp-file"); err != nil {
+		panic(err)
+	}
+
+	rootCmd.PersistentFlags().StringVarP(&expressionFile, "from-file", "", "", "Load expression from specified file.")
+	if err = rootCmd.MarkPersistentFlagFilename("from-file"); err != nil {
+		panic(err)
+	}
+
+	rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableEnvOps, "security-disable-env-ops", "", false, "Disable env related operations.")
+	rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableFileOps, "security-disable-file-ops", "", false, "Disable file related operations (e.g. load)")
+
 	rootCmd.AddCommand(
 		createEvaluateSequenceCommand(),
 		createEvaluateAllCommand(),

cmd/root_test.go

@@ -0,0 +1,264 @@
+package cmd
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestNewRuneVar(t *testing.T) {
+	var r rune
+	runeVar := newRuneVar(&r)
+
+	if runeVar == nil {
+		t.Fatal("newRuneVar returned nil")
+	}
+}
+
+func TestRuneValue_String(t *testing.T) {
+	tests := []struct {
+		name     string
+		runeVal  rune
+		expected string
+	}{
+		{
+			name:     "simple character",
+			runeVal:  'a',
+			expected: "a",
+		},
+		{
+			name:     "special character",
+			runeVal:  '\n',
+			expected: "\n",
+		},
+		{
+			name:     "unicode character",
+			runeVal:  'ñ',
+			expected: "ñ",
+		},
+		{
+			name:     "zero rune",
+			runeVal:  0,
+			expected: string(rune(0)),
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			runeVal := runeValue(tt.runeVal)
+			result := runeVal.String()
+			if result != tt.expected {
+				t.Errorf("runeValue.String() = %q, want %q", result, tt.expected)
+			}
+		})
+	}
+}
+
+func TestRuneValue_Set(t *testing.T) {
+	tests := []struct {
+		name        string
+		input       string
+		expected    rune
+		expectError bool
+	}{
+		{
+			name:        "simple character",
+			input:       "a",
+			expected:    'a',
+			expectError: false,
+		},
+		{
+			name:        "newline escape",
+			input:       "\\n",
+			expected:    '\n',
+			expectError: false,
+		},
+		{
+			name:        "tab escape",
+			input:       "\\t",
+			expected:    '\t',
+			expectError: false,
+		},
+		{
+			name:        "carriage return escape",
+			input:       "\\r",
+			expected:    '\r',
+			expectError: false,
+		},
+		{
+			name:        "form feed escape",
+			input:       "\\f",
+			expected:    '\f',
+			expectError: false,
+		},
+		{
+			name:        "vertical tab escape",
+			input:       "\\v",
+			expected:    '\v',
+			expectError: false,
+		},
+		{
+			name:        "empty string",
+			input:       "",
+			expected:    0,
+			expectError: true,
+		},
+		{
+			name:        "multiple characters",
+			input:       "ab",
+			expected:    0,
+			expectError: true,
+		},
+		{
+			name:        "special character",
+			input:       "ñ",
+			expected:    'ñ',
+			expectError: true, // This will fail because the Set function checks len(val) != 1
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var r rune
+			runeVal := newRuneVar(&r)
+
+			err := runeVal.Set(tt.input)
+
+			if tt.expectError {
+				if err == nil {
+					t.Errorf("Expected error for input %q, but got none", tt.input)
+				}
+			} else {
+				if err != nil {
+					t.Errorf("Unexpected error for input %q: %v", tt.input, err)
+				}
+				if r != tt.expected {
+					t.Errorf("Expected rune %q (%d), got %q (%d)",
+						string(tt.expected), tt.expected, string(r), r)
+				}
+			}
+		})
+	}
+}
+
+func TestRuneValue_Set_ErrorMessages(t *testing.T) {
+	tests := []struct {
+		name          string
+		input         string
+		expectedError string
+	}{
+		{
+			name:          "empty string error",
+			input:         "",
+			expectedError: "[] is not a valid character. Must be length 1 was 0",
+		},
+		{
+			name:          "multiple characters error",
+			input:         "abc",
+			expectedError: "[abc] is not a valid character. Must be length 1 was 3",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var r rune
+			runeVal := newRuneVar(&r)
+
+			err := runeVal.Set(tt.input)
+
+			if err == nil {
+				t.Errorf("Expected error for input %q, but got none", tt.input)
+				return
+			}
+
+			if !strings.Contains(err.Error(), tt.expectedError) {
+				t.Errorf("Expected error message to contain %q, got %q",
+					tt.expectedError, err.Error())
+			}
+		})
+	}
+}
+
+func TestRuneValue_Type(t *testing.T) {
+	var r rune
+	runeVal := newRuneVar(&r)
+
+	result := runeVal.Type()
+	expected := "char"
+
+	if result != expected {
+		t.Errorf("runeValue.Type() = %q, want %q", result, expected)
+	}
+}
+
+func TestNew(t *testing.T) {
+	rootCmd := New()
+
+	if rootCmd == nil {
+		t.Fatal("New() returned nil")
+	}
+
+	// Test basic command properties
+	if rootCmd.Use != "yq" {
+		t.Errorf("Expected Use to be 'yq', got %q", rootCmd.Use)
+	}
+
+	if rootCmd.Short == "" {
+		t.Error("Expected Short description to be non-empty")
+	}
+
+	if rootCmd.Long == "" {
+		t.Error("Expected Long description to be non-empty")
+	}
+
+	// Test that the command has the expected subcommands
+	expectedCommands := []string{"eval", "eval-all", "completion"}
+	actualCommands := make([]string, 0, len(rootCmd.Commands()))
+
+	for _, cmd := range rootCmd.Commands() {
+		actualCommands = append(actualCommands, cmd.Name())
+	}
+
+	for _, expected := range expectedCommands {
+		found := false
+		for _, actual := range actualCommands {
+			if actual == expected {
+				found = true
+				break
+			}
+		}
+		if !found {
+			t.Errorf("Expected command %q not found in actual commands: %v",
+				expected, actualCommands)
+		}
+	}
+}
+
+func TestNew_FlagCompletions(t *testing.T) {
+	rootCmd := New()
+
+	// Test that flag completion functions are registered
+	// This is a basic smoke test - we can't easily test the actual completion logic
+	// without more complex setup
+	flags := []string{
+		"output-format",
+		"input-format",
+		"xml-attribute-prefix",
+		"xml-content-name",
+		"xml-proc-inst-prefix",
+		"xml-directive-name",
+		"lua-prefix",
+		"lua-suffix",
+		"properties-separator",
+		"indent",
+		"front-matter",
+		"expression",
+		"split-exp",
+	}
+
+	for _, flagName := range flags {
+		flag := rootCmd.PersistentFlags().Lookup(flagName)
+		if flag == nil {
+			t.Errorf("Expected flag %q to exist", flagName)
+		}
+	}
+}

cmd/unwrap_flag.go

@@ -0,0 +1,46 @@
+package cmd
+
+import (
+	"strconv"
+
+	"github.com/spf13/pflag"
+)
+
+type boolFlag interface {
+	pflag.Value
+	IsExplicitlySet() bool
+	IsSet() bool
+}
+
+type unwrapScalarFlagStrc struct {
+	explicitlySet bool
+	value         bool
+}
+
+func newUnwrapFlag() boolFlag {
+	return &unwrapScalarFlagStrc{value: true}
+}
+
+func (f *unwrapScalarFlagStrc) IsExplicitlySet() bool {
+	return f.explicitlySet
+}
+
+func (f *unwrapScalarFlagStrc) IsSet() bool {
+	return f.value
+}
+
+func (f *unwrapScalarFlagStrc) String() string {
+	return strconv.FormatBool(f.value)
+}
+
+func (f *unwrapScalarFlagStrc) Set(value string) error {
+
+	v, err := strconv.ParseBool(value)
+	f.value = v
+	f.explicitlySet = true
+	return err
+}
+
+func (*unwrapScalarFlagStrc) Type() string {
+	return "bool"
+}

cmd/utils.go

@@ -0,0 +1,295 @@
+package cmd
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	"github.com/mikefarah/yq/v4/pkg/yqlib"
+	"github.com/spf13/cobra"
+	"gopkg.in/op/go-logging.v1"
+)
+
+func isAutomaticOutputFormat() bool {
+	return outputFormat == "" || outputFormat == "auto" || outputFormat == "a"
+}
+
+func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
+	cmd.SilenceUsage = true
+
+	setupColors()
+
+	expression, args, err := processArgs(args)
+	if err != nil {
+		return "", nil, err
+	}
+
+	if err := loadSplitFileExpression(); err != nil {
+		return "", nil, err
+	}
+
+	handleBackwardsCompatibility()
+
+	if err := validateCommandFlags(args); err != nil {
+		return "", nil, err
+	}
+
+	if err := configureFormats(args); err != nil {
+		return "", nil, err
+	}
+
+	configureUnwrapScalar()
+
+	return expression, args, nil
+}
+
+func setupColors() {
+	fileInfo, _ := os.Stdout.Stat()
+
+	if forceColor || (!forceNoColor && (fileInfo.Mode()&os.ModeCharDevice) != 0) {
+		colorsEnabled = true
+	}
+}
+
+func loadSplitFileExpression() error {
+	if splitFileExpFile != "" {
+		splitExpressionBytes, err := os.ReadFile(splitFileExpFile)
+		if err != nil {
+			return err
+		}
+		splitFileExp = string(splitExpressionBytes)
+	}
+	return nil
+}
+
+func handleBackwardsCompatibility() {
+	// backwards compatibility
+	if outputToJSON {
+		outputFormat = "json"
+	}
+}
+
+func validateCommandFlags(args []string) error {
+	if writeInplace && (len(args) == 0 || args[0] == "-") {
+		return fmt.Errorf("write in place flag only applicable when giving an expression and at least one file")
+	}
+
+	if frontMatter != "" && len(args) == 0 {
+		return fmt.Errorf("front matter flag only applicable when giving an expression and at least one file")
+	}
+
+	if writeInplace && splitFileExp != "" {
+		return fmt.Errorf("write in place cannot be used with split file")
+	}
+
+	if nullInput && len(args) > 0 {
+		return fmt.Errorf("cannot pass files in when using null-input flag")
+	}
+
+	return nil
+}
+
+func configureFormats(args []string) error {
+	inputFilename := ""
+	if len(args) > 0 {
+		inputFilename = args[0]
+	}
+
+	if err := configureInputFormat(inputFilename); err != nil {
+		return err
+	}
+
+	if err := configureOutputFormat(); err != nil {
+		return err
+	}
+
+	yqlib.GetLogger().Debug("Using input format %v", inputFormat)
+	yqlib.GetLogger().Debug("Using output format %v", outputFormat)
+
+	return nil
+}
+
+func configureInputFormat(inputFilename string) error {
+	if inputFormat == "" || inputFormat == "auto" || inputFormat == "a" {
+		inputFormat = yqlib.FormatStringFromFilename(inputFilename)
+
+		_, err := yqlib.FormatFromString(inputFormat)
+		if err != nil {
+			// unknown file type, default to yaml
+			yqlib.GetLogger().Debug("Unknown file format extension '%v', defaulting to yaml", inputFormat)
+			inputFormat = "yaml"
+			if isAutomaticOutputFormat() {
+				outputFormat = "yaml"
+			}
+		} else if isAutomaticOutputFormat() {
+			outputFormat = inputFormat
+		}
+	} else if isAutomaticOutputFormat() {
+		// backwards compatibility -
+		// before this was introduced, `yq -pcsv things.csv`
+		// would produce *yaml* output.
+		//
+		outputFormat = yqlib.FormatStringFromFilename(inputFilename)
+		if inputFilename != "-" {
+			yqlib.GetLogger().Warning("yq default output is now 'auto' (based on the filename extension). Normally yq would output '%v', but for backwards compatibility 'yaml' has been set. Please use -oy to specify yaml, or drop the -p flag.", outputFormat)
+		}
+		outputFormat = "yaml"
+	}
+	return nil
+}
+
+func configureOutputFormat() error {
+	outputFormatType, err := yqlib.FormatFromString(outputFormat)
+	if err != nil {
+		return err
+	}
+
+	if outputFormatType == yqlib.YamlFormat ||
+		outputFormatType == yqlib.PropertiesFormat {
+		unwrapScalar = true
+	}
+
+	return nil
+}
+
+func configureUnwrapScalar() {
+	if unwrapScalarFlag.IsExplicitlySet() {
+		unwrapScalar = unwrapScalarFlag.IsSet()
+	}
+}
+
+func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
+	format, err := yqlib.FormatFromString(inputFormat)
+	if err != nil {
+		return nil, err
+	}
+	yqlib.ConfiguredYamlPreferences.EvaluateTogether = evaluateTogether
+
+	yqlibDecoder := format.DecoderFactory()
+	if yqlibDecoder == nil {
+		return nil, fmt.Errorf("no support for %s input format", inputFormat)
+	}
+	return yqlibDecoder, nil
+}
+
+func configurePrinterWriter(format *yqlib.Format, out io.Writer) (yqlib.PrinterWriter, error) {
+
+	var printerWriter yqlib.PrinterWriter
+
+	if splitFileExp != "" {
+		colorsEnabled = forceColor
+		splitExp, err := yqlib.ExpressionParser.ParseExpression(splitFileExp)
+		if err != nil {
+			return nil, fmt.Errorf("bad split document expression: %w", err)
+		}
+		printerWriter = yqlib.NewMultiPrinterWriter(splitExp, format)
+	} else {
+		printerWriter = yqlib.NewSinglePrinterWriter(out)
+	}
+	return printerWriter, nil
+}
+
+func configureEncoder() (yqlib.Encoder, error) {
+	yqlibOutputFormat, err := yqlib.FormatFromString(outputFormat)
+	if err != nil {
+		return nil, err
+	}
+	yqlib.ConfiguredXMLPreferences.Indent = indent
+	yqlib.ConfiguredYamlPreferences.Indent = indent
+	yqlib.ConfiguredJSONPreferences.Indent = indent
+
+	yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
+	yqlib.ConfiguredPropertiesPreferences.UnwrapScalar = unwrapScalar
+	yqlib.ConfiguredJSONPreferences.UnwrapScalar = unwrapScalar
+
+	yqlib.ConfiguredYamlPreferences.ColorsEnabled = colorsEnabled
+	yqlib.ConfiguredJSONPreferences.ColorsEnabled = colorsEnabled
+	yqlib.ConfiguredHclPreferences.ColorsEnabled = colorsEnabled
+
+	yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
+
+	encoder := yqlibOutputFormat.EncoderFactory()
+
+	if encoder == nil {
+		return nil, fmt.Errorf("no support for %s output format", outputFormat)
+	}
+	return encoder, err
+}
+
+// this is a hack to enable backwards compatibility with githubactions (which pipe /dev/null into everything)
+// and being able to call yq with the filename as a single parameter
+//
+// without this - yq detects there is stdin (thanks githubactions),
+// then tries to parse the filename as an expression
+func maybeFile(str string) bool {
+	yqlib.GetLogger().Debugf("checking '%v' is a file", str)
+	stat, err := os.Stat(str) // #nosec
+	result := err == nil && !stat.IsDir()
+	if yqlib.GetLogger().IsEnabledFor(logging.DEBUG) {
+		if err != nil {
+			yqlib.GetLogger().Debugf("error: %v", err)
+		} else {
+			yqlib.GetLogger().Debugf("error: %v, dir: %v", err, stat.IsDir())
+		}
+		yqlib.GetLogger().Debugf("result: %v", result)
+	}
+	return result
+}
+
+func processStdInArgs(args []string) []string {
+	stat, err := os.Stdin.Stat()
+	if err != nil {
+		yqlib.GetLogger().Debugf("error getting stdin: %v", err)
+	}
+	pipingStdin := stat != nil && (stat.Mode()&os.ModeCharDevice) == 0
+
+	// if we've been given a file, don't automatically
+	// read from stdin.
+	// this happens if there is more than one argument
+	// or only one argument and its a file
+	if nullInput || !pipingStdin || len(args) > 1 || (len(args) > 0 && maybeFile(args[0])) {
+		return args
+	}
+
+	for _, arg := range args {
+		if arg == "-" {
+			return args
+		}
+	}
+	yqlib.GetLogger().Debugf("missing '-', adding it to the end")
+
+	// we're piping from stdin, but there's no '-' arg
+	// lets add one to the end
+	return append(args, "-")
+}
+
+func processArgs(originalArgs []string) (string, []string, error) {
+	expression := forceExpression
+	args := processStdInArgs(originalArgs)
+	maybeFirstArgIsAFile := len(args) > 0 && maybeFile(args[0])
+
+	if expressionFile == "" && maybeFirstArgIsAFile && strings.HasSuffix(args[0], ".yq") {
+		// lets check if an expression file was given
+		yqlib.GetLogger().Debug("Assuming arg %v is an expression file", args[0])
+		expressionFile = args[0]
+		args = args[1:]
+	}
+
+	if expressionFile != "" {
+		expressionBytes, err := os.ReadFile(expressionFile)
+		if err != nil {
+			return "", nil, err
+		}
+		//replace \r\n (windows) with good ol' unix file endings.
+		expression = strings.ReplaceAll(string(expressionBytes), "\r\n", "\n")
+	}
+
+	yqlib.GetLogger().Debugf("processed args: %v", args)
+	if expression == "" && len(args) > 0 && args[0] != "-" && !maybeFile(args[0]) {
+		yqlib.GetLogger().Debug("assuming expression is '%v'", args[0])
+		expression = args[0]
+		args = args[1:]
+	}
+	return expression, args, nil
+}

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.13.5"
+	Version = "v4.49.2"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release
@@ -45,5 +45,5 @@ func getHumanVersion() string {
 	}
 
 	// Strip off any single quotes added by the git information.
-	return strings.Replace(version, "'", "", -1)
+	return strings.ReplaceAll(version, "'", "")
 }

cspell.config.yaml

@@ -0,0 +1,14 @@
+---
+$schema: https://raw.githubusercontent.com/streetsidesoftware/cspell/main/cspell.schema.json
+version: '0.2'
+language: en-GB
+dictionaryDefinitions:
+  - name: project-words
+    path: './project-words.txt'
+    addWords: true
+dictionaries:
+  - project-words
+ignorePaths:
+  - 'vendor'
+  - 'bin'
+  - '/project-words.txt'

debian/changelog

@@ -1,101 +0,0 @@
-yq (4.13.0) focal; urgency=medium
-
-  * New `with` operator for making multiple changes to a given path
-  * New `contains` operator, works like the `jq` equivalent
-  * Subtract operator now supports subtracting elements from arrays!
-  * Fixed Swapping values using variables #934
-  * Github Action now properly supports multiline output #936, thanks @pjxiao
-  * Fixed missing closing bracket validation #932
-  * Fix processing of hex numbers #929
-  * Fixed alternative and union operator issues #930
-  * Can now convert yaml to properties properties format (`-o=props`), See [docs](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties) for more info.
-  * Fixed document header/footer comment handling when merging (https://github.com/mikefarah/yq/issues/919)
-  * pretty print yaml 1.1 compatibility  (https://github.com/mikefarah/yq/issues/914)
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Thu, 16 Sep 2021 20:58:30 +0200
-
-yq (4.9.6) focal; urgency=medium
-
-  * Added darwin/arm64 build, thanks @alecthomas
-  * Incremented docker alpine base version, thanks @da6d6i7-bronga
-  * Bug fix: multine expression
-  * Bug fix: special character
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Tue, 29 Jun 2021 21:32:14 +0200
-
-yq (3.3.2) focal; urgency=medium
-
-  * Bug fix: existStatus bug (#459)
-  * Automatically makes a os temp directory if it does not exist (#461)
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Fri, 07 Aug 2020 18:53:01 +0200
-
-yq (3.3-0) focal; urgency=medium
-
-  * You can control string styles (quotes) using the new --style flag
-  * String values now always have quotes when outputting to json
-  * Negative array indices now traverse the array backwards
-  * Added a --stripComments flag to print yaml without any comments
-  * Bumped go to version 1.14
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Thu, 30 Apr 2020 20:45:44 +0200
-
-yq (3.1-2) eoan; urgency=medium
-
-  * Bug fix: yq 3 was removing empty inline-style objects and arrays (#355)
-  * Bug fix: Merge option returned different output when switching order of 
-    merging files(#347)
-  * Bug fix: Add new object to existing array object was failing in 3.1.1 (#361)
-  * Bug fix: yq 3 empty keys did not allow merging of values (#356)
-  * Bug fix: keys quoted during merge (#363)
-  * Bug fix: Correct length with wc -l (#362)
-  * Bug fix: Write to empty document removed path (#359)
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Mon, 24 Feb 2020 20:31:58 +0100
-
-yq (3.1-1) eoan; urgency=medium
-
-  * Keeps yaml comments and formatting, can specify yaml tags when updating.
-  * Handles anchors
-  * Can print out matching paths and values when splatting
-  * JSON output works for all commands
-  * Yaml files with multiple documents are printed out as one JSON 
-    document per line.
-  * Deep splat (**) to match arbitrary paths
-  * Update scripts file format has changed to be more powerful
-  * Reading and splatting, matching results are printed once per line
-  * Bugfixing
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Tue, 11 Feb 2020 22:18:24 +0100
-
-yq (2.2-1) bionic; urgency=medium
-
-  * Added Windows support for the "--inplace" command flag
-  * Prefix now supports arrays
-  * Add prefix command
-  * Bump Alpine version to 3.8
-  * Improved docker build process
-  * Lint fixes
-  * Build support for all linux architectures supported by gox
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Sat, 19 Jan 2019 15:50:47 +0100
-
-yq (2.1-0) bionic; urgency=medium
-
-  * Ability to read multiple documents in a single file
-  * Ability to append list items instead of overwriting
-
- -- Roberto Mier Escandón <rmescandon@gmail.com>  Tue, 10 Jul 2018 14:02:42 +0200
-
-yq (2.0-0) bionic; urgency=medium
-
-  * Release 2.0.0
-
- -- Roberto Mier Escandón <rmescandon@gmail.com>  Wed, 20 Jun 2018 10:29:53 +0200
-
-yq (1.15-0) bionic; urgency=medium
-
-  * Release 1.15
-
- -- Roberto Mier Escandón <rmescandon@gmail.com>  Wed, 06 Jun 2018 11:32:03 +0200
-

debian/compat

@@ -1 +0,0 @@
-10

debian/control

@@ -1,21 +0,0 @@
-Source: yq
-Section: devel
-Priority: optional
-Maintainer: Roberto Mier Escandón <rmescandon@gmail.com>
-Build-Depends: debhelper (>=10),
-               golang-1.15,
-               rsync
-Standards-Version: 4.1.4
-Homepage: https://github.com/mikefarah/yq.git
-Vcs-Browser: https://github.com/mikefarah/yq.git
-Vcs-Git: https://github.com/mikefarah/yq.git
-XS-Go-Import-Path: github.com/mikefarah/yq
-XSBC-Original-Maintainer: Roberto Mier Escandón <rmescandon@gmail.com>
-
-Package: yq
-Architecture: any
-Depends: ${shlibs:Depends}, ${misc:Depends}
-Description: lightweight and portable command-line YAML processor
- .
- The aim of the project is to be the 
- [jq](https://github.com/stedolan/jq) or sed of yaml files.

debian/copyright

@@ -1,24 +0,0 @@
-Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-Upstream-Name: yq
-Source: https://github.com/mikefarah/yq.git
-
-Files: *
-Copyright: 2017 Mike Farah
-License: Expat
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to deal
-    in the Software without restriction, including without limitation the rights
-    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-    copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
-    .
-    The above copyright notice and this permission notice shall be included in all
-    copies or substantial portions of the Software.
-    .
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-    SOFTWARE.

debian/gbp.conf

@@ -1,2 +0,0 @@
-[DEFAULT]
-pristine-tar = True

debian/rules

@@ -1,59 +0,0 @@
-#!/usr/bin/make -f
-#
-# Copyright (C) 2018 Roberto Mier Escandón <rmescandon@gmail.com>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-PROJECT := yq
-OWNER := mikefarah
-REPO := github.com
-GOVERSION := 1.15
-
-export DH_OPTIONS
-export DH_GOPKG := ${REPO}/${OWNER}/${PROJECT}
-export GOROOT := /usr/lib/go-${GOVERSION}
-export GOPATH := ${CURDIR}/_build
-export GOBIN := ${GOPATH}/bin
-export PATH := ${GOROOT}/bin:${GOBIN}:${PATH}
-export GOCACHE := /tmp/gocache
-export GOFLAGS := -mod=vendor
-
-SRCDIR := ${GOPATH}/src/${DH_GOPKG}
-DESTDIR := ${CURDIR}/debian/${PROJECT}
-BINDIR := /usr/bin
-ASSETSDIR := /usr/share/${PROJECT}
-
-%:
-	dh $@
-
-override_dh_auto_build:
-	mkdir -p ${SRCDIR}
-	mkdir -p ${GOBIN}
-	# copy project to local srcdir to build from there
-	rsync -avz --progress --exclude=_build --exclude=debian --exclude=tmp. --exclude=go.mod --exclude=docs . $(SRCDIR)
-	# build go code
-	(cd ${SRCDIR} && go install -buildmode=pie ./...)
-	
-override_dh_auto_test:
-ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
-	(cd ${SRCDIR} && go test -v ./...)
-endif
-
-override_dh_auto_install:
-	cp ${GOBIN}/yq ${DESTDIR}/${BINDIR}
-	cp -f ${SRCDIR}/LICENSE ${DESTDIR}/${ASSETSDIR}
-	chmod a+x ${DESTDIR}/${BINDIR}/yq
-
-override_dh_auto_clean:
-	dh_clean
-	rm -rf ${CURDIR}/_build

debian/source/format

@@ -1 +0,0 @@
-3.0 (native)

debian/yq.dirs

@@ -1,3 +0,0 @@
-usr/bin
-usr/share/yq
-usr/share/man/man1

examples/array.yaml

@@ -1,11 +1,2 @@
----
-- become: true
-  gather_facts: false
-  hosts: lalaland
-  name: "Apply smth"
-  roles:
-    - lala
-    - land
-  serial: 1
-- become: false
-  gather_facts: true
+- [cat, dog, frog, cow]
+- [apple, banana, grape, mango]

examples/base64.txt

@@ -0,0 +1 @@
+bXkgc2VjcmV0IGNoaWxsaSByZWNpcGUgaXMuLi4u

examples/data.lua

@@ -0,0 +1,9 @@
+return {
+	["country"] = "Australia"; -- this place
+	["cities"] = {
+		"Sydney",
+		"Melbourne",
+		"Brisbane",
+		"Perth",
+	};
+};

examples/data1.yaml

@@ -1,5 +1 @@
-Resources:
-  S3Bucket:
-    Type: AWS::CloudFormation::Stack
-    Properties:
-      BucketName: !Ref MyBucketNameA
+a: apple

examples/data2.yaml

@@ -1,3 +1,5 @@
-#b1 
-b: 2
-#b2
+# --------------------------------------------------
+# It's a test with comment
+# --------------------------------------------------
+groups:
+  - name: d

examples/environment.yq

@@ -0,0 +1,5 @@
+#! yq
+.[] |(
+    ( select(kind == "scalar") | key + "='" + . + "'"),
+    ( select(kind == "seq") | key + "=(" + (map("'" + . + "'") | join(",")) + ")")
+)

examples/example.properties

@@ -0,0 +1,6 @@
+# comments on values appear
+person.name = Mike
+
+# comments on array values appear
+person.pets.0 = cat
+person.food.0 = pizza

examples/front-matter.yaml

@@ -1,6 +1,6 @@
 ---
 a: apple
-b: bannana
+b: banana
 ---
 hello there
 apples: great

examples/mike.xml

@@ -0,0 +1 @@
+<zoo><thing><frog>boing</frog></thing></zoo>

examples/mike2.xml

@@ -0,0 +1,11 @@
+  <?xml version="1.0" encoding="UTF-8"?>
+  <!-- osm-->
+  <osm version="0.6" generator="CGImap 0.0.2">
+  <!-- bounds-->
+   <bounds minlat="54.0889580" minlon="12.2487570" maxlat="54.0913900" maxlon="12.2524800">
+   <!-- great -->
+   cool
+   </bounds>
+   <foo>ba2234r</foo>
+   <foo>bar2234233</foo>
+  </osm>

examples/sample.hcl

@@ -0,0 +1,8 @@
+# Arithmetic with literals and application-provided variables
+sum = 1 + addend
+
+# String interpolation and templates
+message = "Hello, ${name}!"
+
+# Application-provided functions
+shouty_message = upper(message)

examples/sample.ini

@@ -0,0 +1,14 @@
+
+; This is a INI document
+
+[owner]
+name = "Tom Preston-Werner"
+dob = 1979-05-27T07:32:00-08:00
+
+[database]
+db_host = "localhost"
+db_port = 5432
+db_user = "postgres"
+db_password = ""
+db_name = "postgres"
+

examples/sample.toml

@@ -0,0 +1,6 @@
+[[fruits]]
+
+[animals]
+
+[[fruits.varieties]]  # nested array of tables
+name = "red delicious"

examples/sample.yaml

@@ -1,11 +1,2 @@
-# Some doc
-
-a: true
-b:
-  c: 2
-  d: [3, 4, 5]
-  e:
-    - name: fred
-      value: 3
-    - name: sam
-      value: 4
+ # things
+a: apple

examples/sample2.hcl

@@ -0,0 +1,8 @@
+# Arithmetic with literals and application-provided variables
+sum = 1 + addend
+
+# String interpolation and templates
+message = "Hello, ${name}!"
+
+# Application-provided functions
+shouty_message = upper(message)

examples/sample_no_sections.ini

@@ -0,0 +1,8 @@
+
+; This is a INI document
+name = "Tom Preston-Werner"
+dob = 1979-05-27T07:32:00-08:00
+enabled = true
+ip = "10.0.0.1"
+role = "frontend"
+treads = 4

examples/sample_objects.csv

@@ -0,0 +1,3 @@
+name,numberOfCats,likesApples,height
+,1,true,168.8
+Samantha's Rabbit,2,false,-188.8

examples/small.properties

@@ -0,0 +1 @@
+this.is = a properties file

examples/small.xml

@@ -0,0 +1 @@
+<this>is some xml</this>

examples/small.yaml

@@ -0,0 +1,4 @@
+---
+# comment
+# about things
+a: cat

examples/thing.yml

@@ -0,0 +1,2 @@
+a: apple is included
+b: cool.

github-action/Dockerfile

@@ -1,8 +1,9 @@
-FROM mikefarah/yq:4.13.5
+FROM mikefarah/yq:4
 
 COPY entrypoint.sh /entrypoint.sh
 
-# this seems to be the default user in github actions
-USER 1001
+# github action recommendation is to run as root.
+# https://docs.github.com/en/actions/creating-actions/dockerfile-support-for-github-actions#user
+USER root
 
 ENTRYPOINT ["/entrypoint.sh"]

github-action/entrypoint.sh

@@ -2,8 +2,11 @@
 set -e
 echo "::debug::\$cmd: $1"
 RESULT=$(eval "$1")
-RESULT="${RESULT//'%'/'%25'}"
-RESULT="${RESULT//$'\n'/'%0A'}"
-RESULT="${RESULT//$'\r'/'%0D'}"
 echo "::debug::\$RESULT: $RESULT"
-echo ::set-output name=result::"$RESULT"
+# updating from 
+# https://github.com/orgs/community/discussions/26288#discussioncomment-3876281
+# https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-output-parameter
+delimiter=$(cat /proc/sys/kernel/random/uuid)
+echo "result<<${delimiter}" >> "${GITHUB_OUTPUT}"
+echo "${RESULT}" >> "${GITHUB_OUTPUT}"
+echo "${delimiter}" >> "${GITHUB_OUTPUT}"

go.mod

@@ -1,25 +1,44 @@
 module github.com/mikefarah/yq/v4
 
 require (
-	github.com/elliotchance/orderedmap v1.4.0
-	github.com/fatih/color v1.10.0
-	github.com/goccy/go-yaml v1.8.9
-	github.com/jinzhu/copier v0.2.8
-	github.com/magiconair/properties v1.8.5
-	github.com/spf13/cobra v1.1.3
-	github.com/timtadh/lexmachine v0.2.2
+	github.com/a8m/envsubst v1.4.3
+	github.com/alecthomas/participle/v2 v2.1.4
+	github.com/alecthomas/repr v0.5.2
+	github.com/dimchansky/utfbom v1.1.1
+	github.com/elliotchance/orderedmap v1.8.0
+	github.com/fatih/color v1.18.0
+	github.com/go-ini/ini v1.67.0
+	github.com/goccy/go-json v0.10.5
+	github.com/goccy/go-yaml v1.19.0
+	github.com/hashicorp/hcl/v2 v2.24.0
+	github.com/jinzhu/copier v0.4.0
+	github.com/magiconair/properties v1.8.10
+	github.com/pelletier/go-toml/v2 v2.2.4
+	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
+	github.com/spf13/cobra v1.10.2
+	github.com/spf13/pflag v1.0.10
+	github.com/yuin/gopher-lua v1.1.1
+	github.com/zclconf/go-cty v1.16.3
+	go.yaml.in/yaml/v4 v4.0.0-rc.3
+	golang.org/x/net v0.47.0
+	golang.org/x/text v0.31.0
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 )
 
 require (
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/mattn/go-colorable v0.1.8 // indirect
-	github.com/mattn/go-isatty v0.0.12 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/timtadh/data-structures v0.5.3 // indirect
-	golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e // indirect
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	github.com/agext/levenshtein v1.2.1 // indirect
+	github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/mattn/go-colorable v0.1.14 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
+	golang.org/x/mod v0.29.0 // indirect
+	golang.org/x/sync v0.18.0 // indirect
+	golang.org/x/sys v0.38.0 // indirect
+	golang.org/x/tools v0.38.0 // indirect
 )
 
-go 1.17
+go 1.24.0
+
+toolchain go1.24.1

go.sum

@@ -1,330 +1,90 @@
-cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
-cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
-cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
-cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
-cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
-cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
-cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
-cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
-cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
-cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
-github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
-github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
-github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
-github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
-github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
-github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
-github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
-github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
-github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
-github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
-github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
-github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
-github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/a8m/envsubst v1.4.3 h1:kDF7paGK8QACWYaQo6KtyYBozY2jhQrTuNNuUxQkhJY=
+github.com/a8m/envsubst v1.4.3/go.mod h1:4jjHWQlZoaXPoLQUb7H2qT4iLkZDdmEQiOUogdUmqVU=
+github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8=
+github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
+github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8vS6K3D0=
+github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
+github.com/alecthomas/participle/v2 v2.1.4 h1:W/H79S8Sat/krZ3el6sQMvMaahJ+XcM9WSI2naI7w2U=
+github.com/alecthomas/participle/v2 v2.1.4/go.mod h1:8tqVbpTX20Ru4NfYQgZf4mP18eXPTBViyMWiArNEgGI=
+github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=
+github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
+github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
+github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
-github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/elliotchance/orderedmap v1.4.0 h1:wZtfeEONCbx6in1CZyE6bELEt/vFayMvsxqI5SgsR+A=
-github.com/elliotchance/orderedmap v1.4.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
-github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
-github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
-github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
-github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
-github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
-github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
-github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
-github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
-github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
-github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
-github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
-github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
-github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/goccy/go-yaml v1.8.9 h1:4AEXg2qx+/w29jXnXpMY6mTckmYu1TMoHteKuMf0HFg=
-github.com/goccy/go-yaml v1.8.9/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
-github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
-github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
-github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
-github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
-github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
-github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
-github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
-github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
-github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
-github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
-github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
-github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
-github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
-github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
-github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
-github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
-github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
-github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
-github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
-github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
-github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
-github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
-github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
-github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
-github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
-github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
-github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/jinzhu/copier v0.2.8 h1:N8MbL5niMwE3P4dOwurJixz5rMkKfujmMRFmAanSzWE=
-github.com/jinzhu/copier v0.2.8/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro=
-github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
-github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
-github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
-github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
-github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
-github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
-github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
-github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls=
-github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
-github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
-github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8=
-github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
-github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
-github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
-github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
-github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
-github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
-github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
-github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
-github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
-github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
-github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
-github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
+github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
+github.com/elliotchance/orderedmap v1.8.0 h1:TrOREecvh3JbS+NCgwposXG5ZTFHtEsQiCGOhPElnMw=
+github.com/elliotchance/orderedmap v1.8.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
+github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
+github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
+github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
+github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
+github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
+github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
+github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
+github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
+github.com/goccy/go-yaml v1.19.0 h1:EmkZ9RIsX+Uq4DYFowegAuJo8+xdX3T/2dwNPXbxEYE=
+github.com/goccy/go-yaml v1.19.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/hashicorp/hcl/v2 v2.24.0 h1:2QJdZ454DSsYGoaE6QheQZjtKZSUs9Nh2izTWiwQxvE=
+github.com/hashicorp/hcl/v2 v2.24.0/go.mod h1:oGoO1FIQYfn/AgyOhlg9qLC6/nOJPX3qGbkZpYAcqfM=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
+github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
+github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
+github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
+github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
+github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
+github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
+github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
+github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
-github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
-github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
-github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
-github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
-github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
-github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
-github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
-github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
-github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
-github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
-github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
-github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU=
+github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4=
+github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
+github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
-github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/timtadh/data-structures v0.5.3 h1:F2tEjoG9qWIyUjbvXVgJqEOGJPMIiYn7U5W5mE+i/vQ=
-github.com/timtadh/data-structures v0.5.3/go.mod h1:9R4XODhJ8JdWFEI8P/HJKqxuJctfBQw6fDibMQny2oU=
-github.com/timtadh/lexmachine v0.2.2 h1:g55RnjdYazm5wnKv59pwFcBJHOyvTPfDEoz21s4PHmY=
-github.com/timtadh/lexmachine v0.2.2/go.mod h1:GBJvD5OAfRn/gnp92zb9KTgHLB7akKyxmVivoYCcjQI=
-github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
-github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
-go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
-go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
-go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
-go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
-go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
-go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
-golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
-golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
-golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
-golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
-golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
-golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
-golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
-golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
-golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
-golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
-golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
-golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA=
-golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
-google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
-google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
-google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
-google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
-google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
-google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
-google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
-google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
-google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
-google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
-gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
+github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
+github.com/zclconf/go-cty v1.16.3 h1:osr++gw2T61A8KVYHoQiFbFd1Lh3JOCXc/jFLJXKTxk=
+github.com/zclconf/go-cty v1.16.3/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE=
+github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo=
+github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go=
+go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
+golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
+golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
+golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
+golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
+golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
+golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
+golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
+golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
+golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
+golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW8s2qTSe3wGBtvo0MbVQG/c5k8RE=
 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog=
-gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
-gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
-gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
-gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
-rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

how-it-works.md

@@ -0,0 +1,183 @@
+# Expression Syntax: A Visual Guide
+In `yq`, expressions are made up of operators and pipes. A context of nodes is passed through the expression, and each operation takes the context as input and returns a new context as output. That output is piped in as input for the next operation in the expression.
+
+Let's break down the process step by step using a diagram. We'll start with a single YAML document, apply an expression, and observe how the context changes at each step.
+
+Given a document like:
+
+```yaml
+root:
+  items:
+    - name: apple
+      type: fruit
+    - name: carrot
+      type: vegetable
+    - name: banana
+      type: fruit
+```
+
+You can use dot notation to access nested structures. For example, to access the `name` of the first item, you would use the expression `.root.items[0].name`, which would return `apple`.
+
+But lets see how we could find all the fruit under `items`
+
+## Step 1: Initial Context
+The context starts at the root of the YAML document. In this case, the entire document is the initial context. 
+
+```
+root
+└── items
+    ├── name: apple
+    │   type: fruit
+    ├── name: carrot
+    │   type: vegetable
+    └── name: banana
+        type: fruit
+```
+
+## Step 2: Splatting the Array
+Using the expression `.root.items[]`, we "splat" the items array. This means each element of the array becomes its own node in the context:
+
+```
+Node 1: { name: apple, type: fruit }
+Node 2: { name: carrot, type: vegetable }
+Node 3: { name: banana, type: fruit }
+```
+
+## Step 3: Filtering the Nodes
+Next, we apply a filter to select only the nodes where type is fruit. The expression `.root.items[] | select(.type == "fruit")` filters the nodes:
+
+```
+Filtered Node 1: { name: apple, type: fruit }
+Filtered Node 2: { name: banana, type: fruit }
+```
+
+## Step 4: Extracting a Field
+Finally, we extract the name field from the filtered nodes using `.root.items[] | select(.type == "fruit") | .name` This results in:
+
+```
+apple
+banana
+```
+
+## Simple assignment example
+
+Given a document like:
+
+```yaml
+a: cat
+b: dog
+```
+
+with an expression:
+
+```
+.a = .b
+```
+
+Like math expressions - operator precedence is important. 
+
+The `=` operator takes two arguments, a `lhs` expression, which in this case is `.a` and `rhs` expression which is `.b`. 
+
+It pipes the current, lets call it 'root' context through the `lhs` expression of `.a` to return the node 
+
+```yaml
+cat
+```
+
+Side note: this node holds not only its value 'cat', but comments and metadata too, including path and parent information.
+
+The `=` operator then pipes the 'root' context through the `rhs` expression of `.b` to return the node
+
+```yaml
+dog
+```
+
+Both sides have now been evaluated, so now the operator copies across the value from the RHS (`.b`) to the LHS (`.a`), and it returns the now updated context:
+
+```yaml
+a: dog
+b: dog
+```
+
+## Complex assignment, operator precedence rules
+
+Just like math expressions - `yq` expressions have an order of precedence. The pipe `|` operator has a low order of precedence, so operators with higher precedence will get evaluated first. 
+
+Most of the time, this is intuitively what you'd want, for instance `.a = "cat" | .b = "dog"` is effectively: `(.a = "cat") | (.b = "dog")`.
+
+However, this is not always the case, particularly if you have a complex LHS or RHS expression, for instance if you want to select particular nodes to update. 
+
+Lets say you had:
+
+```yaml
+- name: bob
+  fruit: apple
+- name: sally
+  fruit: orange
+
+```
+
+Lets say you wanted to update the `sally` entry to have fruit: 'mango'. The _incorrect_ way to do that is:
+`.[] | select(.name == "sally") | .fruit = "mango"`.
+
+Because `|` has a low operator precedence, this will be evaluated (_incorrectly_) as : `(.[]) | (select(.name == "sally")) | (.fruit = "mango")`. What you'll see is only the updated segment returned:
+
+```yaml
+name: sally
+fruit: mango
+```
+
+**Important**: To properly update this YAML, you must wrap the entire LHS in parentheses. Think of it like using brackets in math to ensure the correct order of operations.
+`(.[] | select(.name == "sally") | .fruit) = "mango"`
+
+
+Now that entire LHS expression is passed to the 'assign' (`=`) operator, and the yaml is correctly updated and returned:
+
+
+```yaml
+- name: bob
+  fruit: apple
+- name: sally
+  fruit: mango
+
+```
+
+## Relative update (e.g. `|=`)
+There is another form of the `=` operator which we call the relative form. It's very similar to `=` but with one key difference when evaluating the RHS expression.
+
+In the plain form, we pass in the 'root' level context to the RHS expression. In relative form, we pass in _each result of the LHS_ to the RHS expression. Let's go through an example.
+
+Given a document like:
+
+```yaml
+a: 1
+b: thing
+```
+
+with an expression:
+
+```
+.a |= . + 1
+```
+
+Similar to the `=` operator, `|=` takes two operands, the LHS and RHS.
+
+It pipes the current context (the whole document) through the LHS expression of `.a` to get the node value:
+
+```
+1
+```
+
+Now it pipes _that LHS context_ into the RHS expression `. + 1` (whereas in the `=` plain form it piped the original document context into the RHS) to yield:
+
+
+```
+2
+```
+
+The assignment operator then copies across the value from the RHS to the value on the LHS, and it returns the now updated 'root' context:
+
+```yaml
+a: 2
+b: thing
+```

pkg/yqlib/all_at_once_evaluator.go

@@ -2,16 +2,14 @@ package yqlib
 
 import (
 	"container/list"
-
-	yaml "gopkg.in/yaml.v3"
 )
 
 // A yaml expression evaluator that runs the expression once against all files/nodes in memory.
 type Evaluator interface {
-	EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error
+	EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error
 
 	// EvaluateNodes takes an expression and one or more yaml nodes, returning a list of matching candidate nodes
-	EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error)
+	EvaluateNodes(expression string, nodes ...*CandidateNode) (*list.List, error)
 
 	// EvaluateCandidateNodes takes an expression and list of candidate nodes, returning a list of matching candidate nodes
 	EvaluateCandidateNodes(expression string, inputCandidateNodes *list.List) (*list.List, error)
@@ -19,23 +17,23 @@ type Evaluator interface {
 
 type allAtOnceEvaluator struct {
 	treeNavigator DataTreeNavigator
-	treeCreator   ExpressionParser
 }
 
 func NewAllAtOnceEvaluator() Evaluator {
-	return &allAtOnceEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: NewExpressionParser()}
+	InitExpressionParser()
+	return &allAtOnceEvaluator{treeNavigator: NewDataTreeNavigator()}
 }
 
-func (e *allAtOnceEvaluator) EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error) {
+func (e *allAtOnceEvaluator) EvaluateNodes(expression string, nodes ...*CandidateNode) (*list.List, error) {
 	inputCandidates := list.New()
 	for _, node := range nodes {
-		inputCandidates.PushBack(&CandidateNode{Node: node})
+		inputCandidates.PushBack(node)
 	}
 	return e.EvaluateCandidateNodes(expression, inputCandidates)
 }
 
 func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCandidates *list.List) (*list.List, error) {
-	node, err := e.treeCreator.ParseExpression(expression)
+	node, err := ExpressionParser.ParseExpression(expression)
 	if err != nil {
 		return nil, err
 	}
@@ -46,22 +44,17 @@ func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCand
 	return context.MatchingNodes, nil
 }
 
-func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool) error {
+func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, decoder Decoder) error {
 	fileIndex := 0
-	firstFileLeadingContent := ""
 
-	var allDocuments *list.List = list.New()
+	var allDocuments = list.New()
 	for _, filename := range filenames {
-		reader, leadingContent, err := readStream(filename, fileIndex == 0 && leadingContentPreProcessing)
+		reader, err := readStream(filename)
 		if err != nil {
 			return err
 		}
 
-		if fileIndex == 0 {
-			firstFileLeadingContent = leadingContent
-		}
-
-		fileDocuments, err := readDocuments(reader, filename, fileIndex)
+		fileDocuments, err := readDocuments(reader, filename, fileIndex, decoder)
 		if err != nil {
 			return err
 		}
@@ -70,15 +63,8 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
 	}
 
 	if allDocuments.Len() == 0 {
-		candidateNode := &CandidateNode{
-			Document:  0,
-			Filename:  "",
-			Node:      &yaml.Node{Kind: yaml.DocumentNode, HeadComment: firstFileLeadingContent, Content: []*yaml.Node{{Tag: "!!null", Kind: yaml.ScalarNode}}},
-			FileIndex: 0,
-		}
+		candidateNode := createScalarNode(nil, "")
 		allDocuments.PushBack(candidateNode)
-	} else {
-		allDocuments.Front().Value.(*CandidateNode).Node.HeadComment = firstFileLeadingContent
 	}
 
 	matches, err := e.EvaluateCandidateNodes(expression, allDocuments)

pkg/yqlib/all_at_once_evaluator_test.go

@@ -1,6 +1,8 @@
 package yqlib
 
 import (
+	"bufio"
+	"strings"
 	"testing"
 
 	"github.com/mikefarah/yq/v4/test"
@@ -18,7 +20,7 @@ var evaluateNodesScenario = []expressionScenario{
 		document:   `a: hello`,
 		expression: `.`,
 		expected: []string{
-			"D0, P[], (doc)::a: hello\n",
+			"D0, P[], (!!map)::a: hello\n",
 		},
 	},
 	{
@@ -32,9 +34,23 @@ var evaluateNodesScenario = []expressionScenario{
 
 func TestAllAtOnceEvaluateNodes(t *testing.T) {
 	var evaluator = NewAllAtOnceEvaluator()
+	// logging.SetLevel(logging.DEBUG, "")
 	for _, tt := range evaluateNodesScenario {
-		node := test.ParseData(tt.document)
-		list, _ := evaluator.EvaluateNodes(tt.expression, &node)
-		test.AssertResultComplex(t, tt.expected, resultsToString(list))
+		decoder := NewYamlDecoder(ConfiguredYamlPreferences)
+		reader := bufio.NewReader(strings.NewReader(tt.document))
+		err := decoder.Init(reader)
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		candidateNode, errorReading := decoder.Decode()
+
+		if errorReading != nil {
+			t.Error(errorReading)
+			return
+		}
+
+		list, _ := evaluator.EvaluateNodes(tt.expression, candidateNode)
+		test.AssertResultComplex(t, tt.expected, resultsToString(t, list))
 	}
 }

pkg/yqlib/base64_test.go

@@ -0,0 +1,269 @@
+//go:build !yq_nobase64
+
+package yqlib
+
+import (
+	"bufio"
+	"fmt"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+const base64EncodedSimple = "YSBzcGVjaWFsIHN0cmluZw=="
+const base64DecodedSimpleExtraSpaces = "\n " + base64EncodedSimple + "  \n"
+const base64DecodedSimple = "a special string"
+
+const base64EncodedUTF8 = "V29ya3Mgd2l0aCBVVEYtMTYg8J+Yig=="
+const base64DecodedUTF8 = "Works with UTF-16 😊"
+
+const base64EncodedYaml = "YTogYXBwbGUK"
+const base64DecodedYaml = "a: apple\n"
+
+const base64EncodedEmpty = ""
+const base64DecodedEmpty = ""
+
+const base64MissingPadding = "Y2F0cw"
+const base64DecodedMissingPadding = "cats"
+
+const base64EncodedCats = "Y2F0cw=="
+const base64DecodedCats = "cats"
+
+var base64Scenarios = []formatScenario{
+	{
+		skipDoc:      true,
+		description:  "empty decode",
+		input:        base64EncodedEmpty,
+		expected:     base64DecodedEmpty + "\n",
+		scenarioType: "decode",
+	},
+	{
+		skipDoc:      true,
+		description:  "simple decode",
+		input:        base64EncodedSimple,
+		expected:     base64DecodedSimple + "\n",
+		scenarioType: "decode",
+	},
+	{
+		description:    "Decode base64: simple",
+		subdescription: "Decoded data is assumed to be a string.",
+		input:          base64EncodedSimple,
+		expected:       base64DecodedSimple + "\n",
+		scenarioType:   "decode",
+	},
+	{
+		description:    "Decode base64: UTF-8",
+		subdescription: "Base64 decoding supports UTF-8 encoded strings.",
+		input:          base64EncodedUTF8,
+		expected:       base64DecodedUTF8 + "\n",
+		scenarioType:   "decode",
+	},
+	{
+		skipDoc:      true,
+		description:  "decode missing padding",
+		input:        base64MissingPadding,
+		expected:     base64DecodedMissingPadding + "\n",
+		scenarioType: "decode",
+	},
+	{
+
+		description:    "Decode with extra spaces",
+		subdescription: "Extra leading/trailing whitespace is stripped",
+		input:          base64DecodedSimpleExtraSpaces,
+		expected:       base64DecodedSimple + "\n",
+		scenarioType:   "decode",
+	},
+	{
+		skipDoc:      true,
+		description:  "decode with padding",
+		input:        base64EncodedCats,
+		expected:     base64DecodedCats + "\n",
+		scenarioType: "decode",
+	},
+	{
+		skipDoc:      true,
+		description:  "decode yaml document",
+		input:        base64EncodedYaml,
+		expected:     base64DecodedYaml + "\n",
+		scenarioType: "decode",
+	},
+	{
+		description:  "Encode base64: string",
+		input:        "\"" + base64DecodedSimple + "\"",
+		expected:     base64EncodedSimple,
+		scenarioType: "encode",
+	},
+	{
+		description:    "Encode base64: string from document",
+		subdescription: "Extract a string field and encode it to base64.",
+		input:          "coolData: \"" + base64DecodedSimple + "\"",
+		expression:     ".coolData",
+		expected:       base64EncodedSimple,
+		scenarioType:   "encode",
+	},
+	{
+		skipDoc:      true,
+		description:  "encode empty string",
+		input:        "\"\"",
+		expected:     "",
+		scenarioType: "encode",
+	},
+	{
+		skipDoc:      true,
+		description:  "encode UTF-8 string",
+		input:        "\"" + base64DecodedUTF8 + "\"",
+		expected:     base64EncodedUTF8,
+		scenarioType: "encode",
+	},
+	{
+		skipDoc:      true,
+		description:  "encode cats",
+		input:        "\"" + base64DecodedCats + "\"",
+		expected:     base64EncodedCats,
+		scenarioType: "encode",
+	},
+	{
+		description:  "Roundtrip: simple",
+		skipDoc:      true,
+		input:        base64EncodedSimple,
+		expected:     base64EncodedSimple,
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "Roundtrip: UTF-8",
+		skipDoc:      true,
+		input:        base64EncodedUTF8,
+		expected:     base64EncodedUTF8,
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "Roundtrip: missing padding",
+		skipDoc:      true,
+		input:        base64MissingPadding,
+		expected:     base64EncodedCats,
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "Roundtrip: empty",
+		skipDoc:      true,
+		input:        base64EncodedEmpty,
+		expected:     base64EncodedEmpty,
+		scenarioType: "roundtrip",
+	},
+	{
+		description:   "Encode error: non-string",
+		skipDoc:       true,
+		input:         "123",
+		expectedError: "cannot encode !!int as base64, can only operate on strings",
+		scenarioType:  "encode-error",
+	},
+	{
+		description:   "Encode error: array",
+		skipDoc:       true,
+		input:         "[1, 2, 3]",
+		expectedError: "cannot encode !!seq as base64, can only operate on strings",
+		scenarioType:  "encode-error",
+	},
+	{
+		description:   "Encode error: map",
+		skipDoc:       true,
+		input:         "{b: c}",
+		expectedError: "cannot encode !!map as base64, can only operate on strings",
+		scenarioType:  "encode-error",
+	},
+}
+
+func testBase64Scenario(t *testing.T, s formatScenario) {
+	switch s.scenarioType {
+	case "", "decode":
+		yamlPrefs := ConfiguredYamlPreferences.Copy()
+		yamlPrefs.Indent = 4
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewBase64Decoder(), NewYamlEncoder(yamlPrefs)), s.description)
+	case "encode":
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewBase64Encoder()), s.description)
+	case "roundtrip":
+		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewBase64Decoder(), NewBase64Encoder()), s.description)
+	case "encode-error":
+		result, err := processFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewBase64Encoder())
+		if err == nil {
+			t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result)
+		} else {
+			test.AssertResultComplexWithContext(t, s.expectedError, err.Error(), s.description)
+		}
+
+	default:
+		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
+	}
+}
+
+func documentBase64Scenario(_ *testing.T, w *bufio.Writer, i interface{}) {
+	s := i.(formatScenario)
+
+	if s.skipDoc {
+		return
+	}
+	switch s.scenarioType {
+	case "", "decode":
+		documentBase64DecodeScenario(w, s)
+	case "encode":
+		documentBase64EncodeScenario(w, s)
+
+	default:
+		panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
+	}
+}
+
+func documentBase64DecodeScenario(w *bufio.Writer, s formatScenario) {
+	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
+
+	if s.subdescription != "" {
+		writeOrPanic(w, s.subdescription)
+		writeOrPanic(w, "\n\n")
+	}
+
+	writeOrPanic(w, "Given a sample.txt file of:\n")
+	writeOrPanic(w, fmt.Sprintf("```\n%v\n```\n", s.input))
+
+	writeOrPanic(w, "then\n")
+	expression := s.expression
+	if expression == "" {
+		expression = "."
+	}
+	writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=base64 -oy '%v' sample.txt\n```\n", expression))
+	writeOrPanic(w, "will output\n")
+
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewBase64Decoder(), NewYamlEncoder(ConfiguredYamlPreferences))))
+}
+
+func documentBase64EncodeScenario(w *bufio.Writer, s formatScenario) {
+	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
+
+	if s.subdescription != "" {
+		writeOrPanic(w, s.subdescription)
+		writeOrPanic(w, "\n\n")
+	}
+
+	writeOrPanic(w, "Given a sample.yml file of:\n")
+	writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input))
+
+	writeOrPanic(w, "then\n")
+	expression := s.expression
+	if expression == "" {
+		expression = "."
+	}
+	writeOrPanic(w, fmt.Sprintf("```bash\nyq -o=base64 '%v' sample.yml\n```\n", expression))
+	writeOrPanic(w, "will output\n")
+
+	writeOrPanic(w, fmt.Sprintf("```\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewBase64Encoder())))
+}
+
+func TestBase64Scenarios(t *testing.T) {
+	for _, tt := range base64Scenarios {
+		testBase64Scenario(t, tt)
+	}
+	genericScenarios := make([]interface{}, len(base64Scenarios))
+	for i, s := range base64Scenarios {
+		genericScenarios[i] = s
+	}
+	documentScenarios(t, "usage", "base64", genericScenarios, documentBase64Scenario)
+}

pkg/yqlib/candidate_node.go

@@ -3,30 +3,215 @@ package yqlib
 import (
 	"container/list"
 	"fmt"
-
-	"github.com/jinzhu/copier"
-	yaml "gopkg.in/yaml.v3"
+	"strconv"
+	"strings"
 )
 
+type Kind uint32
+
+const (
+	SequenceNode Kind = 1 << iota
+	MappingNode
+	ScalarNode
+	AliasNode
+)
+
+type Style uint32
+
+const (
+	TaggedStyle Style = 1 << iota
+	DoubleQuotedStyle
+	SingleQuotedStyle
+	LiteralStyle
+	FoldedStyle
+	FlowStyle
+)
+
+func createStringScalarNode(stringValue string) *CandidateNode {
+	var node = &CandidateNode{Kind: ScalarNode}
+	node.Value = stringValue
+	node.Tag = "!!str"
+	return node
+}
+
+func createScalarNode(value interface{}, stringValue string) *CandidateNode {
+	var node = &CandidateNode{Kind: ScalarNode}
+	node.Value = stringValue
+
+	switch value.(type) {
+	case float32, float64:
+		node.Tag = "!!float"
+	case int, int64, int32:
+		node.Tag = "!!int"
+	case bool:
+		node.Tag = "!!bool"
+	case string:
+		node.Tag = "!!str"
+	case nil:
+		node.Tag = "!!null"
+	}
+	return node
+}
+
+type NodeInfo struct {
+	Kind        string      `yaml:"kind"`
+	Style       string      `yaml:"style,omitempty"`
+	Anchor      string      `yaml:"anchor,omitempty"`
+	Tag         string      `yaml:"tag,omitempty"`
+	HeadComment string      `yaml:"headComment,omitempty"`
+	LineComment string      `yaml:"lineComment,omitempty"`
+	FootComment string      `yaml:"footComment,omitempty"`
+	Value       string      `yaml:"value,omitempty"`
+	Line        int         `yaml:"line,omitempty"`
+	Column      int         `yaml:"column,omitempty"`
+	Content     []*NodeInfo `yaml:"content,omitempty"`
+}
+
 type CandidateNode struct {
-	Node      *yaml.Node     // the actual node
-	Parent    *CandidateNode // parent node
-	Path      []interface{}  /// the path we took to get to this node
-	Document  uint           // the document index of this node
-	Filename  string
-	FileIndex int
+	Kind  Kind
+	Style Style
+
+	Tag     string
+	Value   string
+	Anchor  string
+	Alias   *CandidateNode
+	Content []*CandidateNode
+
+	HeadComment string
+	LineComment string
+	FootComment string
+
+	Parent *CandidateNode // parent node
+	Key    *CandidateNode // node key, if this is a value from a map (or index in an array)
+
+	LeadingContent string
+
+	document uint // the document index of this node
+	filename string
+
+	Line   int
+	Column int
+
+	fileIndex int
 	// when performing op against all nodes given, this will treat all the nodes as one
-	// (e.g. top level cross document merge). This property does not propegate to child nodes.
+	// (e.g. top level cross document merge). This property does not propagate to child nodes.
 	EvaluateTogether bool
 	IsMapKey         bool
 }
 
+func (n *CandidateNode) CreateChild() *CandidateNode {
+	return &CandidateNode{
+		Parent: n,
+	}
+}
+
+func (n *CandidateNode) SetDocument(idx uint) {
+	n.document = idx
+}
+
+func (n *CandidateNode) GetDocument() uint {
+	// defer to parent
+	if n.Parent != nil {
+		return n.Parent.GetDocument()
+	}
+	return n.document
+}
+
+func (n *CandidateNode) SetFilename(name string) {
+	n.filename = name
+}
+
+func (n *CandidateNode) GetFilename() string {
+	if n.Parent != nil {
+		return n.Parent.GetFilename()
+	}
+	return n.filename
+}
+
+func (n *CandidateNode) SetFileIndex(idx int) {
+	n.fileIndex = idx
+}
+
+func (n *CandidateNode) GetFileIndex() int {
+	if n.Parent != nil {
+		return n.Parent.GetFileIndex()
+	}
+	return n.fileIndex
+}
+
 func (n *CandidateNode) GetKey() string {
 	keyPrefix := ""
 	if n.IsMapKey {
-		keyPrefix = "key-"
+		keyPrefix = fmt.Sprintf("key-%v-", n.Value)
 	}
-	return fmt.Sprintf("%v%v - %v", keyPrefix, n.Document, n.Path)
+	key := ""
+	if n.Key != nil {
+		key = n.Key.Value
+	}
+	return fmt.Sprintf("%v%v - %v", keyPrefix, n.GetDocument(), key)
+}
+
+func (n *CandidateNode) getParsedKey() interface{} {
+	if n.IsMapKey {
+		return n.Value
+	}
+	if n.Key == nil {
+		return nil
+	}
+	if n.Key.Tag == "!!str" {
+		return n.Key.Value
+	}
+	index, err := parseInt(n.Key.Value)
+	if err != nil {
+		return n.Key.Value
+	}
+	return index
+
+}
+
+func (n *CandidateNode) FilterMapContentByKey(keyPredicate func(*CandidateNode) bool) []*CandidateNode {
+	var result []*CandidateNode
+	for index := 0; index < len(n.Content); index = index + 2 {
+		keyNode := n.Content[index]
+		valueNode := n.Content[index+1]
+		if keyPredicate(keyNode) {
+			result = append(result, keyNode, valueNode)
+		}
+	}
+	return result
+}
+
+func (n *CandidateNode) GetPath() []interface{} {
+	key := n.getParsedKey()
+	if n.Parent != nil && key != nil {
+		return append(n.Parent.GetPath(), key)
+	}
+
+	if key != nil {
+		return []interface{}{key}
+	}
+	return make([]interface{}, 0)
+}
+
+func (n *CandidateNode) GetNicePath() string {
+	var sb strings.Builder
+	path := n.GetPath()
+	for i, element := range path {
+		elementStr := fmt.Sprintf("%v", element)
+		switch element.(type) {
+		case int:
+			sb.WriteString("[" + elementStr + "]")
+		default:
+			if i == 0 {
+				sb.WriteString(elementStr)
+			} else if strings.ContainsRune(elementStr, '.') {
+				sb.WriteString("[" + elementStr + "]")
+			} else {
+				sb.WriteString("." + elementStr)
+			}
+		}
+	}
+	return sb.String()
 }
 
 func (n *CandidateNode) AsList() *list.List {
@@ -35,74 +220,321 @@ func (n *CandidateNode) AsList() *list.List {
 	return elMap
 }
 
-func (n *CandidateNode) CreateChild(path interface{}, node *yaml.Node) *CandidateNode {
-	return &CandidateNode{
-		Node:      node,
-		Path:      n.createChildPath(path),
-		Parent:    n,
-		Document:  n.Document,
-		Filename:  n.Filename,
-		FileIndex: n.FileIndex,
+func (n *CandidateNode) SetParent(parent *CandidateNode) {
+	n.Parent = parent
+}
+
+type ValueVisitor func(*CandidateNode) error
+
+func (n *CandidateNode) VisitValues(visitor ValueVisitor) error {
+	switch n.Kind {
+	case MappingNode:
+		for i := 1; i < len(n.Content); i = i + 2 {
+			if err := visitor(n.Content[i]); err != nil {
+				return err
+			}
+		}
+	case SequenceNode:
+		for i := 0; i < len(n.Content); i = i + 1 {
+			if err := visitor(n.Content[i]); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (n *CandidateNode) CanVisitValues() bool {
+	return n.Kind == MappingNode || n.Kind == SequenceNode
+}
+
+func (n *CandidateNode) AddKeyValueChild(rawKey *CandidateNode, rawValue *CandidateNode) (*CandidateNode, *CandidateNode) {
+	key := rawKey.Copy()
+	key.SetParent(n)
+	key.IsMapKey = true
+
+	value := rawValue.Copy()
+	value.SetParent(n)
+	value.IsMapKey = false // force this, incase we are creating a value from a key
+	value.Key = key
+
+	n.Content = append(n.Content, key, value)
+	return key, value
+}
+
+func (n *CandidateNode) AddChild(rawChild *CandidateNode) {
+	value := rawChild.Copy()
+	value.SetParent(n)
+	value.IsMapKey = false
+
+	index := len(n.Content)
+	keyNode := createScalarNode(index, fmt.Sprintf("%v", index))
+	keyNode.SetParent(n)
+	value.Key = keyNode
+
+	n.Content = append(n.Content, value)
+}
+
+func (n *CandidateNode) AddChildren(children []*CandidateNode) {
+	if n.Kind == MappingNode {
+		for i := 0; i < len(children); i += 2 {
+			key := children[i]
+			value := children[i+1]
+			n.AddKeyValueChild(key, value)
+		}
+
+	} else {
+		for _, rawChild := range children {
+			n.AddChild(rawChild)
+		}
 	}
 }
 
-func (n *CandidateNode) createChildPath(path interface{}) []interface{} {
-	if path == nil {
-		newPath := make([]interface{}, len(n.Path))
-		copy(newPath, n.Path)
-		return newPath
+func (n *CandidateNode) GetValueRep() (interface{}, error) {
+	log.Debugf("GetValueRep for %v value: %v", n.GetNicePath(), n.Value)
+	realTag := n.guessTagFromCustomType()
+
+	switch realTag {
+	case "!!int":
+		_, val, err := parseInt64(n.Value)
+		return val, err
+	case "!!float":
+		// need to test this
+		return strconv.ParseFloat(n.Value, 64)
+	case "!!bool":
+		return isTruthyNode(n), nil
+	case "!!null":
+		return nil, nil
 	}
 
-	//don't use append as they may actually modify the path of the orignal node!
-	newPath := make([]interface{}, len(n.Path)+1)
-	copy(newPath, n.Path)
-	newPath[len(n.Path)] = path
-	return newPath
+	return n.Value, nil
 }
 
-func (n *CandidateNode) Copy() (*CandidateNode, error) {
-	clone := &CandidateNode{}
-	err := copier.Copy(clone, n)
-	if err != nil {
-		return nil, err
+func (n *CandidateNode) guessTagFromCustomType() string {
+	if strings.HasPrefix(n.Tag, "!!") {
+		return n.Tag
+	} else if n.Value == "" {
+		log.Debug("guessTagFromCustomType: node has no value to guess the type with")
+		return n.Tag
 	}
-	return clone, nil
+	dataBucket, errorReading := parseSnippet(n.Value)
+
+	if errorReading != nil {
+		log.Debug("guessTagFromCustomType: could not guess underlying tag type %v", errorReading)
+		return n.Tag
+	}
+	guessedTag := dataBucket.Tag
+	log.Info("im guessing the tag %v is a %v", n.Tag, guessedTag)
+	return guessedTag
+}
+
+func (n *CandidateNode) CreateReplacement(kind Kind, tag string, value string) *CandidateNode {
+	node := &CandidateNode{
+		Kind:  kind,
+		Tag:   tag,
+		Value: value,
+	}
+	return n.CopyAsReplacement(node)
+}
+
+func (n *CandidateNode) CopyAsReplacement(replacement *CandidateNode) *CandidateNode {
+	newCopy := replacement.Copy()
+	newCopy.Parent = n.Parent
+
+	if n.IsMapKey {
+		newCopy.Key = n
+	} else {
+		newCopy.Key = n.Key
+	}
+
+	return newCopy
+}
+
+func (n *CandidateNode) CreateReplacementWithComments(kind Kind, tag string, style Style) *CandidateNode {
+	replacement := n.CreateReplacement(kind, tag, "")
+	replacement.LeadingContent = n.LeadingContent
+	replacement.HeadComment = n.HeadComment
+	replacement.LineComment = n.LineComment
+	replacement.FootComment = n.FootComment
+	replacement.Style = style
+	return replacement
+}
+
+func (n *CandidateNode) Copy() *CandidateNode {
+	return n.doCopy(true)
+}
+
+func (n *CandidateNode) CopyWithoutContent() *CandidateNode {
+	return n.doCopy(false)
+}
+
+func (n *CandidateNode) doCopy(cloneContent bool) *CandidateNode {
+	var content []*CandidateNode
+
+	var copyKey *CandidateNode
+	if n.Key != nil {
+		copyKey = n.Key.Copy()
+	}
+
+	clone := &CandidateNode{
+		Kind:  n.Kind,
+		Style: n.Style,
+
+		Tag:    n.Tag,
+		Value:  n.Value,
+		Anchor: n.Anchor,
+
+		// ok not to clone this,
+		// as its a reference to somewhere else.
+		Alias:   n.Alias,
+		Content: content,
+
+		HeadComment: n.HeadComment,
+		LineComment: n.LineComment,
+		FootComment: n.FootComment,
+
+		Parent: n.Parent,
+		Key:    copyKey,
+
+		LeadingContent: n.LeadingContent,
+
+		document:  n.document,
+		filename:  n.filename,
+		fileIndex: n.fileIndex,
+
+		Line:   n.Line,
+		Column: n.Column,
+
+		EvaluateTogether: n.EvaluateTogether,
+		IsMapKey:         n.IsMapKey,
+	}
+
+	if cloneContent {
+		clone.AddChildren(n.Content)
+	}
+
+	return clone
 }
 
 // updates this candidate from the given candidate node
-func (n *CandidateNode) UpdateFrom(other *CandidateNode) {
+func (n *CandidateNode) UpdateFrom(other *CandidateNode, prefs assignPreferences) {
+	if n == other {
+		log.Debugf("UpdateFrom, no need to update from myself.")
+		return
+	}
+	// if this is an empty map or empty array, use the style of other node.
+	if (n.Kind != ScalarNode && len(n.Content) == 0) ||
+		// if the tag has changed (e.g. from str to bool)
+		(n.guessTagFromCustomType() != other.guessTagFromCustomType()) {
+		n.Style = other.Style
+	}
+
+	n.Content = make([]*CandidateNode, 0)
+	n.Kind = other.Kind
+	n.AddChildren(other.Content)
+
+	n.Value = other.Value
+
+	n.UpdateAttributesFrom(other, prefs)
 
-	n.UpdateAttributesFrom(other)
-	n.Node.Content = other.Node.Content
-	n.Node.Value = other.Node.Value
 }
 
-func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode) {
-	log.Debug("UpdateAttributesFrom: n: %v other: %v", n.GetKey(), other.GetKey())
-	if n.Node.Kind != other.Node.Kind {
+func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignPreferences) {
+	log.Debug("UpdateAttributesFrom: n: %v other: %v", NodeToString(n), NodeToString(other))
+	if n.Kind != other.Kind {
 		// clear out the contents when switching to a different type
 		// e.g. map to array
-		n.Node.Content = make([]*yaml.Node, 0)
-		n.Node.Value = ""
+		n.Content = make([]*CandidateNode, 0)
+		n.Value = ""
+	}
+	n.Kind = other.Kind
+
+	// don't clobber custom tags...
+	if prefs.ClobberCustomTags || strings.HasPrefix(n.Tag, "!!") || n.Tag == "" {
+		n.Tag = other.Tag
+	}
+
+	n.Alias = other.Alias
+
+	if !prefs.DontOverWriteAnchor {
+		n.Anchor = other.Anchor
 	}
-	n.Node.Kind = other.Node.Kind
-	n.Node.Tag = other.Node.Tag
-	n.Node.Alias = other.Node.Alias
-	n.Node.Anchor = other.Node.Anchor
 
 	// merge will pickup the style of the new thing
 	// when autocreating nodes
-	if n.Node.Style == 0 {
-		n.Node.Style = other.Node.Style
+
+	if n.Style == 0 {
+		n.Style = other.Style
 	}
 
-	if other.Node.FootComment != "" {
-		n.Node.FootComment = other.Node.FootComment
+	if other.FootComment != "" {
+		n.FootComment = other.FootComment
 	}
-	if other.Node.HeadComment != "" {
-		n.Node.HeadComment = other.Node.HeadComment
+	if other.HeadComment != "" {
+		n.HeadComment = other.HeadComment
 	}
-	if other.Node.LineComment != "" {
-		n.Node.LineComment = other.Node.LineComment
+	if other.LineComment != "" {
+		n.LineComment = other.LineComment
 	}
 }
+
+func (n *CandidateNode) ConvertToNodeInfo() *NodeInfo {
+	info := &NodeInfo{
+		Kind:        kindToString(n.Kind),
+		Style:       styleToString(n.Style),
+		Anchor:      n.Anchor,
+		Tag:         n.Tag,
+		HeadComment: n.HeadComment,
+		LineComment: n.LineComment,
+		FootComment: n.FootComment,
+		Value:       n.Value,
+		Line:        n.Line,
+		Column:      n.Column,
+	}
+	if len(n.Content) > 0 {
+		info.Content = make([]*NodeInfo, len(n.Content))
+		for i, child := range n.Content {
+			info.Content[i] = child.ConvertToNodeInfo()
+		}
+	}
+	return info
+}
+
+// Helper functions to convert Kind and Style to string for NodeInfo
+func kindToString(k Kind) string {
+	switch k {
+	case SequenceNode:
+		return "SequenceNode"
+	case MappingNode:
+		return "MappingNode"
+	case ScalarNode:
+		return "ScalarNode"
+	case AliasNode:
+		return "AliasNode"
+	default:
+		return "Unknown"
+	}
+}
+
+func styleToString(s Style) string {
+	var styles []string
+	if s&TaggedStyle != 0 {
+		styles = append(styles, "TaggedStyle")
+	}
+	if s&DoubleQuotedStyle != 0 {
+		styles = append(styles, "DoubleQuotedStyle")
+	}
+	if s&SingleQuotedStyle != 0 {
+		styles = append(styles, "SingleQuotedStyle")
+	}
+	if s&LiteralStyle != 0 {
+		styles = append(styles, "LiteralStyle")
+	}
+	if s&FoldedStyle != 0 {
+		styles = append(styles, "FoldedStyle")
+	}
+	if s&FlowStyle != 0 {
+		styles = append(styles, "FlowStyle")
+	}
+	return strings.Join(styles, ",")
+}

pkg/yqlib/candidate_node_goccy_yaml.go

@@ -0,0 +1,210 @@
+package yqlib
+
+import (
+	"fmt"
+	"strings"
+
+	yaml "github.com/goccy/go-yaml"
+	"github.com/goccy/go-yaml/ast"
+	goccyToken "github.com/goccy/go-yaml/token"
+)
+
+func (o *CandidateNode) goccyDecodeIntoChild(childNode ast.Node, cm yaml.CommentMap, anchorMap map[string]*CandidateNode) (*CandidateNode, error) {
+	newChild := o.CreateChild()
+
+	err := newChild.UnmarshalGoccyYAML(childNode, cm, anchorMap)
+	return newChild, err
+}
+
+func (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap, anchorMap map[string]*CandidateNode) error {
+	log.Debugf("UnmarshalYAML %v", node)
+	log.Debugf("UnmarshalYAML %v", node.Type().String())
+	log.Debugf("UnmarshalYAML Node Value: %v", node.String())
+	log.Debugf("UnmarshalYAML Node GetComment: %v", node.GetComment())
+
+	if node.GetComment() != nil {
+		commentMapComments := cm[node.GetPath()]
+		for _, comment := range node.GetComment().Comments {
+			// need to use the comment map to find the position :/
+			log.Debugf("%v has a comment of [%v]", node.GetPath(), comment.Token.Value)
+			for _, commentMapComment := range commentMapComments {
+				commentMapValue := strings.Join(commentMapComment.Texts, "\n")
+				if commentMapValue == comment.Token.Value {
+					log.Debug("found a matching entry in comment map")
+					// we found the comment in the comment map,
+					// now we can process the position
+					switch commentMapComment.Position {
+					case yaml.CommentHeadPosition:
+						o.HeadComment = comment.String()
+						log.Debug("its a head comment %v", comment.String())
+					case yaml.CommentLinePosition:
+						o.LineComment = comment.String()
+						log.Debug("its a line comment %v", comment.String())
+					case yaml.CommentFootPosition:
+						o.FootComment = comment.String()
+						log.Debug("its a foot comment %v", comment.String())
+					}
+				}
+			}
+
+		}
+	}
+
+	o.Value = node.String()
+	o.Line = node.GetToken().Position.Line
+	o.Column = node.GetToken().Position.Column
+
+	switch node.Type() {
+	case ast.IntegerType:
+		o.Kind = ScalarNode
+		o.Tag = "!!int"
+	case ast.FloatType:
+		o.Kind = ScalarNode
+		o.Tag = "!!float"
+	case ast.BoolType:
+		o.Kind = ScalarNode
+		o.Tag = "!!bool"
+	case ast.NullType:
+		log.Debugf("its a null type with value %v", node.GetToken().Value)
+		o.Kind = ScalarNode
+		o.Tag = "!!null"
+		o.Value = node.GetToken().Value
+		if node.GetToken().Type == goccyToken.ImplicitNullType {
+			o.Value = ""
+		}
+	case ast.StringType:
+		o.Kind = ScalarNode
+		o.Tag = "!!str"
+		switch node.GetToken().Type {
+		case goccyToken.SingleQuoteType:
+			o.Style = SingleQuotedStyle
+		case goccyToken.DoubleQuoteType:
+			o.Style = DoubleQuotedStyle
+		}
+		o.Value = node.(*ast.StringNode).Value
+		log.Debugf("string value %v", node.(*ast.StringNode).Value)
+	case ast.LiteralType:
+		o.Kind = ScalarNode
+		o.Tag = "!!str"
+		o.Style = LiteralStyle
+		astLiteral := node.(*ast.LiteralNode)
+		log.Debugf("astLiteral.Start.Type %v", astLiteral.Start.Type)
+		if astLiteral.Start.Type == goccyToken.FoldedType {
+			log.Debugf("folded Type %v", astLiteral.Start.Type)
+			o.Style = FoldedStyle
+		}
+		log.Debug("start value: %v ", node.(*ast.LiteralNode).Start.Value)
+		log.Debug("start value: %v ", node.(*ast.LiteralNode).Start.Type)
+		// TODO: here I could put the original value with line breaks
+		// to solve the multiline > problem
+		o.Value = astLiteral.Value.Value
+	case ast.TagType:
+		if err := o.UnmarshalGoccyYAML(node.(*ast.TagNode).Value, cm, anchorMap); err != nil {
+			return err
+		}
+		o.Tag = node.(*ast.TagNode).Start.Value
+	case ast.MappingType:
+		log.Debugf("UnmarshalYAML -  a mapping node")
+		o.Kind = MappingNode
+		o.Tag = "!!map"
+
+		mappingNode := node.(*ast.MappingNode)
+		if mappingNode.IsFlowStyle {
+			o.Style = FlowStyle
+		}
+		for _, mappingValueNode := range mappingNode.Values {
+			err := o.goccyProcessMappingValueNode(mappingValueNode, cm, anchorMap)
+			if err != nil {
+				return err
+			}
+		}
+		if mappingNode.FootComment != nil {
+			log.Debugf("mapping node has a foot comment of: %v", mappingNode.FootComment)
+			o.FootComment = mappingNode.FootComment.String()
+		}
+	case ast.MappingValueType:
+		log.Debugf("UnmarshalYAML -  a mapping node")
+		o.Kind = MappingNode
+		o.Tag = "!!map"
+		mappingValueNode := node.(*ast.MappingValueNode)
+		err := o.goccyProcessMappingValueNode(mappingValueNode, cm, anchorMap)
+		if err != nil {
+			return err
+		}
+	case ast.SequenceType:
+		log.Debugf("UnmarshalYAML -  a sequence node")
+		o.Kind = SequenceNode
+		o.Tag = "!!seq"
+		sequenceNode := node.(*ast.SequenceNode)
+		if sequenceNode.IsFlowStyle {
+			o.Style = FlowStyle
+		}
+		astSeq := sequenceNode.Values
+		o.Content = make([]*CandidateNode, len(astSeq))
+		for i := 0; i < len(astSeq); i++ {
+			keyNode := o.CreateChild()
+			keyNode.IsMapKey = true
+			keyNode.Tag = "!!int"
+			keyNode.Kind = ScalarNode
+			keyNode.Value = fmt.Sprintf("%v", i)
+
+			valueNode, err := o.goccyDecodeIntoChild(astSeq[i], cm, anchorMap)
+			if err != nil {
+				return err
+			}
+
+			valueNode.Key = keyNode
+			o.Content[i] = valueNode
+		}
+	case ast.AnchorType:
+		log.Debugf("UnmarshalYAML -  an anchor node")
+		anchorNode := node.(*ast.AnchorNode)
+		err := o.UnmarshalGoccyYAML(anchorNode.Value, cm, anchorMap)
+		if err != nil {
+			return err
+		}
+		o.Anchor = anchorNode.Name.String()
+		anchorMap[o.Anchor] = o
+
+	case ast.AliasType:
+		log.Debugf("UnmarshalYAML -  an alias node")
+		aliasNode := node.(*ast.AliasNode)
+		o.Kind = AliasNode
+		o.Value = aliasNode.Value.String()
+		o.Alias = anchorMap[o.Value]
+
+	case ast.MergeKeyType:
+		log.Debugf("UnmarshalYAML -  a merge key")
+		o.Kind = ScalarNode
+		o.Tag = "!!merge" // note - I should be able to get rid of this.
+		o.Value = "<<"
+
+	default:
+		log.Debugf("UnmarshalYAML -  no idea of the type!!\n%v: %v", node.Type(), node.String())
+	}
+	log.Debugf("KIND: %v", o.Kind)
+	return nil
+}
+
+func (o *CandidateNode) goccyProcessMappingValueNode(mappingEntry *ast.MappingValueNode, cm yaml.CommentMap, anchorMap map[string]*CandidateNode) error {
+	log.Debug("UnmarshalYAML MAP KEY entry %v", mappingEntry.Key)
+
+	// AddKeyValueFirst because it clones the nodes, and we want to have the real refs when Unmarshalling
+	// particularly for the anchorMap
+	keyNode, valueNode := o.AddKeyValueChild(&CandidateNode{}, &CandidateNode{})
+
+	if err := keyNode.UnmarshalGoccyYAML(mappingEntry.Key, cm, anchorMap); err != nil {
+		return err
+	}
+
+	log.Debug("UnmarshalYAML MAP VALUE entry %v", mappingEntry.Value)
+	if err := valueNode.UnmarshalGoccyYAML(mappingEntry.Value, cm, anchorMap); err != nil {
+		return err
+	}
+
+	if mappingEntry.FootComment != nil {
+		valueNode.FootComment = mappingEntry.FootComment.String()
+	}
+
+	return nil
+}

pkg/yqlib/candidate_node_test.go

@@ -0,0 +1,394 @@
+package yqlib
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+type valueRepScenario struct {
+	input    string
+	tag      string
+	expected interface{}
+}
+
+var valueRepScenarios = []valueRepScenario{
+	{
+		input:    `"cat"`,
+		expected: `"cat"`,
+	},
+	{
+		input:    `3`,
+		expected: int64(3),
+	},
+	{
+		input:    `3.1`,
+		expected: float64(3.1),
+	},
+	{
+		input:    `true`,
+		expected: true,
+	},
+	{
+		input:    `y`,
+		tag:      "!!bool",
+		expected: true,
+	},
+	{
+		tag:      "!!null",
+		expected: nil,
+	},
+}
+
+func TestCandidateNodeGetValueRepScenarios(t *testing.T) {
+	for _, tt := range valueRepScenarios {
+		node := CandidateNode{Value: tt.input, Tag: tt.tag}
+		actual, err := node.GetValueRep()
+		if err != nil {
+			t.Error(err)
+			return
+		}
+		test.AssertResult(t, tt.expected, actual)
+	}
+}
+
+func TestCandidateNodeChildWhenParentUpdated(t *testing.T) {
+	parent := CandidateNode{}
+	child := parent.CreateChild()
+	parent.SetDocument(1)
+	parent.SetFileIndex(2)
+	parent.SetFilename("meow")
+	test.AssertResultWithContext(t, "meow", child.GetFilename(), "filename")
+	test.AssertResultWithContext(t, 2, child.GetFileIndex(), "file index")
+	test.AssertResultWithContext(t, uint(1), child.GetDocument(), "document index")
+}
+
+type createScalarNodeScenario struct {
+	value       interface{}
+	stringValue string
+	expectedTag string
+}
+
+var createScalarScenarios = []createScalarNodeScenario{
+	{
+		value:       "mike",
+		stringValue: "mike",
+		expectedTag: "!!str",
+	},
+	{
+		value:       3,
+		stringValue: "3",
+		expectedTag: "!!int",
+	},
+	{
+		value:       3.1,
+		stringValue: "3.1",
+		expectedTag: "!!float",
+	},
+	{
+		value:       true,
+		stringValue: "true",
+		expectedTag: "!!bool",
+	},
+	{
+		value:       nil,
+		stringValue: "~",
+		expectedTag: "!!null",
+	},
+}
+
+func TestCreateScalarNodeScenarios(t *testing.T) {
+	for _, tt := range createScalarScenarios {
+		actual := createScalarNode(tt.value, tt.stringValue)
+		test.AssertResultWithContext(t, tt.stringValue, actual.Value, fmt.Sprintf("Value for: Value: [%v], String: %v", tt.value, tt.stringValue))
+		test.AssertResultWithContext(t, tt.expectedTag, actual.Tag, fmt.Sprintf("Value for: Value: [%v], String: %v", tt.value, tt.stringValue))
+	}
+}
+
+func TestGetKeyForMapValue(t *testing.T) {
+	key := createStringScalarNode("yourKey")
+	n := CandidateNode{Key: key, Value: "meow", document: 3}
+	test.AssertResult(t, "3 - yourKey", n.GetKey())
+}
+
+func TestGetKeyForMapKey(t *testing.T) {
+	key := createStringScalarNode("yourKey")
+	key.IsMapKey = true
+	key.document = 3
+	test.AssertResult(t, "key-yourKey-3 - ", key.GetKey())
+}
+
+func TestGetKeyForValue(t *testing.T) {
+	n := CandidateNode{Value: "meow", document: 3}
+	test.AssertResult(t, "3 - ", n.GetKey())
+}
+
+func TestGetParsedKeyForMapKey(t *testing.T) {
+	key := createStringScalarNode("yourKey")
+	key.IsMapKey = true
+	key.document = 3
+	test.AssertResult(t, "yourKey", key.getParsedKey())
+}
+
+func TestGetParsedKeyForLooseValue(t *testing.T) {
+	n := CandidateNode{Value: "meow", document: 3}
+	test.AssertResult(t, nil, n.getParsedKey())
+}
+
+func TestGetParsedKeyForMapValue(t *testing.T) {
+	key := createStringScalarNode("yourKey")
+	n := CandidateNode{Key: key, Value: "meow", document: 3}
+	test.AssertResult(t, "yourKey", n.getParsedKey())
+}
+
+func TestGetParsedKeyForArrayValue(t *testing.T) {
+	key := createScalarNode(4, "4")
+	n := CandidateNode{Key: key, Value: "meow", document: 3}
+	test.AssertResult(t, 4, n.getParsedKey())
+}
+
+func TestCandidateNodeAddKeyValueChild(t *testing.T) {
+	key := CandidateNode{Value: "cool", IsMapKey: true}
+	node := CandidateNode{}
+
+	_, keyIsValueNow := node.AddKeyValueChild(&CandidateNode{Value: "newKey"}, &key)
+
+	test.AssertResult(t, keyIsValueNow.IsMapKey, false)
+	test.AssertResult(t, key.IsMapKey, true)
+
+}
+
+func TestConvertToNodeInfo(t *testing.T) {
+	child := &CandidateNode{
+		Kind:   ScalarNode,
+		Style:  DoubleQuotedStyle,
+		Tag:    "!!str",
+		Value:  "childValue",
+		Line:   2,
+		Column: 3,
+	}
+	parent := &CandidateNode{
+		Kind:        MappingNode,
+		Style:       TaggedStyle,
+		Tag:         "!!map",
+		Value:       "",
+		Line:        1,
+		Column:      1,
+		Content:     []*CandidateNode{child},
+		HeadComment: "head",
+		LineComment: "line",
+		FootComment: "foot",
+		Anchor:      "anchor",
+	}
+	info := parent.ConvertToNodeInfo()
+	test.AssertResult(t, "MappingNode", info.Kind)
+	test.AssertResult(t, "TaggedStyle", info.Style)
+	test.AssertResult(t, "!!map", info.Tag)
+	test.AssertResult(t, "head", info.HeadComment)
+	test.AssertResult(t, "line", info.LineComment)
+	test.AssertResult(t, "foot", info.FootComment)
+	test.AssertResult(t, "anchor", info.Anchor)
+	test.AssertResult(t, 1, info.Line)
+	test.AssertResult(t, 1, info.Column)
+	if len(info.Content) != 1 {
+		t.Fatalf("Expected 1 child, got %d", len(info.Content))
+	}
+	childInfo := info.Content[0]
+	test.AssertResult(t, "ScalarNode", childInfo.Kind)
+	test.AssertResult(t, "DoubleQuotedStyle", childInfo.Style)
+	test.AssertResult(t, "!!str", childInfo.Tag)
+	test.AssertResult(t, "childValue", childInfo.Value)
+	test.AssertResult(t, 2, childInfo.Line)
+	test.AssertResult(t, 3, childInfo.Column)
+}
+
+func TestCandidateNodeGetPath(t *testing.T) {
+	// Test root node with no parent
+	root := CandidateNode{Value: "root"}
+	path := root.GetPath()
+	test.AssertResult(t, 0, len(path))
+
+	// Test node with key
+	key := createStringScalarNode("myKey")
+	node := CandidateNode{Key: key, Value: "myValue"}
+	path = node.GetPath()
+	test.AssertResult(t, 1, len(path))
+	test.AssertResult(t, "myKey", path[0])
+
+	// Test nested path
+	parent := CandidateNode{}
+	parentKey := createStringScalarNode("parent")
+	parent.Key = parentKey
+	node.Parent = &parent
+	path = node.GetPath()
+	test.AssertResult(t, 2, len(path))
+	test.AssertResult(t, "parent", path[0])
+	test.AssertResult(t, "myKey", path[1])
+}
+
+func TestCandidateNodeGetNicePath(t *testing.T) {
+	// Test simple key
+	key := createStringScalarNode("simple")
+	node := CandidateNode{Key: key}
+	nicePath := node.GetNicePath()
+	test.AssertResult(t, "simple", nicePath)
+
+	// Test array index
+	arrayKey := createScalarNode(0, "0")
+	arrayNode := CandidateNode{Key: arrayKey}
+	nicePath = arrayNode.GetNicePath()
+	test.AssertResult(t, "[0]", nicePath)
+
+	dotKey := createStringScalarNode("key.with.dots")
+	dotNode := CandidateNode{Key: dotKey}
+	nicePath = dotNode.GetNicePath()
+	test.AssertResult(t, "key.with.dots", nicePath)
+
+	// Test nested path
+	parentKey := createStringScalarNode("parent")
+	parent := CandidateNode{Key: parentKey}
+	childKey := createStringScalarNode("child")
+	child := CandidateNode{Key: childKey, Parent: &parent}
+	nicePath = child.GetNicePath()
+	test.AssertResult(t, "parent.child", nicePath)
+}
+
+func TestCandidateNodeFilterMapContentByKey(t *testing.T) {
+	// Create a map with multiple key-value pairs
+	key1 := createStringScalarNode("key1")
+	value1 := createStringScalarNode("value1")
+	key2 := createStringScalarNode("key2")
+	value2 := createStringScalarNode("value2")
+	key3 := createStringScalarNode("key3")
+	value3 := createStringScalarNode("value3")
+
+	mapNode := &CandidateNode{
+		Kind:    MappingNode,
+		Content: []*CandidateNode{key1, value1, key2, value2, key3, value3},
+	}
+
+	// Filter by key predicate that matches key1 and key3
+	filtered := mapNode.FilterMapContentByKey(func(key *CandidateNode) bool {
+		return key.Value == "key1" || key.Value == "key3"
+	})
+
+	// Should return key1, value1, key3, value3
+	test.AssertResult(t, 4, len(filtered))
+	test.AssertResult(t, "key1", filtered[0].Value)
+	test.AssertResult(t, "value1", filtered[1].Value)
+	test.AssertResult(t, "key3", filtered[2].Value)
+	test.AssertResult(t, "value3", filtered[3].Value)
+}
+
+func TestCandidateNodeVisitValues(t *testing.T) {
+	// Test mapping node
+	key1 := createStringScalarNode("key1")
+	value1 := createStringScalarNode("value1")
+	key2 := createStringScalarNode("key2")
+	value2 := createStringScalarNode("value2")
+
+	mapNode := &CandidateNode{
+		Kind:    MappingNode,
+		Content: []*CandidateNode{key1, value1, key2, value2},
+	}
+
+	var visited []string
+	err := mapNode.VisitValues(func(node *CandidateNode) error {
+		visited = append(visited, node.Value)
+		return nil
+	})
+
+	test.AssertResult(t, nil, err)
+	test.AssertResult(t, 2, len(visited))
+	test.AssertResult(t, "value1", visited[0])
+	test.AssertResult(t, "value2", visited[1])
+
+	// Test sequence node
+	item1 := createStringScalarNode("item1")
+	item2 := createStringScalarNode("item2")
+
+	seqNode := &CandidateNode{
+		Kind:    SequenceNode,
+		Content: []*CandidateNode{item1, item2},
+	}
+
+	visited = []string{}
+	err = seqNode.VisitValues(func(node *CandidateNode) error {
+		visited = append(visited, node.Value)
+		return nil
+	})
+
+	test.AssertResult(t, nil, err)
+	test.AssertResult(t, 2, len(visited))
+	test.AssertResult(t, "item1", visited[0])
+	test.AssertResult(t, "item2", visited[1])
+
+	// Test scalar node (should not visit anything)
+	scalarNode := &CandidateNode{
+		Kind:  ScalarNode,
+		Value: "scalar",
+	}
+
+	visited = []string{}
+	err = scalarNode.VisitValues(func(node *CandidateNode) error {
+		visited = append(visited, node.Value)
+		return nil
+	})
+
+	test.AssertResult(t, nil, err)
+	test.AssertResult(t, 0, len(visited))
+}
+
+func TestCandidateNodeCanVisitValues(t *testing.T) {
+	mapNode := &CandidateNode{Kind: MappingNode}
+	seqNode := &CandidateNode{Kind: SequenceNode}
+	scalarNode := &CandidateNode{Kind: ScalarNode}
+
+	test.AssertResult(t, true, mapNode.CanVisitValues())
+	test.AssertResult(t, true, seqNode.CanVisitValues())
+	test.AssertResult(t, false, scalarNode.CanVisitValues())
+}
+
+func TestCandidateNodeAddChild(t *testing.T) {
+	parent := &CandidateNode{Kind: SequenceNode}
+	child := createStringScalarNode("child")
+
+	parent.AddChild(child)
+
+	test.AssertResult(t, 1, len(parent.Content))
+	test.AssertResult(t, false, parent.Content[0].IsMapKey)
+	test.AssertResult(t, "0", parent.Content[0].Key.Value)
+	// Check that parent is set correctly
+	if parent.Content[0].Parent != parent {
+		t.Errorf("Expected parent to be set correctly")
+	}
+}
+
+func TestCandidateNodeAddChildren(t *testing.T) {
+	// Test sequence node
+	parent := &CandidateNode{Kind: SequenceNode}
+	child1 := createStringScalarNode("child1")
+	child2 := createStringScalarNode("child2")
+
+	parent.AddChildren([]*CandidateNode{child1, child2})
+
+	test.AssertResult(t, 2, len(parent.Content))
+	test.AssertResult(t, "child1", parent.Content[0].Value)
+	test.AssertResult(t, "child2", parent.Content[1].Value)
+
+	// Test mapping node
+	mapParent := &CandidateNode{Kind: MappingNode}
+	key1 := createStringScalarNode("key1")
+	value1 := createStringScalarNode("value1")
+	key2 := createStringScalarNode("key2")
+	value2 := createStringScalarNode("value2")
+
+	mapParent.AddChildren([]*CandidateNode{key1, value1, key2, value2})
+
+	test.AssertResult(t, 4, len(mapParent.Content))
+	test.AssertResult(t, true, mapParent.Content[0].IsMapKey)  // key1
+	test.AssertResult(t, false, mapParent.Content[1].IsMapKey) // value1
+	test.AssertResult(t, true, mapParent.Content[2].IsMapKey)  // key2
+	test.AssertResult(t, false, mapParent.Content[3].IsMapKey) // value2
+}

pkg/yqlib/candidate_node_yaml.go

@@ -0,0 +1,215 @@
+package yqlib
+
+import (
+	"fmt"
+
+	yaml "go.yaml.in/yaml/v4"
+)
+
+func MapYamlStyle(original yaml.Style) Style {
+	switch original {
+	case yaml.TaggedStyle:
+		return TaggedStyle
+	case yaml.DoubleQuotedStyle:
+		return DoubleQuotedStyle
+	case yaml.SingleQuotedStyle:
+		return SingleQuotedStyle
+	case yaml.LiteralStyle:
+		return LiteralStyle
+	case yaml.FoldedStyle:
+		return FoldedStyle
+	case yaml.FlowStyle:
+		return FlowStyle
+	case 0:
+		return 0
+	}
+	return Style(original)
+}
+
+func MapToYamlStyle(original Style) yaml.Style {
+	switch original {
+	case TaggedStyle:
+		return yaml.TaggedStyle
+	case DoubleQuotedStyle:
+		return yaml.DoubleQuotedStyle
+	case SingleQuotedStyle:
+		return yaml.SingleQuotedStyle
+	case LiteralStyle:
+		return yaml.LiteralStyle
+	case FoldedStyle:
+		return yaml.FoldedStyle
+	case FlowStyle:
+		return yaml.FlowStyle
+	case 0:
+		return 0
+	}
+	return yaml.Style(original)
+}
+
+func (o *CandidateNode) copyFromYamlNode(node *yaml.Node, anchorMap map[string]*CandidateNode) {
+	o.Style = MapYamlStyle(node.Style)
+
+	o.Tag = node.Tag
+	o.Value = node.Value
+	o.Anchor = node.Anchor
+
+	if o.Anchor != "" {
+		anchorMap[o.Anchor] = o
+		log.Debug("set anchor %v to %v", o.Anchor, NodeToString(o))
+	}
+
+	// its a single alias
+	if node.Alias != nil && node.Alias.Anchor != "" {
+		o.Alias = anchorMap[node.Alias.Anchor]
+		log.Debug("set alias to %v", NodeToString(anchorMap[node.Alias.Anchor]))
+	}
+	o.HeadComment = node.HeadComment
+	o.LineComment = node.LineComment
+	o.FootComment = node.FootComment
+
+	o.Line = node.Line
+	o.Column = node.Column
+}
+
+func (o *CandidateNode) copyToYamlNode(node *yaml.Node) {
+	node.Style = MapToYamlStyle(o.Style)
+
+	node.Tag = o.Tag
+	node.Value = o.Value
+	node.Anchor = o.Anchor
+
+	node.HeadComment = o.HeadComment
+
+	node.LineComment = o.LineComment
+	node.FootComment = o.FootComment
+
+	node.Line = o.Line
+	node.Column = o.Column
+}
+
+func (o *CandidateNode) decodeIntoChild(childNode *yaml.Node, anchorMap map[string]*CandidateNode) (*CandidateNode, error) {
+	newChild := o.CreateChild()
+
+	// null yaml.Nodes to not end up calling UnmarshalYAML
+	// so we call it explicitly
+	if childNode.Tag == "!!null" {
+		newChild.Kind = ScalarNode
+		newChild.copyFromYamlNode(childNode, anchorMap)
+		return newChild, nil
+	}
+
+	err := newChild.UnmarshalYAML(childNode, anchorMap)
+	return newChild, err
+}
+
+func (o *CandidateNode) UnmarshalYAML(node *yaml.Node, anchorMap map[string]*CandidateNode) error {
+	log.Debugf("UnmarshalYAML %v", node.Tag)
+	switch node.Kind {
+	case yaml.AliasNode:
+		log.Debug("UnmarshalYAML - alias from yaml: %v", o.Tag)
+		o.Kind = AliasNode
+		o.copyFromYamlNode(node, anchorMap)
+		return nil
+	case yaml.ScalarNode:
+		log.Debugf("UnmarshalYAML -  a scalar")
+		o.Kind = ScalarNode
+		o.copyFromYamlNode(node, anchorMap)
+		return nil
+	case yaml.MappingNode:
+		log.Debugf("UnmarshalYAML -  a mapping node")
+		o.Kind = MappingNode
+		o.copyFromYamlNode(node, anchorMap)
+		o.Content = make([]*CandidateNode, len(node.Content))
+		for i := 0; i < len(node.Content); i += 2 {
+
+			keyNode, err := o.decodeIntoChild(node.Content[i], anchorMap)
+			if err != nil {
+				return err
+			}
+
+			keyNode.IsMapKey = true
+
+			valueNode, err := o.decodeIntoChild(node.Content[i+1], anchorMap)
+			if err != nil {
+				return err
+			}
+
+			valueNode.Key = keyNode
+
+			o.Content[i] = keyNode
+			o.Content[i+1] = valueNode
+		}
+		log.Debugf("UnmarshalYAML -  finished mapping node")
+		return nil
+	case yaml.SequenceNode:
+		log.Debugf("UnmarshalYAML -  a sequence: %v", len(node.Content))
+		o.Kind = SequenceNode
+
+		o.copyFromYamlNode(node, anchorMap)
+		log.Debugf("node Style: %v", node.Style)
+		log.Debugf("o Style: %v", o.Style)
+		o.Content = make([]*CandidateNode, len(node.Content))
+		for i := 0; i < len(node.Content); i++ {
+			keyNode := o.CreateChild()
+			keyNode.IsMapKey = true
+			keyNode.Tag = "!!int"
+			keyNode.Kind = ScalarNode
+			keyNode.Value = fmt.Sprintf("%v", i)
+
+			valueNode, err := o.decodeIntoChild(node.Content[i], anchorMap)
+			if err != nil {
+				return err
+			}
+
+			valueNode.Key = keyNode
+			o.Content[i] = valueNode
+		}
+		return nil
+	case 0:
+		// not sure when this happens
+		o.copyFromYamlNode(node, anchorMap)
+		log.Debugf("UnmarshalYAML -  err.. %v", NodeToString(o))
+		return nil
+	default:
+		return fmt.Errorf("orderedMap: invalid yaml node")
+	}
+}
+
+func (o *CandidateNode) MarshalYAML() (*yaml.Node, error) {
+	log.Debug("MarshalYAML to yaml: %v", o.Tag)
+	switch o.Kind {
+	case AliasNode:
+		log.Debug("MarshalYAML - alias to yaml: %v", o.Tag)
+		target := &yaml.Node{Kind: yaml.AliasNode}
+		o.copyToYamlNode(target)
+		return target, nil
+	case ScalarNode:
+		log.Debug("MarshalYAML - scalar: %v", o.Value)
+		target := &yaml.Node{Kind: yaml.ScalarNode}
+		o.copyToYamlNode(target)
+		return target, nil
+	case MappingNode, SequenceNode:
+		targetKind := yaml.MappingNode
+		if o.Kind == SequenceNode {
+			targetKind = yaml.SequenceNode
+		}
+		target := &yaml.Node{Kind: targetKind}
+		o.copyToYamlNode(target)
+		log.Debugf("original style: %v", o.Style)
+		log.Debugf("original: %v, tag: %v, style: %v, kind: %v", NodeToString(o), target.Tag, target.Style, target.Kind == yaml.SequenceNode)
+		target.Content = make([]*yaml.Node, len(o.Content))
+		for i := 0; i < len(o.Content); i++ {
+
+			child, err := o.Content[i].MarshalYAML()
+
+			if err != nil {
+				return nil, err
+			}
+			target.Content[i] = child
+		}
+		return target, nil
+	}
+	target := &yaml.Node{}
+	o.copyToYamlNode(target)
+	return target, nil
+}

pkg/yqlib/candidiate_node_json.go

@@ -0,0 +1,179 @@
+//go:build !yq_nojson
+
+package yqlib
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+
+	"github.com/goccy/go-json"
+)
+
+func (o *CandidateNode) setScalarFromJson(value interface{}) error {
+	o.Kind = ScalarNode
+	switch rawData := value.(type) {
+	case nil:
+		o.Tag = "!!null"
+		o.Value = "null"
+	case float64, float32:
+		o.Value = fmt.Sprintf("%v", value)
+		o.Tag = "!!float"
+		// json decoder returns ints as float.
+		if value == float64(int64(rawData.(float64))) {
+			// aha it's an int disguised as a float
+			o.Tag = "!!int"
+			o.Value = fmt.Sprintf("%v", int64(value.(float64)))
+		}
+	case int, int64, int32:
+		o.Value = fmt.Sprintf("%v", value)
+		o.Tag = "!!int"
+	case bool:
+		o.Value = fmt.Sprintf("%v", value)
+		o.Tag = "!!bool"
+	case string:
+		o.Value = rawData
+		o.Tag = "!!str"
+	default:
+		return fmt.Errorf("unrecognised type :( %v", rawData)
+	}
+	return nil
+}
+
+func (o *CandidateNode) UnmarshalJSON(data []byte) error {
+	log.Debug("UnmarshalJSON")
+	switch data[0] {
+	case '{':
+		log.Debug("UnmarshalJSON -  its a map!")
+		// its a map
+		o.Kind = MappingNode
+		o.Tag = "!!map"
+
+		dec := json.NewDecoder(bytes.NewReader(data))
+		_, err := dec.Token() // open object
+		if err != nil {
+			return err
+		}
+
+		// cycle through k/v
+		var tok json.Token
+		for tok, err = dec.Token(); err == nil; tok, err = dec.Token() {
+			// we can expect two types: string or Delim. Delim automatically means
+			// that it is the closing bracket of the object, whereas string means
+			// that there is another key.
+			if _, ok := tok.(json.Delim); ok {
+				break
+			}
+
+			childKey := o.CreateChild()
+			childKey.IsMapKey = true
+			childKey.Value = tok.(string)
+			childKey.Kind = ScalarNode
+			childKey.Tag = "!!str"
+
+			childValue := o.CreateChild()
+			childValue.Key = childKey
+
+			if err := dec.Decode(childValue); err != nil {
+				return err
+			}
+			o.Content = append(o.Content, childKey, childValue)
+		}
+		// unexpected error
+		if err != nil && !errors.Is(err, io.EOF) {
+			return err
+		}
+		return nil
+	case '[':
+		o.Kind = SequenceNode
+		o.Tag = "!!seq"
+		log.Debug("UnmarshalJSON -  its an array!")
+		var children []*CandidateNode
+		if err := json.Unmarshal(data, &children); err != nil {
+			return err
+		}
+		// now we put the children into the content, and set a key value for them
+		for i, child := range children {
+
+			if child == nil {
+				// need to represent it as a null scalar
+				child = createScalarNode(nil, "null")
+			}
+			childKey := o.CreateChild()
+			childKey.Kind = ScalarNode
+			childKey.Tag = "!!int"
+			childKey.Value = fmt.Sprintf("%v", i)
+			childKey.IsMapKey = true
+
+			child.Parent = o
+			child.Key = childKey
+			o.Content = append(o.Content, child)
+		}
+		return nil
+	}
+	log.Debug("UnmarshalJSON -  its a scalar!")
+	// otherwise, must be a scalar
+	var scalar interface{}
+	err := json.Unmarshal(data, &scalar)
+
+	if err != nil {
+		return err
+	}
+	log.Debug("UnmarshalJSON -  scalar is %v", scalar)
+
+	return o.setScalarFromJson(scalar)
+
+}
+
+func (o *CandidateNode) MarshalJSON() ([]byte, error) {
+	log.Debugf("MarshalJSON %v", NodeToString(o))
+	buf := new(bytes.Buffer)
+	enc := json.NewEncoder(buf)
+	enc.SetIndent("", " ")
+	enc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
+
+	switch o.Kind {
+	case AliasNode:
+		log.Debugf("MarshalJSON AliasNode")
+		err := enc.Encode(o.Alias)
+		return buf.Bytes(), err
+	case ScalarNode:
+		log.Debugf("MarshalJSON ScalarNode")
+		value, err := o.GetValueRep()
+		if err != nil {
+			return buf.Bytes(), err
+		}
+		err = enc.Encode(value)
+		return buf.Bytes(), err
+	case MappingNode:
+		log.Debugf("MarshalJSON MappingNode")
+		buf.WriteByte('{')
+		for i := 0; i < len(o.Content); i += 2 {
+			if err := enc.Encode(o.Content[i].Value); err != nil {
+				return nil, err
+			}
+			buf.WriteByte(':')
+			if err := enc.Encode(o.Content[i+1]); err != nil {
+				return nil, err
+			}
+			if i != len(o.Content)-2 {
+				buf.WriteByte(',')
+			}
+		}
+		buf.WriteByte('}')
+		return buf.Bytes(), nil
+	case SequenceNode:
+		log.Debugf("MarshalJSON SequenceNode, %v, len: %v", o.Content, len(o.Content))
+		var err error
+		if len(o.Content) == 0 {
+			buf.WriteString("[]")
+		} else {
+			err = enc.Encode(o.Content)
+		}
+		return buf.Bytes(), err
+	default:
+		err := enc.Encode(nil)
+		return buf.Bytes(), err
+	}
+}

pkg/yqlib/chown_linux.go

@@ -0,0 +1,25 @@
+//go:build linux
+
+package yqlib
+
+import (
+	"io/fs"
+	"os"
+	"syscall"
+)
+
+func changeOwner(info fs.FileInfo, file *os.File) error {
+	if stat, ok := info.Sys().(*syscall.Stat_t); ok {
+		uid := int(stat.Uid)
+		gid := int(stat.Gid)
+
+		err := os.Chown(file.Name(), uid, gid)
+		if err != nil {
+			// this happens with snap confinement
+			// not really a big issue as users can chown
+			// the file themselves if required.
+			log.Info("Skipping chown: %v", err)
+		}
+	}
+	return nil
+}

pkg/yqlib/chown_linux_test.go

@@ -0,0 +1,139 @@
+//go:build linux
+
+package yqlib
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestChangeOwner(t *testing.T) {
+	// Create a temporary file for testing
+	tempDir := t.TempDir()
+	testFile := filepath.Join(tempDir, "testfile.txt")
+
+	// Create a test file
+	err := os.WriteFile(testFile, []byte("test content"), 0600)
+	if err != nil {
+		t.Fatalf("Failed to create test file: %v", err)
+	}
+
+	// Get file info
+	info, err := os.Stat(testFile)
+	if err != nil {
+		t.Fatalf("Failed to stat test file: %v", err)
+	}
+
+	// Create another temporary file to change ownership of
+	tempFile, err := os.CreateTemp(tempDir, "chown_test_*.txt")
+	if err != nil {
+		t.Fatalf("Failed to create temp file: %v", err)
+	}
+	defer os.Remove(tempFile.Name())
+	tempFile.Close()
+
+	// Test changeOwner function
+	err = changeOwner(info, tempFile)
+	if err != nil {
+		t.Errorf("changeOwner failed: %v", err)
+	}
+
+	// Verify that the function doesn't panic with valid input
+	tempFile2, err := os.CreateTemp(tempDir, "chown_test2_*.txt")
+	if err != nil {
+		t.Fatalf("Failed to create second temp file: %v", err)
+	}
+	defer os.Remove(tempFile2.Name())
+	tempFile2.Close()
+
+	// Test with the second file
+	err = changeOwner(info, tempFile2)
+	if err != nil {
+		t.Errorf("changeOwner failed on second file: %v", err)
+	}
+}
+
+func TestChangeOwnerWithInvalidFileInfo(t *testing.T) {
+	// Create a mock file info that doesn't have syscall.Stat_t
+	mockInfo := &mockFileInfo{
+		name: "mock",
+		size: 0,
+		mode: 0600,
+	}
+
+	// Create a temporary file
+	tempFile, err := os.CreateTemp(t.TempDir(), "chown_test_*.txt")
+	if err != nil {
+		t.Fatalf("Failed to create temp file: %v", err)
+	}
+	defer os.Remove(tempFile.Name())
+	tempFile.Close()
+
+	// Test changeOwner with mock file info (should not panic)
+	err = changeOwner(mockInfo, tempFile)
+	if err != nil {
+		t.Errorf("changeOwner failed with mock file info: %v", err)
+	}
+}
+
+func TestChangeOwnerWithNonExistentFile(t *testing.T) {
+	// Create a temporary file
+	tempFile, err := os.CreateTemp(t.TempDir(), "chown_test_*.txt")
+	if err != nil {
+		t.Fatalf("Failed to create temp file: %v", err)
+	}
+	defer os.Remove(tempFile.Name())
+	tempFile.Close()
+
+	// Get file info
+	info, err := os.Stat(tempFile.Name())
+	if err != nil {
+		t.Fatalf("Failed to stat temp file: %v", err)
+	}
+
+	// Remove the file
+	os.Remove(tempFile.Name())
+
+	err = changeOwner(info, tempFile)
+	// The function should not panic even if the file doesn't exist
+	if err != nil {
+		t.Logf("Expected error when changing owner of non-existent file: %v", err)
+	}
+}
+
+// mockFileInfo implements fs.FileInfo but doesn't have syscall.Stat_t
+type mockFileInfo struct {
+	name string
+	size int64
+	mode os.FileMode
+}
+
+func (m *mockFileInfo) Name() string       { return m.name }
+func (m *mockFileInfo) Size() int64        { return m.size }
+func (m *mockFileInfo) Mode() os.FileMode  { return m.mode }
+func (m *mockFileInfo) ModTime() time.Time { return time.Time{} }
+func (m *mockFileInfo) IsDir() bool        { return false }
+func (m *mockFileInfo) Sys() interface{}   { return nil } // This will cause the type assertion to fail
+
+func TestChangeOwnerWithSyscallStatT(t *testing.T) {
+	// Create a temporary file
+	tempFile, err := os.CreateTemp(t.TempDir(), "chown_test_*.txt")
+	if err != nil {
+		t.Fatalf("Failed to create temp file: %v", err)
+	}
+	defer os.Remove(tempFile.Name())
+	tempFile.Close()
+
+	// Get file info
+	info, err := os.Stat(tempFile.Name())
+	if err != nil {
+		t.Fatalf("Failed to stat temp file: %v", err)
+	}
+
+	err = changeOwner(info, tempFile)
+	if err != nil {
+		t.Logf("changeOwner returned error (this might be expected in some environments): %v", err)
+	}
+}

pkg/yqlib/chown_not_linux_os.go

@@ -0,0 +1,12 @@
+//go:build !linux
+
+package yqlib
+
+import (
+	"io/fs"
+	"os"
+)
+
+func changeOwner(_ fs.FileInfo, _ *os.File) error {
+	return nil
+}