Compare commits

..
Author SHA1 Message Date
7bc2d0ca11 Fix spelling: multibyte -> multi-byte in Unicode test subdescription
Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/6e7b304b-5b52-4e89-8bad-ba22813305c7

Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com>
2026-04-06 08:51:11 +00:00
adb150d1fb S2-S4: tighten lexer condition, fix doc header, add Unicode example
Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/ec06083e-e20a-45d2-bf7e-4e1fa7be1073

Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com>
2026-04-06 08:45:00 +00:00
1afa3c73dc Merge master: use clampSliceIndex in sliceStringNode, resolve conflicts
Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com>
2026-04-06 08:44:35 +00:00
341e2524b9 Fix array slice out-of-bounds panic with very negative indices
Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/7c146762-d251-45fd-8555-2488f59fc57b

Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com>
2026-03-28 03:20:37 +00:00
778088d70c Update pkg/yqlib/operator_slice.go
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2026-03-28 14:18:51 +11:00
9a9399ad00 Fix sliceStringNode signature and fix test descriptions/expressions
Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/58726b13-68ae-4f93-971f-eb70459edcf4

Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com>
2026-03-28 02:01:01 +00:00
6d345ac795 Add string slicing support to yq
Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/a8525fbb-77a7-4bb0-a3a7-b24f99ae8710

Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com>
2026-03-27 10:12:55 +00:00
copilot-swe-agent[bot]andGitHub 2692987998 Initial plan 2026-03-27 10:01:02 +00:00
97 changed files with 703 additions and 2415 deletions
+4 -6
View File
@@ -20,8 +20,6 @@ on:
schedule:
- cron: '24 3 * * 1'
permissions: {}
jobs:
analyze:
name: Analyze
@@ -40,11 +38,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@v6
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
uses: github/codeql-action/init@v4
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
@@ -55,7 +53,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
uses: github/codeql-action/autobuild@v4
# ️ Command-line programs to run using the OS shell.
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -69,4 +67,4 @@ jobs:
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7
uses: github/codeql-action/analyze@v4
-107
View File
@@ -1,107 +0,0 @@
name: Release Docker GitHub Action
on:
workflow_dispatch:
permissions: {}
jobs:
publishGithubActionDocker:
environment: dockerhub
env:
IMAGE_NAME: mikefarah/yq
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- name: Set up QEMU
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
with:
platforms: all
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
with:
version: latest
- name: Verify Dockerfile base image digest matches yq:4
run: |
PINNED_DIGEST=$(grep -oE 'sha256:[a-f0-9]{64}' github-action/Dockerfile | head -1)
if [ -z "${PINNED_DIGEST}" ]; then
echo "::error::Could not find a sha256 digest in github-action/Dockerfile"
exit 1
fi
LATEST_DIGEST=$(docker buildx imagetools inspect "${IMAGE_NAME}:4" --format '{{printf "%s" .Manifest.Digest}}')
echo "Dockerfile pins: ${PINNED_DIGEST}"
echo "mikefarah/yq:4 is: ${LATEST_DIGEST}"
if [ "${PINNED_DIGEST}" != "${LATEST_DIGEST}" ]; then
echo "::error::github-action/Dockerfile digest does not match the current mikefarah/yq:4 image"
echo "Update the FROM line in github-action/Dockerfile to:"
echo " FROM mikefarah/yq:4@${LATEST_DIGEST}"
exit 1
fi
- name: Resolve version from yq:4
run: |
IMAGE_VERSION=$(docker run --rm "${IMAGE_NAME}:4" --version | awk '{print $NF}' | sed 's/^v//')
if [ -z "${IMAGE_VERSION}" ]; then
echo "::error::Could not determine yq version from ${IMAGE_NAME}:4"
exit 1
fi
echo "Resolved yq version: ${IMAGE_VERSION}"
echo "IMAGE_VERSION=${IMAGE_VERSION}" >> "${GITHUB_ENV}"
- name: Login to Docker Hub
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push github-action image
working-directory: github-action
run: |
PLATFORMS="linux/amd64,linux/ppc64le,linux/arm64,linux/arm/v7,linux/s390x"
echo "Building and pushing github-action image for version ${IMAGE_VERSION}"
docker buildx build \
--label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
--label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
--label "org.opencontainers.image.description=yq is a portable command-line data file processor" \
--label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
--label "org.opencontainers.image.licenses=MIT" \
--label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \
--label "org.opencontainers.image.source=https://github.com/mikefarah/yq" \
--label "org.opencontainers.image.title=yq" \
--label "org.opencontainers.image.url=https://mikefarah.gitbook.io/yq/" \
--label "org.opencontainers.image.version=${IMAGE_VERSION}" \
--platform "${PLATFORMS}" \
--pull \
--push \
-t "${IMAGE_NAME}:${IMAGE_VERSION}-githubaction" \
-t "${IMAGE_NAME}:4-githubaction" \
-t "${IMAGE_NAME}:latest-githubaction" \
-t "ghcr.io/${IMAGE_NAME}:${IMAGE_VERSION}-githubaction" \
-t "ghcr.io/${IMAGE_NAME}:4-githubaction" \
-t "ghcr.io/${IMAGE_NAME}:latest-githubaction" \
.
- name: Report action.yml digest to pin
run: |
GITHUBACTION_DIGEST=$(docker buildx imagetools inspect "${IMAGE_NAME}:4-githubaction" --format '{{printf "%s" .Manifest.Digest}}')
echo "Published ${IMAGE_NAME}:4-githubaction at ${GITHUBACTION_DIGEST}"
echo "Update action.yml image to:"
echo " docker://${IMAGE_NAME}:4-githubaction@${GITHUBACTION_DIGEST}"
+28 -10
View File
@@ -7,28 +7,23 @@ on:
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
permissions: {}
jobs:
publishDocker:
environment: dockerhub
env:
IMAGE_NAME: mikefarah/yq
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@v6
- name: Set up QEMU
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
uses: docker/setup-qemu-action@v4
with:
platforms: all
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
uses: docker/setup-buildx-action@v4
with:
version: latest
@@ -36,13 +31,13 @@ jobs:
run: echo ${{ steps.buildx.outputs.platforms }} && docker version
- name: Login to Docker Hub
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
uses: docker/login-action@v4
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
uses: docker/login-action@v4
with:
registry: ghcr.io
username: ${{ github.actor }}
@@ -80,3 +75,26 @@ jobs:
-t "ghcr.io/${IMAGE_NAME}:4" \
-t "ghcr.io/${IMAGE_NAME}:latest" \
.
cd github-action
docker buildx build \
--label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
--label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
--label "org.opencontainers.image.description=yq is a portable command-line data file processor" \
--label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
--label "org.opencontainers.image.licenses=MIT" \
--label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \
--label "org.opencontainers.image.source=https://github.com/mikefarah/yq" \
--label "org.opencontainers.image.title=yq" \
--label "org.opencontainers.image.url=https://mikefarah.gitbook.io/yq/" \
--label "org.opencontainers.image.version=${IMAGE_VERSION}" \
--platform "${PLATFORMS}" \
--pull \
--push \
-t "${IMAGE_NAME}:${IMAGE_VERSION}-githubaction" \
-t "${IMAGE_NAME}:4-githubaction" \
-t "${IMAGE_NAME}:latest-githubaction" \
-t "ghcr.io/${IMAGE_NAME}:${IMAGE_VERSION}-githubaction" \
-t "ghcr.io/${IMAGE_NAME}:4-githubaction" \
-t "ghcr.io/${IMAGE_NAME}:latest-githubaction" \
.
+3 -29
View File
@@ -5,51 +5,25 @@ permissions:
jobs:
verify-action-digest:
name: Verify action.yml image digest
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- name: Verify action.yml digest matches published image
run: |
PINNED_DIGEST=$(grep -oE 'sha256:[a-f0-9]{64}' action.yml | head -1)
if [ -z "${PINNED_DIGEST}" ]; then
echo "::error::action.yml does not pin the runtime image by digest"
exit 1
fi
LATEST_DIGEST=$(docker buildx imagetools inspect docker.io/mikefarah/yq:4-githubaction --format '{{printf "%s" .Manifest.Digest}}')
echo "action.yml pins: ${PINNED_DIGEST}"
echo "mikefarah/yq:4-githubaction: ${LATEST_DIGEST}"
if [ "${PINNED_DIGEST}" != "${LATEST_DIGEST}" ]; then
echo "::error::action.yml digest does not match the current mikefarah/yq:4-githubaction image"
echo "Update the image line in action.yml to:"
echo " docker://mikefarah/yq:4-githubaction@${LATEST_DIGEST}"
exit 1
fi
build:
name: Build
runs-on: ubuntu-latest
steps:
- name: Set up Go
uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
uses: actions/setup-go@v6
with:
go-version: '^1.20'
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
uses: actions/checkout@v6
- name: Get dependencies
run: |
go get -v -t -d ./...
if [ -f Gopkg.toml ]; then
curl -sSfL https://raw.githubusercontent.com/golang/dep/1f7c19e5f52f49ffb9f956f64c010be14683468b/install.sh | env DEP_RELEASE_TAG=v0.5.4 sh
curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
dep ensure
fi
+5 -18
View File
@@ -5,17 +5,12 @@ on:
- 'v4.*'
- 'draft-*'
permissions: {}
jobs:
publishGitRelease:
runs-on: ubuntu-latest
permissions:
contents: write
id-token: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
- uses: actions/checkout@v6
- uses: actions/setup-go@v6
with:
go-version: '^1.20'
check-latest: true
@@ -29,7 +24,7 @@ jobs:
run: echo "VERSION=${GITHUB_REF##*/}" >> "${GITHUB_OUTPUT}"
- name: Generate man page
uses: docker://pandoc/core:2.14.2@sha256:04e127c6642a2b9d447c26fe0ac6a5932efa8f508eda9f07da51b6e621dd7c19
uses: docker://pandoc/core:2.14.2
id: gen-man-page
with:
args: >-
@@ -42,22 +37,14 @@ jobs:
--output=yq.1
man.md
- name: Install cosign
uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
- name: Cross compile
run: |
sudo apt-get install rhash -y
go install github.com/goreleaser/goreleaser/v2@v2.16.0
go install github.com/goreleaser/goreleaser/v2@latest
./scripts/xcompile.sh
- name: Sign checksums
run: |
cosign sign-blob --yes --bundle build/checksums.bundle build/checksums
cosign sign-blob --yes --bundle build/checksums-bsd.bundle build/checksums-bsd
- name: Release
uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1
uses: softprops/action-gh-release@v1
with:
files: build/*
draft: true
-78
View File
@@ -1,78 +0,0 @@
# This workflow uses actions that are not certified by GitHub. They are provided
# by a third-party and are governed by separate terms of service, privacy
# policy, and support documentation.
name: Scorecard supply-chain security
on:
# For Branch-Protection check. Only the default branch is supported. See
# https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
branch_protection_rule:
# To guarantee Maintained check is occasionally updated. See
# https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
schedule:
- cron: '39 7 * * 2'
push:
branches: [ "master" ]
# Declare default permissions as read only.
permissions: read-all
jobs:
analysis:
name: Scorecard analysis
runs-on: ubuntu-latest
# `publish_results: true` only works when run from the default branch. conditional can be removed if disabled.
if: github.event.repository.default_branch == github.ref_name || github.event_name == 'pull_request'
permissions:
# Needed to upload the results to code-scanning dashboard.
security-events: write
# Needed to publish results and get a badge (see publish_results below).
id-token: write
# Uncomment the permissions below if installing in a private repository.
# contents: read
# actions: read
steps:
- name: "Checkout code"
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: "Run analysis"
uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4
with:
results_file: results.sarif
results_format: sarif
# (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
# - you want to enable the Branch-Protection check on a *public* repository, or
# - you are installing Scorecard on a *private* repository
# To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional.
# repo_token: ${{ secrets.SCORECARD_TOKEN }}
# Public repositories:
# - Publish results to OpenSSF REST API for easy access by consumers
# - Allows the repository to include the Scorecard badge.
# - See https://github.com/ossf/scorecard-action#publishing-results.
# For private repositories:
# - `publish_results` will always be set to `false`, regardless
# of the value entered here.
publish_results: true
# (Optional) Uncomment file_mode if you have a .gitattributes with files marked export-ignore
# file_mode: git
# Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
# format to the repository Actions tab.
- name: "Upload artifact"
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: SARIF file
path: results.sarif
retention-days: 5
# Upload the results to GitHub's code scanning dashboard (optional).
# Commenting out will disable upload of results to your repo's Code Scanning dashboard
- name: "Upload to code-scanning"
uses: github/codeql-action/upload-sarif@d1ba80a13dd99fba24a470575428917156a28b43 # v4.37.5
with:
sarif_file: results.sarif
+3 -7
View File
@@ -7,23 +7,19 @@ on:
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
permissions: {}
jobs:
buildSnap:
environment: snap
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: snapcore/action-build@3bdaa03e1ba6bf59a65f84a751d943d549a54e79 # v1.3.0
- uses: actions/checkout@v6
- uses: snapcore/action-build@v1
id: build
env:
SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.STORE_LOGIN }}
with:
snapcraft-args: "remote-build --launchpad-accept-public-upload"
- uses: snapcore/action-publish@214b86e5ca036ead1668c79afb81e550e6c54d40 # v1.2.0
- uses: snapcore/action-publish@v1
env:
SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.STORE_LOGIN }}
with:
+1 -1
View File
@@ -13,7 +13,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
- uses: actions/checkout@v6
- name: Get test
id: get_value
uses: mikefarah/yq@master
+2 -2
View File
@@ -1,4 +1,4 @@
FROM golang:1.26.5@sha256:079e59808d2d252516e27e3f3a9c003740dee7f75e55aa71528766d52bcfc16a AS builder
FROM golang:1.26.1 AS builder
WORKDIR /go/src/mikefarah/yq
@@ -10,7 +10,7 @@ RUN ./scripts/acceptance.sh
# Choose alpine as a base image to make this useful for CI, as many
# CI tools expect an interactive shell inside the container
FROM alpine:3@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b AS production
FROM alpine:3 AS production
LABEL maintainer="Mike Farah <mikefarah@users.noreply.github.com>"
COPY --from=builder /go/src/mikefarah/yq/yq /usr/bin/yq
+5 -1
View File
@@ -1,4 +1,8 @@
FROM golang:1.26.5@sha256:079e59808d2d252516e27e3f3a9c003740dee7f75e55aa71528766d52bcfc16a
FROM golang:1.26.1
RUN apt-get update && \
apt-get install -y npm && \
npm install -g npx cspell@latest
COPY scripts/devtools.sh /opt/devtools.sh
+1 -1
View File
@@ -42,7 +42,7 @@ quiet: # this is silly but shuts up 'Nothing to be done for `local`'
@:
prepare: tmp/dev_image_id
tmp/dev_image_id: Dockerfile.dev scripts/devtools.sh _typos.toml
tmp/dev_image_id: Dockerfile.dev scripts/devtools.sh
@mkdir -p tmp
@${ENGINE} rmi -f ${DEV_IMAGE} > /dev/null 2>&1 || true
@${ENGINE} build -t ${DEV_IMAGE} -f Dockerfile.dev .
-26
View File
@@ -1,26 +0,0 @@
# Security Policy
## Reporting a Vulnerability
Please **do not** report security vulnerabilities through public GitHub issues.
Instead, use GitHub's private vulnerability reporting feature:
👉 https://github.com/mikefarah/yq/security
This allows vulnerabilities to be triaged and addressed confidentially before any public disclosure.
## Scope
### HTTP / TLS / Network vulnerabilities
yq is a command-line YAML/JSON/TOML processor that reads from files or standard input and writes to standard output. **yq does not include any HTTP or network libraries** and makes no network connections at runtime. CVEs related to HTTP, TLS, or networking are therefore **not applicable** to yq.
### Dependency version bumps
yq uses [Dependabot](https://docs.github.com/en/code-security/dependabot) to automatically raise pull requests for:
- Go module dependencies
- Go toolchain version
- Docker base images
Please **do not** raise pull requests or issues solely to bump dependency or Go versions — Dependabot handles this automatically and the maintainers merge those PRs regularly.
-20
View File
@@ -1,20 +0,0 @@
[files]
extend-exclude = ["vendor", "bin"]
[default]
locale = "en"
extend-ignore-identifiers-re = [
"NdJson",
]
[default.extend-identifiers]
AttributeIDSupressMenu = "AttributeIDSupressMenu"
[default.extend-words]
Teh = "Teh"
teh = "teh"
Supress = "Supress"
HashiCorp = "HashiCorp"
Hashi = "Hashi"
fot = "fot"
nd = "nd"
+1 -1
View File
@@ -12,6 +12,6 @@ outputs:
description: "The complete result from the yq command being run"
runs:
using: 'docker'
image: 'docker://mikefarah/yq:4-githubaction@sha256:e1b8c865f299ea6b02910a7ddf147d5d431244d4cc116f89c2148c9f53822906'
image: 'docker://mikefarah/yq:4-githubaction'
args:
- ${{ inputs.cmd }}
-78
View File
@@ -1,79 +1,3 @@
# yq — agent instructions
## ⚠️ MANDATORY: GitHub agent disclosure
**Always required. No exceptions.**
Whenever you perform **any** GitHub action on behalf of the user, you **must** disclose that an AI agent (Cursor) wrote the content and is acting on the user's behalf — **not the user personally**. Do this **before** submitting; never post first and add the disclosure later.
Applies to **all** GitHub interactions, including:
- Pull requests (titles, descriptions, and reviews)
- PR comments and inline review comments
- Issues (new issues, comments, and updates)
- Any other post or reply on GitHub
**How to disclose:** Put it prominently at the **top** of every PR description, review body, comment, or issue. Use wording like:
Inline review comments must include a short disclosure too (e.g. `> Generated by Cursor acting on the user's behalf, not the user personally.`).
**Never** submit a GitHub action without this disclosure.
---
Always run the spellcheck before raising a PR:
```bash
bash scripts/spelling.sh
```
This is also included in the full CI pipeline via `make local test`.
## Cursor Cloud specific instructions
### Overview
**yq** is a Go CLI for querying and transforming YAML, JSON, XML, INI, and other structured formats. There are no long-running services — development is build-and-test against a local `./yq` binary.
### Prerequisites
- **Go ≥ 1.25** (see `go.mod`)
- **Bash** (acceptance tests)
- **Docker/Podman** is optional; use `make local <target>` to run natively when containers are unavailable
### PATH
After `scripts/devtools.sh`, add Go tool binaries to PATH:
```bash
export PATH="$HOME/go/bin:$PATH"
```
`golangci-lint` and `typos` install to `$HOME/go/bin`; `gosec` installs to `./bin/gosec` in the repo root.
### Common commands (local, no Docker)
| Task | Command |
|------|---------|
| Install dev tools | `bash scripts/devtools.sh` |
| Vendor dependencies | `make local vendor` |
| Build binary | `go build -o yq .` or `make local build` |
| Format | `make local format` |
| Lint | `make local check` |
| Unit tests | `make local test` or `bash scripts/test.sh` |
| Acceptance (E2E) | `bash scripts/acceptance.sh` (requires `./yq` built first) |
`make local build` runs the full CI chain (format → spelling → gosec → lint → unit tests → build → acceptance). For a faster loop, build with `go build -o yq .` and run `bash scripts/acceptance.sh`.
### Caveats
- **`make` without `local`** tries Docker/Podman (`Dockerfile.dev`). In Cloud Agent VMs without Docker, always prefix with `make local`.
- **Spelling step** uses `typos` (installed by `scripts/devtools.sh`).
- **`make local test` / `scripts/check.sh`** require `golangci-lint` on PATH (`devtools.sh`).
---
# General rules
✅ **DO:**
- You can use ./yq with the `--debug-node-info` flag to get a deeper understanding of the ast.
@@ -81,12 +5,10 @@ export PATH="$HOME/go/bin:$PATH"
- Add comprehensive tests to cover the changes
- Run test suite to ensure there is no regression
- Use UK english spelling
- **Follow the mandatory GitHub agent disclosure rule above** on every GitHub action — no exceptions
❌ **DON'T:**
- Git add or commit
- Add comments to functions that are self-explanatory
- **Post to GitHub without the mandatory agent disclosure** (PRs, reviews, comments, issues, or any other GitHub interaction)
-3
View File
@@ -156,8 +156,6 @@ yq -P -oy sample.json
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.UnquotedKeys, "lua-unquoted", yqlib.ConfiguredLuaPreferences.UnquotedKeys, "output unquoted string keys (e.g. {foo=\"bar\"})")
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.Globals, "lua-globals", yqlib.ConfiguredLuaPreferences.Globals, "output keys as top-level global variables")
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredINIPreferences.PreserveSurroundedQuote, "ini-preserve-quotes", yqlib.ConfiguredINIPreferences.PreserveSurroundedQuote, "preserve surrounding quotes on INI values during round-trip")
rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, "properties-separator", yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, "separator to use between keys and values")
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredPropertiesPreferences.UseArrayBrackets, "properties-array-brackets", yqlib.ConfiguredPropertiesPreferences.UseArrayBrackets, "use [x] in array paths (e.g. for SpringBoot)")
@@ -214,7 +212,6 @@ yq -P -oy sample.json
rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableEnvOps, "security-disable-env-ops", "", false, "Disable env related operations.")
rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableFileOps, "security-disable-file-ops", "", false, "Disable file related operations (e.g. load)")
rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.EnableSystemOps, "security-enable-system-operator", "", false, "Enable system operator to allow execution of external commands.")
rootCmd.AddCommand(
createEvaluateSequenceCommand(),
-4
View File
@@ -87,10 +87,6 @@ func validateCommandFlags(args []string) error {
return fmt.Errorf("cannot pass files in when using null-input flag")
}
if indent < 0 {
return fmt.Errorf("indent must not be negative")
}
return nil
}
-25
View File
@@ -1086,7 +1086,6 @@ func TestValidateCommandFlags(t *testing.T) {
frontMatter string
splitFileExp string
nullInput bool
indent int
expectError bool
errorContains string
}{
@@ -1149,27 +1148,6 @@ func TestValidateCommandFlags(t *testing.T) {
expectError: true,
errorContains: "cannot pass files in when using null-input flag",
},
{
name: "negative indent",
args: []string{"file.yaml"},
writeInplace: false,
frontMatter: "",
splitFileExp: "",
nullInput: false,
indent: -1,
expectError: true,
errorContains: "indent must not be negative",
},
{
name: "zero indent is valid",
args: []string{"file.yaml"},
writeInplace: false,
frontMatter: "",
splitFileExp: "",
nullInput: false,
indent: 0,
expectError: false,
},
}
for _, tt := range tests {
@@ -1179,20 +1157,17 @@ func TestValidateCommandFlags(t *testing.T) {
originalFrontMatter := frontMatter
originalSplitFileExp := splitFileExp
originalNullInput := nullInput
originalIndent := indent
defer func() {
writeInplace = originalWriteInplace
frontMatter = originalFrontMatter
splitFileExp = originalSplitFileExp
nullInput = originalNullInput
indent = originalIndent
}()
writeInplace = tt.writeInplace
frontMatter = tt.frontMatter
splitFileExp = tt.splitFileExp
nullInput = tt.nullInput
indent = tt.indent
err := validateCommandFlags(tt.args)
if tt.expectError {
+1 -1
View File
@@ -11,7 +11,7 @@ var (
GitDescribe string
// Version is main version number that is being run at the moment.
Version = "v4.53.4"
Version = "v4.52.5"
// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
+14
View File
@@ -0,0 +1,14 @@
---
$schema: https://raw.githubusercontent.com/streetsidesoftware/cspell/main/cspell.schema.json
version: '0.2'
language: en-GB
dictionaryDefinitions:
- name: project-words
path: './project-words.txt'
addWords: true
dictionaries:
- project-words
ignorePaths:
- 'vendor'
- 'bin'
- '/project-words.txt'
+1 -1
View File
@@ -1,4 +1,4 @@
FROM mikefarah/yq:4@sha256:11a1f0b604b13dbbdc662260d8db6f644b22d8553122a25c1b5b2e8713ca6977
FROM mikefarah/yq:4
COPY entrypoint.sh /entrypoint.sh
+12 -13
View File
@@ -3,7 +3,7 @@ module github.com/mikefarah/yq/v4
require (
github.com/a8m/envsubst v1.4.3
github.com/alecthomas/participle/v2 v2.1.4
github.com/alecthomas/repr v0.5.4
github.com/alecthomas/repr v0.5.2
github.com/dimchansky/utfbom v1.1.1
github.com/elliotchance/orderedmap v1.8.0
github.com/fatih/color v1.19.0
@@ -12,31 +12,30 @@ require (
github.com/goccy/go-yaml v1.19.2
github.com/hashicorp/hcl/v2 v2.24.0
github.com/jinzhu/copier v0.4.0
github.com/magiconair/properties v1.18.11
github.com/pelletier/go-toml/v2 v2.4.3
github.com/magiconair/properties v1.8.10
github.com/pelletier/go-toml/v2 v2.3.0
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
github.com/spf13/cobra v1.10.2
github.com/spf13/pflag v1.0.10
github.com/yuin/gopher-lua v1.1.2
github.com/zclconf/go-cty v1.19.0
go.yaml.in/yaml/v4 v4.0.0-rc.6
golang.org/x/mod v0.38.0
golang.org/x/net v0.57.0
golang.org/x/text v0.40.0
github.com/yuin/gopher-lua v1.1.1
github.com/zclconf/go-cty v1.18.0
go.yaml.in/yaml/v4 v4.0.0-rc.3
golang.org/x/mod v0.34.0
golang.org/x/net v0.52.0
golang.org/x/text v0.35.0
)
require (
github.com/agext/levenshtein v1.2.1 // indirect
github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
github.com/apparentlymart/go-textseg/v17 v17.0.1 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
golang.org/x/sync v0.22.0 // indirect
golang.org/x/sys v0.47.0 // indirect
golang.org/x/tools v0.47.0 // indirect
golang.org/x/sync v0.20.0 // indirect
golang.org/x/sys v0.42.0 // indirect
golang.org/x/tools v0.42.0 // indirect
)
go 1.25.0
+24 -26
View File
@@ -6,12 +6,10 @@ github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8v
github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
github.com/alecthomas/participle/v2 v2.1.4 h1:W/H79S8Sat/krZ3el6sQMvMaahJ+XcM9WSI2naI7w2U=
github.com/alecthomas/participle/v2 v2.1.4/go.mod h1:8tqVbpTX20Ru4NfYQgZf4mP18eXPTBViyMWiArNEgGI=
github.com/alecthomas/repr v0.5.4 h1:OVP7JEcuzU9CCDsT6STCr3rg17oQfWILtPWd2EG0uN4=
github.com/alecthomas/repr v0.5.4/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=
github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
github.com/apparentlymart/go-textseg/v17 v17.0.1 h1:bpMXRgQ5cEoRNuQke1a80/Nl6w3G5eoIbWo9f3gXkAs=
github.com/apparentlymart/go-textseg/v17 v17.0.1/go.mod h1:fa8X4jgGeevslICIY6LcdjkSecWnXmYd9Lk34z/VxZs=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -40,16 +38,16 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
github.com/magiconair/properties v1.18.11 h1:j5ozYZl0zCjG7ahMDH0GWIobOvvUzT0BdAguG0ViKy0=
github.com/magiconair/properties v1.18.11/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pelletier/go-toml/v2 v2.3.0 h1:k59bC/lIZREW0/iVaQR8nDHxVq8OVlIzYCOJf421CaM=
github.com/pelletier/go-toml/v2 v2.3.0/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -63,28 +61,28 @@ github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3A
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/yuin/gopher-lua v1.1.2 h1:yF/FjE3hD65tBbt0VXLE13HWS9h34fdzJmrWRXwobGA=
github.com/yuin/gopher-lua v1.1.2/go.mod h1:7aRmXIWl37SqRf0koeyylBEzJ+aPt8A+mmkQ4f1ntR8=
github.com/zclconf/go-cty v1.19.0 h1:IV8WdqYZc2c5rLX9bEoLNXKojBAp0MZPBHMIrCoa/s4=
github.com/zclconf/go-cty v1.19.0/go.mod h1:12W89jGn3JCOIQi7infWr9m80rOkb5RNYJqXMZcN4c8=
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
github.com/zclconf/go-cty v1.18.0 h1:pJ8+HNI4gFoyRNqVE37wWbJWVw43BZczFo7KUoRczaA=
github.com/zclconf/go-cty v1.18.0/go.mod h1:qpnV6EDNgC1sns/AleL1fvatHw72j+S+nS+MJ+T2CSg=
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo=
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=
golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go=
go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-6
View File
@@ -1,5 +1,3 @@
//go:build goinstall
package main
import (
@@ -13,10 +11,6 @@ import (
// TestGoInstallCompatibility ensures the module can be zipped for go install.
// This is an integration test that uses the same zip.CreateFromDir function
// that go install uses internally. If this test fails, go install will fail.
//
// Built with the goinstall tag and run after the main test suite (see scripts/test.sh)
// so it does not race with pkg/yqlib tests that rewrite doc/*.md during execution.
//
// See: https://github.com/mikefarah/yq/issues/2587
func TestGoInstallCompatibility(t *testing.T) {
mod := module.Version{
+1 -2
View File
@@ -49,13 +49,12 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
var allDocuments = list.New()
for _, filename := range filenames {
reader, cleanup, err := readStream(filename)
reader, err := readStream(filename)
if err != nil {
return err
}
fileDocuments, err := readDocuments(reader, filename, fileIndex, decoder)
cleanup()
if err != nil {
return err
}
+7 -27
View File
@@ -27,30 +27,10 @@ const (
FlowStyle
)
// EncodeHint controls how a mapping node is serialised by format-specific encoders
// that distinguish between inline and block/section representations (e.g. TOML, HCL).
type EncodeHint int
const (
// EncodeHintDefault lets the encoder choose the representation (e.g. TOML block
// mappings default to [section] headers).
EncodeHintDefault EncodeHint = iota
// EncodeHintSeparateBlock forces the node to be emitted as a separate block or
// table-section header (used by TOML [section] and HCL block decoders).
EncodeHintSeparateBlock
// EncodeHintInline forces the node to be emitted as an inline / flow table
// (used by TOML inline-table decoder and TOML encoder).
EncodeHintInline
)
func createStringScalarNode(stringValue string) *CandidateNode {
var node = &CandidateNode{Kind: ScalarNode}
node.Value = stringValue
if stringValue == "<<" {
node.Tag = "!!merge"
} else {
node.Tag = "!!str"
}
node.Tag = "!!str"
return node
}
@@ -117,9 +97,9 @@ type CandidateNode struct {
// (e.g. top level cross document merge). This property does not propagate to child nodes.
EvaluateTogether bool
IsMapKey bool
// EncodeHint controls how a mapping node is serialised by format-specific encoders
// (e.g. TOML, HCL) that support both inline and block/section representations.
EncodeHint EncodeHint
// For formats like HCL and TOML: indicates that child entries should be emitted as separate blocks/tables
// rather than consolidated into nested mappings (default behaviour)
EncodeSeparate bool
}
func (n *CandidateNode) CreateChild() *CandidateNode {
@@ -431,7 +411,7 @@ func (n *CandidateNode) doCopy(cloneContent bool) *CandidateNode {
EvaluateTogether: n.EvaluateTogether,
IsMapKey: n.IsMapKey,
EncodeHint: n.EncodeHint,
EncodeSeparate: n.EncodeSeparate,
}
if cloneContent {
@@ -485,8 +465,8 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignP
n.Anchor = other.Anchor
}
// Preserve EncodeHint for format-specific encoding hints
n.EncodeHint = other.EncodeHint
// Preserve EncodeSeparate flag for format-specific encoding hints
n.EncodeSeparate = other.EncodeSeparate
// merge will pickup the style of the new thing
// when autocreating nodes
-91
View File
@@ -7,9 +7,6 @@ import (
"errors"
"fmt"
"io"
"math"
"strconv"
"strings"
"github.com/goccy/go-json"
)
@@ -143,12 +140,6 @@ func (o *CandidateNode) MarshalJSON() ([]byte, error) {
return buf.Bytes(), err
case ScalarNode:
log.Debugf("MarshalJSON ScalarNode")
if o.guessTagFromCustomType() == "!!float" {
if raw, ok := jsonFloatLiteral(o.Value); ok {
buf.WriteString(raw)
return buf.Bytes(), nil
}
}
value, err := o.GetValueRep()
if err != nil {
return buf.Bytes(), err
@@ -186,85 +177,3 @@ func (o *CandidateNode) MarshalJSON() ([]byte, error) {
return buf.Bytes(), err
}
}
// jsonFloatLiteral returns a JSON-shaped representation of a YAML !!float scalar
// value, preserving the original textual form (e.g. "50.0" stays "50.0") whenever
// possible. The second return value is false when the value cannot be safely
// rendered as a JSON number (e.g. ".inf", ".nan", or anything that parses to a
// non-finite float); callers should fall back to the normal encoding path in
// that case, which preserves the existing behaviour for those inputs.
func jsonFloatLiteral(raw string) (string, bool) {
if raw == "" {
return "", false
}
f, err := strconv.ParseFloat(raw, 64)
if err != nil {
return "", false
}
if math.IsInf(f, 0) || math.IsNaN(f) {
return "", false
}
if isJSONNumberLiteral(raw) {
return raw, true
}
formatted := strconv.FormatFloat(f, 'f', -1, 64)
if !strings.ContainsAny(formatted, ".eE") {
formatted += ".0"
}
return formatted, true
}
// isJSONNumberLiteral reports whether s is already a valid JSON number literal
// representing a fractional value (i.e. contains a "." or an exponent), so it
// can be emitted verbatim without round-tripping through a float64.
func isJSONNumberLiteral(s string) bool {
if s == "" {
return false
}
i := 0
if s[i] == '-' {
i++
if i == len(s) {
return false
}
}
// integer part: 0 or [1-9][0-9]*
if s[i] == '0' {
i++
} else if s[i] >= '1' && s[i] <= '9' {
for i < len(s) && s[i] >= '0' && s[i] <= '9' {
i++
}
} else {
return false
}
hasFraction := false
if i < len(s) && s[i] == '.' {
hasFraction = true
i++
if i == len(s) || s[i] < '0' || s[i] > '9' {
return false
}
for i < len(s) && s[i] >= '0' && s[i] <= '9' {
i++
}
}
hasExponent := false
if i < len(s) && (s[i] == 'e' || s[i] == 'E') {
hasExponent = true
i++
if i < len(s) && (s[i] == '+' || s[i] == '-') {
i++
}
if i == len(s) || s[i] < '0' || s[i] > '9' {
return false
}
for i < len(s) && s[i] >= '0' && s[i] <= '9' {
i++
}
}
if i != len(s) {
return false
}
return hasFraction || hasExponent
}
-25
View File
@@ -106,31 +106,6 @@ func TestCreateScalarNodeScenarios(t *testing.T) {
}
}
type createStringScalarNodeScenario struct {
stringValue string
expectedTag string
}
var createStringScalarNodeScenarios = []createStringScalarNodeScenario{
{
stringValue: "yourKey",
expectedTag: "!!str",
},
{
stringValue: "<<",
expectedTag: "!!merge",
},
}
func TestCreateStringScalarNodeScenarios(t *testing.T) {
for _, tt := range createStringScalarNodeScenarios {
actual := createStringScalarNode(tt.stringValue)
test.AssertResultWithContext(t, tt.stringValue, actual.Value, fmt.Sprintf("Value for: %v", tt.stringValue))
test.AssertResultWithContext(t, tt.expectedTag, actual.Tag, fmt.Sprintf("Tag for: %v", tt.stringValue))
test.AssertResultWithContext(t, ScalarNode, actual.Kind, fmt.Sprintf("Kind for: %v", tt.stringValue))
}
}
func TestGetKeyForMapValue(t *testing.T) {
key := createStringScalarNode("yourKey")
n := CandidateNode{Key: key, Value: "meow", document: 3}
+6 -4
View File
@@ -226,7 +226,7 @@ func addBlockToMapping(parent *CandidateNode, block *hclsyntax.Block, src []byte
// Mark the type node if there are multiple blocks of this type at this level
// This tells the encoder to emit them as separate blocks rather than consolidating them
if isMultipleBlocksOfType {
typeNode.EncodeHint = EncodeHintSeparateBlock
typeNode.EncodeSeparate = true
}
}
current = typeNode
@@ -300,7 +300,8 @@ func convertHclExprToNode(expr hclsyntax.Expression, src []byte) *CandidateNode
it := v.ElementIterator()
for it.Next() {
key, val := it.Element()
keyNode := convertCtyValueToNode(key)
keyStr := key.AsString()
keyNode := createStringScalarNode(keyStr)
valNode := convertCtyValueToNode(val)
m.AddKeyValueChild(keyNode, valNode)
}
@@ -337,7 +338,8 @@ func convertHclExprToNode(expr hclsyntax.Expression, src []byte) *CandidateNode
}
continue
}
keyNode := convertCtyValueToNode(keyVal)
keyStr := keyVal.AsString()
keyNode := createStringScalarNode(keyStr)
valNode := convertHclExprToNode(item.ValueExpr, src)
m.AddKeyValueChild(keyNode, valNode)
}
@@ -458,7 +460,7 @@ func convertCtyValueToNode(v cty.Value) *CandidateNode {
it := v.ElementIterator()
for it.Next() {
key, val := it.Element()
keyNode := convertCtyValueToNode(key)
keyNode := createStringScalarNode(key.AsString())
valNode := convertCtyValueToNode(val)
m.AddKeyValueChild(keyNode, valNode)
}
+2 -8
View File
@@ -12,20 +12,17 @@ import (
type iniDecoder struct {
reader io.Reader
finished bool // Flag to signal completion of processing
prefs INIPreferences
}
func NewINIDecoder(prefs INIPreferences) Decoder {
func NewINIDecoder() Decoder {
return &iniDecoder{
finished: false, // Initialise the flag as false
prefs: prefs,
}
}
func (dec *iniDecoder) Init(reader io.Reader) error {
// Store the reader for use in Decode
dec.reader = reader
dec.finished = false
return nil
}
@@ -42,10 +39,7 @@ func (dec *iniDecoder) Decode() (*CandidateNode, error) {
}
// Parse the INI content
loadOpts := ini.LoadOptions{
PreserveSurroundedQuote: dec.prefs.PreserveSurroundedQuote,
}
cfg, err := ini.LoadSources(loadOpts, content)
cfg, err := ini.Load(content)
if err != nil {
return nil, fmt.Errorf("failed to parse INI content: %w", err)
}
+11 -48
View File
@@ -16,11 +16,10 @@ type propertiesDecoder struct {
reader io.Reader
finished bool
d DataTreeNavigator
prefs PropertiesPreferences
}
func NewPropertiesDecoder() Decoder {
return &propertiesDecoder{d: NewDataTreeNavigator(), finished: false, prefs: ConfiguredPropertiesPreferences.Copy()}
return &propertiesDecoder{d: NewDataTreeNavigator(), finished: false}
}
func (dec *propertiesDecoder) Init(reader io.Reader) error {
@@ -29,56 +28,20 @@ func (dec *propertiesDecoder) Init(reader io.Reader) error {
return nil
}
func parsePropKey(key string, prefs PropertiesPreferences) []interface{} {
func parsePropKey(key string) []interface{} {
pathStrArray := strings.Split(key, ".")
path := make([]interface{}, 0, len(pathStrArray))
for _, pathStr := range pathStrArray {
path = appendPropKeySegment(path, pathStr, prefs.UseArrayBrackets)
path := make([]interface{}, len(pathStrArray))
for i, pathStr := range pathStrArray {
num, err := strconv.ParseInt(pathStr, 10, 32)
if err == nil {
path[i] = num
} else {
path[i] = pathStr
}
}
return path
}
func appendPropKeySegment(path []interface{}, segment string, useArrayBrackets bool) []interface{} {
if useArrayBrackets && strings.Contains(segment, "[") {
bracketPath, ok := parsePropKeyArrayBracketSegment(segment)
if ok {
return append(path, bracketPath...)
}
}
num, err := strconv.ParseInt(segment, 10, 32)
if err == nil {
return append(path, num)
}
return append(path, segment)
}
func parsePropKeyArrayBracketSegment(segment string) ([]interface{}, bool) {
path := []interface{}{}
bracketIndex := strings.Index(segment, "[")
if bracketIndex > 0 {
path = append(path, segment[:bracketIndex])
}
remaining := segment[bracketIndex:]
for remaining != "" {
if !strings.HasPrefix(remaining, "[") {
return nil, false
}
closingBracket := strings.Index(remaining, "]")
if closingBracket < 0 {
return nil, false
}
arrayIndex, err := strconv.ParseInt(remaining[1:closingBracket], 10, 32)
if err != nil {
return nil, false
}
path = append(path, arrayIndex)
remaining = remaining[closingBracket+1:]
}
return path, true
}
func (dec *propertiesDecoder) processComment(c string) string {
if c == "" {
return ""
@@ -112,7 +75,7 @@ func (dec *propertiesDecoder) applyPropertyComments(context Context, path []inte
func (dec *propertiesDecoder) applyProperty(context Context, properties *properties.Properties, key string) error {
value, _ := properties.Get(key)
path := parsePropKey(key, dec.prefs)
path := parsePropKey(key)
propertyComments := properties.GetComments(key)
if len(propertyComments) > 0 {
+10 -11
View File
@@ -150,10 +150,9 @@ func (dec *tomlDecoder) createInlineTableMap(tomlNode *toml.Node) (*CandidateNod
}
return &CandidateNode{
Kind: MappingNode,
Tag: "!!map",
EncodeHint: EncodeHintInline,
Content: content,
Kind: MappingNode,
Tag: "!!map",
Content: content,
}, nil
}
@@ -346,10 +345,10 @@ func (dec *tomlDecoder) processTable(currentNode *toml.Node) (bool, error) {
}
tableNodeValue := &CandidateNode{
Kind: MappingNode,
Tag: "!!map",
Content: make([]*CandidateNode, 0),
EncodeHint: EncodeHintSeparateBlock,
Kind: MappingNode,
Tag: "!!map",
Content: make([]*CandidateNode, 0),
EncodeSeparate: true,
}
// Attach pending head comments to the table
@@ -443,9 +442,9 @@ func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error)
hasValue := dec.parser.NextExpression()
tableNodeValue := &CandidateNode{
Kind: MappingNode,
Tag: "!!map",
EncodeHint: EncodeHintSeparateBlock,
Kind: MappingNode,
Tag: "!!map",
EncodeSeparate: true,
}
// Attach pending head comments to the array table
@@ -2,7 +2,7 @@
Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names).
`yq` supports merge keys (like `<<: *blah`) from YAML 1.1. These are no longer part of the YAML 1.2 standard, but remain common in practice. Plain `<<:` keys are recognised as merge keys and round-trip as `<<:` without an explicit `!!merge` tag. When the source uses an explicit `!!merge` tag, that is preserved on output. Internally, when `yq` synthesises a `<<` map key (for example during merge operations), it tags the key as `!!merge` rather than `!!str`.
`yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.
## NOTE --yaml-fix-merge-anchor-to-spec flag
@@ -32,7 +32,7 @@ Given a sample.yml file of:
r: 10
- &SMALL
r: 1
- <<: *CENTRE
- !!merge <<: *CENTRE
r: 10
```
then
@@ -213,10 +213,10 @@ item_value: &item_value
value: true
thingOne:
name: item_1
<<: *item_value
!!merge <<: *item_value
thingTwo:
name: item_2
<<: *item_value
!!merge <<: *item_value
```
then
```bash
@@ -231,7 +231,7 @@ thingOne:
value: false
thingTwo:
name: item_2
<<: *item_value
!!merge <<: *item_value
```
## LEGACY: Explode with merge anchors
@@ -249,13 +249,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@@ -298,7 +298,7 @@ Given a sample.yml file of:
r: 10
- &SMALL
r: 1
- <<:
- !!merge <<:
- *CENTRE
- *BIG
```
@@ -328,7 +328,7 @@ Given a sample.yml file of:
r: 10
- &SMALL
r: 1
- <<:
- !!merge <<:
- *BIG
- *LEFT
- *SMALL
@@ -361,13 +361,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@@ -411,7 +411,7 @@ Given a sample.yml file of:
r: 10
- &SMALL
r: 1
- <<:
- !!merge <<:
- *CENTRE
- *BIG
```
@@ -442,7 +442,7 @@ Given a sample.yml file of:
r: 10
- &SMALL
r: 1
- <<:
- !!merge <<:
- *BIG
- *LEFT
- *SMALL
@@ -467,7 +467,7 @@ Given a sample.yml file of:
```yaml
a:
b: &b 42
<<:
!!merge <<:
c: *b
```
then
+2 -2
View File
@@ -55,7 +55,7 @@ yq '.a = .a / 0 | .b = .b / 0' sample.yml
```
will output
```yaml
a: +Inf
b: -Inf
a: !!float +Inf
b: !!float -Inf
```
@@ -2,7 +2,7 @@
Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names).
`yq` supports merge keys (like `<<: *blah`) from YAML 1.1. These are no longer part of the YAML 1.2 standard, but remain common in practice. Plain `<<:` keys are recognised as merge keys and round-trip as `<<:` without an explicit `!!merge` tag. When the source uses an explicit `!!merge` tag, that is preserved on output. Internally, when `yq` synthesises a `<<` map key (for example during merge operations), it tags the key as `!!merge` rather than `!!str`.
`yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.
## NOTE --yaml-fix-merge-anchor-to-spec flag
@@ -1,27 +0,0 @@
# System Operators
The `system` operator allows you to run an external command and use its output as a value in your expression.
**Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it.
**Note:** When enabled, the system operator can replicate the functionality of `env` and `load`
operators via external commands. Enabling it effectively overrides `--security-disable-env-ops`
and `--security-disable-file-ops`.
## Usage
```bash
yq --security-enable-system-operator --null-input '.field = system("command"; "arg1")'
```
The operator takes:
- A command string (required)
- An argument (or an array of arguments), separated from the command by `;` (optional)
The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string.
## Disabling the system operator
The system operator is disabled by default. When disabled, an error is returned instead of running the command, consistent with `--security-disable-env-ops` and `--security-disable-file-ops`.
Use `--security-enable-system-operator` flag to enable it.
+2 -2
View File
@@ -34,7 +34,7 @@ yq '.a = .a % .b' sample.yml
```
will output
```yaml
a: 2
a: !!float 2
b: 2.5
```
@@ -69,7 +69,7 @@ yq '.a = .a % .b' sample.yml
```
will output
```yaml
a: NaN
a: !!float NaN
b: 0
```
+3 -3
View File
@@ -471,13 +471,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@@ -487,7 +487,7 @@ yq '.foobar * .foobarList' sample.yml
will output
```yaml
c: foobarList_c
<<:
!!merge <<:
- *foo
- *bar
thing: foobar_thing
@@ -131,13 +131,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@@ -147,7 +147,7 @@ yq '.foobar | [..]' sample.yml
will output
```yaml
- c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
- foobar_c
- *foo
@@ -1,76 +0,0 @@
# System Operators
The `system` operator allows you to run an external command and use its output as a value in your expression.
**Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it.
**Note:** When enabled, the system operator can replicate the functionality of `env` and `load`
operators via external commands. Enabling it effectively overrides `--security-disable-env-ops`
and `--security-disable-file-ops`.
## Usage
```bash
yq --security-enable-system-operator --null-input '.field = system("command"; "arg1")'
```
The operator takes:
- A command string (required)
- An argument (or an array of arguments), separated from the command by `;` (optional)
The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string.
## Disabling the system operator
The system operator is disabled by default. When disabled, an error is returned instead of running the command, consistent with `--security-disable-env-ops` and `--security-disable-file-ops`.
Use `--security-enable-system-operator` flag to enable it.
## system operator returns error when disabled
Use `--security-enable-system-operator` to enable the system operator.
Given a sample.yml file of:
```yaml
country: Australia
```
then
```bash
yq '.country = system("/usr/bin/echo"; "test")' sample.yml
```
will output
```bash
Error: system operations are disabled, use --security-enable-system-operator to enable
```
## Run a command with an argument
Use `--security-enable-system-operator` to enable the system operator.
Given a sample.yml file of:
```yaml
country: Australia
```
then
```bash
yq --security-enable-system-operator '.country = system("/usr/bin/echo"; "test")' sample.yml
```
will output
```yaml
country: test
```
## Run a command without arguments
Omit the semicolon and args to run the command with no extra arguments.
Given a sample.yml file of:
```yaml
a: hello
```
then
```bash
yq --security-enable-system-operator '.a = system("/usr/bin/echo")' sample.yml
```
will output
```yaml
a: ""
```
+20 -20
View File
@@ -294,13 +294,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@@ -325,13 +325,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@@ -376,13 +376,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@@ -409,13 +409,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@@ -442,13 +442,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@@ -477,13 +477,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@@ -513,13 +513,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@@ -546,13 +546,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@@ -579,13 +579,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@@ -614,13 +614,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
-16
View File
@@ -125,22 +125,6 @@ will output
{"whatever":"cat"}
```
## Encode json: preserve floats with trailing zero
Whole-number floats keep their decimal point so downstream consumers see a JSON number with a fractional part (matches Go's encoding/json, Python's json, and jq).
Given a sample.yml file of:
```yaml
percentiles: [50.0, 95.0, 99.0, 99.9]
```
then
```bash
yq -o=json -I=0 '.' sample.yml
```
will output
```json
{"percentiles":[50.0,95.0,99.0,99.9]}
```
## Roundtrip JSON Lines / NDJSON
Given a sample.json file of:
```json
+1 -1
View File
@@ -1,4 +1,4 @@
# TOML
Encode and decode to and from TOML.
Decode from TOML. Note that `yq` does not yet support outputting in TOML format (and therefore it cannot roundtrip)
+1 -18
View File
@@ -1,6 +1,6 @@
# TOML
Encode and decode to and from TOML.
Decode from TOML. Note that `yq` does not yet support outputting in TOML format (and therefore it cannot roundtrip)
## Parse: Simple
@@ -384,20 +384,3 @@ ip = "10.0.0.2"
role = "backend"
```
## Encode: Simple mapping produces table section
Given a sample.yml file of:
```yaml
arg:
hello: foo
```
then
```bash
yq -o toml '.' sample.yml
```
will output
```toml
[arg]
hello = "foo"
```
+4 -4
View File
@@ -449,8 +449,8 @@ func (he *hclEncoder) encodeBlockIfMapping(body *hclwrite.Body, key string, valu
return false
}
// If EncodeHintSeparateBlock is set, emit children as separate blocks regardless of label extraction
if valueNode.EncodeHint == EncodeHintSeparateBlock {
// If EncodeSeparate is set, emit children as separate blocks regardless of label extraction
if valueNode.EncodeSeparate {
if handled, _ := he.encodeMappingChildrenAsBlocks(body, key, valueNode); handled {
return true
}
@@ -537,9 +537,9 @@ func (he *hclEncoder) encodeMappingChildrenAsBlocks(body *hclwrite.Body, blockTy
return false, nil
}
// Only emit as separate blocks if EncodeHintSeparateBlock is set
// Only emit as separate blocks if EncodeSeparate is true
// This allows the encoder to respect the original block structure preserved by the decoder
if valueNode.EncodeHint != EncodeHintSeparateBlock {
if !valueNode.EncodeSeparate {
return false, nil
}
+99 -116
View File
@@ -69,27 +69,6 @@ func (te *tomlEncoder) CanHandleAliases() bool {
// ---- helpers ----
// tomlKey returns the key quoted if it contains characters that are not valid
// in a TOML bare key. TOML bare keys may only contain ASCII letters, ASCII
// digits, underscores, and dashes.
func tomlKey(key string) string {
for _, r := range key {
if (r < 'A' || r > 'Z') && (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '_' && r != '-' {
return fmt.Sprintf("%q", key)
}
}
return key
}
// tomlDottedKey joins path components, quoting any that require it.
func tomlDottedKey(path []string) string {
parts := make([]string, len(path))
for i, p := range path {
parts[i] = tomlKey(p)
}
return strings.Join(parts, ".")
}
func (te *tomlEncoder) writeComment(w io.Writer, comment string) error {
if comment == "" {
return nil
@@ -132,23 +111,12 @@ func (te *tomlEncoder) encodeRootMapping(w io.Writer, node *CandidateNode) error
}
}
// Preserve existing order by iterating Content
for i := 0; i < len(node.Content); i += 2 {
keyNode := node.Content[i]
valNode := node.Content[i+1]
if isTomlAttribute(valNode) {
if err := te.encodeTopLevelEntry(w, []string{keyNode.Value}, valNode); err != nil {
return err
}
}
}
for i := 0; i < len(node.Content); i += 2 {
keyNode := node.Content[i]
valNode := node.Content[i+1]
if !isTomlAttribute(valNode) {
if err := te.encodeTopLevelEntry(w, []string{keyNode.Value}, valNode); err != nil {
return err
}
if err := te.encodeTopLevelEntry(w, []string{keyNode.Value}, valNode); err != nil {
return err
}
}
return nil
@@ -180,15 +148,9 @@ func (te *tomlEncoder) encodeTopLevelEntry(w io.Writer, path []string, node *Can
}
if allMaps {
key := path[len(path)-1]
quotedKey := tomlKey(key)
if te.wroteRootAttr {
if _, err := w.Write([]byte("\n")); err != nil {
return err
}
te.wroteRootAttr = false
}
for _, it := range node.Content {
if _, err := w.Write([]byte("[[" + quotedKey + "]]\n")); err != nil {
// [[key]] then body
if _, err := w.Write([]byte("[[" + key + "]]\n")); err != nil {
return err
}
if err := te.encodeMappingBodyWithPath(w, []string{key}, it); err != nil {
@@ -200,12 +162,12 @@ func (te *tomlEncoder) encodeTopLevelEntry(w io.Writer, path []string, node *Can
// Regular array attribute
return te.writeArrayAttribute(w, path[len(path)-1], node)
case MappingNode:
// Use inline table syntax only for nodes explicitly marked as TOML inline tables.
// YAML flow-style mappings are not treated as inline tables; the FlowStyle attribute
// is a YAML-specific rendering hint and should not affect TOML output. This ensures
// that auto-detected JSON input (parsed as YAML flow style) produces readable table
// sections, consistent with explicitly parsed JSON input.
if node.EncodeHint == EncodeHintInline {
// Inline table if not EncodeSeparate, else emit separate tables/arrays of tables for children under this path
if !node.EncodeSeparate {
// If children contain mappings or arrays of mappings, prefer separate sections
if te.hasEncodeSeparateChild(node) || te.hasStructuralChildren(node) {
return te.encodeSeparateMapping(w, path, node)
}
return te.writeInlineTableAttribute(w, path[len(path)-1], node)
}
return te.encodeSeparateMapping(w, path, node)
@@ -214,30 +176,7 @@ func (te *tomlEncoder) encodeTopLevelEntry(w io.Writer, path []string, node *Can
}
}
func isTomlArrayOfTables(seq *CandidateNode) bool {
if len(seq.Content) == 0 {
return false
}
for _, it := range seq.Content {
if it.Kind != MappingNode || it.EncodeHint == EncodeHintInline {
return false
}
}
return true
}
func isTomlAttribute(node *CandidateNode) bool {
if node.Kind == ScalarNode {
return true
}
return node.Kind == SequenceNode && !isTomlArrayOfTables(node)
}
func (te *tomlEncoder) writeAttribute(w io.Writer, key string, value *CandidateNode) error {
if value.Tag == "!!null" {
return nil
}
te.wroteRootAttr = true // Mark that we wrote a root attribute
// Write head comment before the attribute
@@ -246,7 +185,7 @@ func (te *tomlEncoder) writeAttribute(w io.Writer, key string, value *CandidateN
}
// Write the attribute
line := tomlKey(key) + " = " + te.formatScalar(value)
line := key + " = " + te.formatScalar(value)
// Add line comment if present
if value.LineComment != "" {
@@ -271,7 +210,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida
// Handle empty arrays
if len(seq.Content) == 0 {
line := tomlKey(key) + " = []"
line := key + " = []"
if seq.LineComment != "" {
lineComment := strings.TrimSpace(seq.LineComment)
if !strings.HasPrefix(lineComment, "#") {
@@ -294,7 +233,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida
if hasElementComments {
// Write multiline array format with comments
if _, err := w.Write([]byte(tomlKey(key) + " = [\n")); err != nil {
if _, err := w.Write([]byte(key + " = [\n")); err != nil {
return err
}
@@ -385,7 +324,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida
}
}
line := tomlKey(key) + " = [" + strings.Join(items, ", ") + "]"
line := key + " = [" + strings.Join(items, ", ") + "]"
// Add line comment if present
if seq.LineComment != "" {
@@ -433,24 +372,21 @@ func (te *tomlEncoder) mappingToInlineTable(m *CandidateNode) (string, error) {
v := m.Content[i+1]
switch v.Kind {
case ScalarNode:
if v.Tag == "!!null" {
continue
}
parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), te.formatScalar(v)))
parts = append(parts, fmt.Sprintf("%s = %s", k, te.formatScalar(v)))
case SequenceNode:
// inline array in inline table
arr, err := te.sequenceToInlineArray(v)
if err != nil {
return "", err
}
parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), arr))
parts = append(parts, fmt.Sprintf("%s = %s", k, arr))
case MappingNode:
// nested inline table
inline, err := te.mappingToInlineTable(v)
if err != nil {
return "", err
}
parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), inline))
parts = append(parts, fmt.Sprintf("%s = %s", k, inline))
default:
return "", fmt.Errorf("unsupported inline table value kind: %v", v.Kind)
}
@@ -463,7 +399,7 @@ func (te *tomlEncoder) writeInlineTableAttribute(w io.Writer, key string, m *Can
if err != nil {
return err
}
_, err = w.Write([]byte(tomlKey(key) + " = " + inline + "\n"))
_, err = w.Write([]byte(key + " = " + inline + "\n"))
return err
}
@@ -485,7 +421,7 @@ func (te *tomlEncoder) writeTableHeader(w io.Writer, path []string, m *Candidate
}
// Write table header [a.b.c]
header := "[" + tomlDottedKey(path) + "]\n"
header := "[" + strings.Join(path, ".") + "]\n"
_, err := w.Write([]byte(header))
return err
}
@@ -493,21 +429,24 @@ func (te *tomlEncoder) writeTableHeader(w io.Writer, path []string, m *Candidate
// encodeSeparateMapping handles a mapping that should be encoded as table sections.
// It emits the table header for this mapping if it has any content, then processes children.
func (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *CandidateNode) error {
// Check if this mapping has any non-mapping, non-array-of-tables children (i.e., attributes).
// Inline mapping children also count as attributes since they render as key = { ... }.
// Check if this mapping has any non-mapping, non-array-of-tables children (i.e., attributes)
hasAttrs := false
for i := 0; i < len(m.Content); i += 2 {
v := m.Content[i+1]
if v.Kind == ScalarNode && v.Tag != "!!null" {
hasAttrs = true
break
}
if v.Kind == MappingNode && v.EncodeHint == EncodeHintInline {
if v.Kind == ScalarNode {
hasAttrs = true
break
}
if v.Kind == SequenceNode {
if !isTomlArrayOfTables(v) {
// Check if it's NOT an array of tables
allMaps := true
for _, it := range v.Content {
if it.Kind != MappingNode {
allMaps = false
break
}
}
if !allMaps {
hasAttrs = true
break
}
@@ -525,26 +464,31 @@ func (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *Cand
return nil
}
// No attributes, just nested table structures - process children recursively
// No attributes, just nested structures - process children
for i := 0; i < len(m.Content); i += 2 {
k := m.Content[i].Value
v := m.Content[i+1]
switch v.Kind {
case MappingNode:
// Emit [path.k]
newPath := append(append([]string{}, path...), k)
if err := te.encodeSeparateMapping(w, newPath, v); err != nil {
if err := te.writeTableHeader(w, newPath, v); err != nil {
return err
}
if err := te.encodeMappingBodyWithPath(w, newPath, v); err != nil {
return err
}
case SequenceNode:
// If sequence of maps, emit [[path.k]] per element
if isTomlArrayOfTables(v) {
key := tomlDottedKey(append(append([]string{}, path...), k))
if te.wroteRootAttr {
if _, err := w.Write([]byte("\n")); err != nil {
return err
}
te.wroteRootAttr = false
allMaps := true
for _, it := range v.Content {
if it.Kind != MappingNode {
allMaps = false
break
}
}
if allMaps {
key := strings.Join(append(append([]string{}, path...), k), ".")
for _, it := range v.Content {
if _, err := w.Write([]byte("[[" + key + "]]\n")); err != nil {
return err
@@ -569,9 +513,42 @@ func (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *Cand
return nil
}
func (te *tomlEncoder) hasEncodeSeparateChild(m *CandidateNode) bool {
for i := 0; i < len(m.Content); i += 2 {
v := m.Content[i+1]
if v.Kind == MappingNode && v.EncodeSeparate {
return true
}
}
return false
}
func (te *tomlEncoder) hasStructuralChildren(m *CandidateNode) bool {
for i := 0; i < len(m.Content); i += 2 {
v := m.Content[i+1]
// Only consider it structural if mapping has EncodeSeparate or is non-empty
if v.Kind == MappingNode && v.EncodeSeparate {
return true
}
if v.Kind == SequenceNode {
allMaps := true
for _, it := range v.Content {
if it.Kind != MappingNode {
allMaps = false
break
}
}
if allMaps {
return true
}
}
}
return false
}
// encodeMappingBodyWithPath encodes attributes and nested arrays of tables using full dotted path context
func (te *tomlEncoder) encodeMappingBodyWithPath(w io.Writer, path []string, m *CandidateNode) error {
// First, attributes (scalars, inline mappings, and non-map arrays)
// First, attributes (scalars and non-map arrays)
for i := 0; i < len(m.Content); i += 2 {
k := m.Content[i].Value
v := m.Content[i+1]
@@ -580,14 +557,15 @@ func (te *tomlEncoder) encodeMappingBodyWithPath(w io.Writer, path []string, m *
if err := te.writeAttribute(w, k, v); err != nil {
return err
}
case MappingNode:
if v.EncodeHint == EncodeHintInline {
if err := te.writeInlineTableAttribute(w, k, v); err != nil {
return err
case SequenceNode:
allMaps := true
for _, it := range v.Content {
if it.Kind != MappingNode {
allMaps = false
break
}
}
case SequenceNode:
if !isTomlArrayOfTables(v) {
if !allMaps {
if err := te.writeArrayAttribute(w, k, v); err != nil {
return err
}
@@ -600,8 +578,15 @@ func (te *tomlEncoder) encodeMappingBodyWithPath(w io.Writer, path []string, m *
k := m.Content[i].Value
v := m.Content[i+1]
if v.Kind == SequenceNode {
if isTomlArrayOfTables(v) {
dotted := tomlDottedKey(append(append([]string{}, path...), k))
allMaps := true
for _, it := range v.Content {
if it.Kind != MappingNode {
allMaps = false
break
}
}
if allMaps {
dotted := strings.Join(append(append([]string{}, path...), k), ".")
for _, it := range v.Content {
if _, err := w.Write([]byte("[[" + dotted + "]]\n")); err != nil {
return err
@@ -614,14 +599,12 @@ func (te *tomlEncoder) encodeMappingBodyWithPath(w io.Writer, path []string, m *
}
}
// Finally, child mappings: inline-hint ones were emitted above as attributes,
// while all others are emitted as separate sub-table sections.
// Finally, child mappings that are not marked EncodeSeparate get inlined as attributes
for i := 0; i < len(m.Content); i += 2 {
k := m.Content[i].Value
v := m.Content[i+1]
if v.Kind == MappingNode && v.EncodeHint != EncodeHintInline {
subPath := append(append([]string{}, path...), k)
if err := te.encodeSeparateMapping(w, subPath, v); err != nil {
if v.Kind == MappingNode && !v.EncodeSeparate {
if err := te.writeInlineTableAttribute(w, k, v); err != nil {
return err
}
}
@@ -677,7 +660,7 @@ func (te *tomlEncoder) colorizeToml(input []byte) []byte {
// Table sections - [section] or [[array]]
// Only treat '[' as a table section if it appears at the start of the line
// (possibly after whitespace). This avoids incorrectly colouring inline arrays like
// (possibly after whitespace). This avoids mis-colouring inline arrays like
// "ports = [8000, 8001]" as table sections.
if ch == '[' {
isSectionHeader := true
+1 -1
View File
@@ -40,7 +40,7 @@ func tryRemoveTempFile(filename string) {
// thanks https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang
func copyFileContents(src, dst string) (err error) {
// ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory,
// and ensuring that it's not possible to give a path to a file outside that directory.
// and ensuring that it's not possible to give a path to a file outside thar directory.
in, err := os.Open(src) // #nosec
if err != nil {
+3 -6
View File
@@ -90,7 +90,7 @@ var LuaFormat = &Format{"lua", []string{"l"},
var INIFormat = &Format{"ini", []string{"i"},
func() Encoder { return NewINIEncoder() },
func() Decoder { return NewINIDecoder(ConfiguredINIPreferences) },
func() Decoder { return NewINIDecoder() },
}
var Formats = []*Format{
@@ -128,11 +128,8 @@ func FormatStringFromFilename(filename string) string {
ext := filepath.Ext(filename)
if len(ext) >= 2 && ext[0] == '.' {
format := strings.ToLower(ext[1:])
if _, err := FormatFromString(format); err == nil {
GetLogger().Debugf("detected format '%s'", format)
return format
}
GetLogger().Debugf("extension '%s' is not a recognised format, defaulting to yaml", format)
GetLogger().Debugf("detected format '%s'", format)
return format
}
}
-3
View File
@@ -59,7 +59,4 @@ func TestFormatStringFromFilename(t *testing.T) {
test.AssertResult(t, "json", FormatStringFromFilename("TEST.JSON"))
test.AssertResult(t, "yaml", FormatStringFromFilename("test.json/foo"))
test.AssertResult(t, "yaml", FormatStringFromFilename(""))
// unrecognised extensions should default to yaml instead of being passed through verbatim
// (see https://github.com/mikefarah/yq/issues/1608)
test.AssertResult(t, "yaml", FormatStringFromFilename("test.tfstate"))
}
+2 -17
View File
@@ -2,26 +2,11 @@ package yqlib
import (
"bufio"
"bytes"
"errors"
"io"
"os"
)
var utf8BOM = []byte{0xEF, 0xBB, 0xBF}
// stripUTF8BOM returns a reader that skips a leading UTF-8 BOM, if present.
func stripUTF8BOM(r io.Reader) io.Reader {
br := bufio.NewReader(r)
peek, err := br.Peek(3)
if err == nil && bytes.Equal(peek, utf8BOM) {
_, _ = br.Discard(3)
}
return br
}
type frontMatterHandler interface {
Split() error
GetYamlFrontMatterFilename() string
@@ -58,13 +43,13 @@ func (f *frontMatterHandlerImpl) Split() error {
var reader *bufio.Reader
var err error
if f.originalFilename == "-" {
reader = bufio.NewReader(stripUTF8BOM(os.Stdin))
reader = bufio.NewReader(os.Stdin)
} else {
file, err := os.Open(f.originalFilename) // #nosec
if err != nil {
return err
}
reader = bufio.NewReader(stripUTF8BOM(file))
reader = bufio.NewReader(file)
}
f.contentReader = reader
+1 -88
View File
@@ -136,11 +136,10 @@ Some content
test.AssertResult(t, originalFilename, resolved)
// Read documents using the temp file, verify they get the original filename
reader, cleanup, err := readStream(tempFilename)
reader, err := readStream(tempFilename)
if err != nil {
panic(err)
}
defer cleanup()
decoder := NewYamlDecoder(ConfiguredYamlPreferences)
docs, err := readDocuments(reader, tempFilename, 0, decoder)
if err != nil {
@@ -158,92 +157,6 @@ Some content
fmHandler.CleanUp()
}
func TestFrontMatterSplitWithBOM(t *testing.T) {
// Regression test for https://github.com/mikefarah/yq/issues/2496
// A UTF-8 BOM before the opening --- must be skipped, otherwise the
// separator isn't recognised and the opening --- is lost.
file := createTestFile("\ufeff---\na: apple\nb: banana\n---\nnot a\nyaml: doc\n")
expectedYamlFm := `---
a: apple
b: banana
`
expectedContent := `---
not a
yaml: doc
`
fmHandler := NewFrontMatterHandler(file)
err := fmHandler.Split()
if err != nil {
panic(err)
}
yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename())
test.AssertResult(t, expectedYamlFm, yamlFm)
contentBytes, err := io.ReadAll(fmHandler.GetContentReader())
if err != nil {
panic(err)
}
test.AssertResult(t, expectedContent, string(contentBytes))
tryRemoveTempFile(file)
fmHandler.CleanUp()
}
func TestFrontMatterSplitWithBOMFromStdin(t *testing.T) {
// Regression test for https://github.com/mikefarah/yq/issues/2496
// A UTF-8 BOM must also be skipped when reading front matter from stdin.
originalStdin := os.Stdin
defer func() { os.Stdin = originalStdin }()
r, w, err := os.Pipe()
if err != nil {
panic(err)
}
os.Stdin = r
defer safelyCloseFile(r)
go func() {
_, writeErr := w.WriteString("\ufeff---\na: apple\nb: banana\n---\nnot a\nyaml: doc\n")
if writeErr != nil {
t.Errorf("failed to write front matter to the stdin pipe: %v", writeErr)
}
safelyCloseFile(w)
}()
expectedYamlFm := `---
a: apple
b: banana
`
expectedContent := `---
not a
yaml: doc
`
fmHandler := NewFrontMatterHandler("-")
err = fmHandler.Split()
if err != nil {
panic(err)
}
yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename())
test.AssertResult(t, expectedYamlFm, yamlFm)
contentBytes, err := io.ReadAll(fmHandler.GetContentReader())
if err != nil {
panic(err)
}
test.AssertResult(t, expectedContent, string(contentBytes))
fmHandler.CleanUp()
}
func TestFrontMatterSplitWithArray(t *testing.T) {
file := createTestFile(`[1,2,3]
---
+2 -1
View File
@@ -230,7 +230,8 @@ var goccyYamlFormatScenarios = []formatScenario{
description: "merge anchor",
skipDoc: true,
input: "a: &remember\n c: mike\nb:\n <<: *remember",
expected: "a: &remember\n c: mike\nb:\n <<: *remember\n",
// fine to have !!merge as that's what the current impl does
expected: "a: &remember\n c: mike\nb:\n !!merge <<: *remember\n",
},
{
description: "custom tag",
-42
View File
@@ -176,41 +176,6 @@ var hclFormatScenarios = []formatScenario{
expected: "obj: {a: 1, b: \"two\"}\n",
scenarioType: "decode",
},
{
description: "object with integer keys",
skipDoc: true,
input: `obj = { 1 = "one", 2 = "two" }`,
expected: "obj: {1: \"one\", 2: \"two\"}\n",
scenarioType: "decode",
},
{
description: "object with boolean keys",
skipDoc: true,
input: `obj = { (true) = "yes", (false) = "no" }`,
expected: "obj: {true: \"yes\", false: \"no\"}\n",
scenarioType: "decode",
},
{
description: "object with float keys",
skipDoc: true,
input: `obj = { (3.14) = "pi" }`,
expected: "obj: {3.14: \"pi\"}\n",
scenarioType: "decode",
},
{
description: "object with mixed scalar keys",
skipDoc: true,
input: `obj = { a = 1, 1 = "one", (true) = "yes" }`,
expected: "obj: {a: 1, 1: \"one\", true: \"yes\"}\n",
scenarioType: "decode",
},
{
description: "nested object with integer keys",
skipDoc: true,
input: `config = { levels = { 1 = "debug", 2 = "info" } }`,
expected: "config: {levels: {1: \"debug\", 2: \"info\"}}\n",
scenarioType: "decode",
},
{
description: "nested block",
skipDoc: true,
@@ -507,13 +472,6 @@ var hclFormatScenarios = []formatScenario{
expected: "service {\n optional_field = null\n}\n",
scenarioType: "roundtrip",
},
{
description: "object with integer key and empty value",
skipDoc: true,
input: `intdict = { 1 = {} }`,
expected: "intdict: {1: {}}\n",
scenarioType: "decode",
},
}
func testHclScenario(t *testing.T, s formatScenario) {
+3 -6
View File
@@ -1,21 +1,18 @@
package yqlib
type INIPreferences struct {
ColorsEnabled bool
PreserveSurroundedQuote bool
ColorsEnabled bool
}
func NewDefaultINIPreferences() INIPreferences {
return INIPreferences{
ColorsEnabled: false,
PreserveSurroundedQuote: false,
ColorsEnabled: false,
}
}
func (p *INIPreferences) Copy() INIPreferences {
return INIPreferences{
ColorsEnabled: p.ColorsEnabled,
PreserveSurroundedQuote: p.PreserveSurroundedQuote,
ColorsEnabled: p.ColorsEnabled,
}
}
+5 -63
View File
@@ -5,7 +5,6 @@ package yqlib
import (
"bufio"
"fmt"
"strings"
"testing"
"github.com/mikefarah/yq/v4/test"
@@ -23,16 +22,6 @@ const expectedSimpleINIYaml = `section:
key: value
`
const quotedINIInput = `[section]
color_theme = "Default"
theme_background = "False"
`
const expectedQuotedINIOutput = `[section]
color_theme = "Default"
theme_background = "False"
`
var iniScenarios = []formatScenario{
{
description: "Parse INI: simple",
@@ -60,22 +49,6 @@ var iniScenarios = []formatScenario{
},
}
// iniPreserveQuotesPrefs returns INIPreferences with PreserveSurroundedQuote enabled.
func iniPreserveQuotesPrefs() INIPreferences {
prefs := NewDefaultINIPreferences()
prefs.PreserveSurroundedQuote = true
return prefs
}
var iniPreserveQuotesScenarios = []formatScenario{
{
description: "Roundtrip INI: preserve quotes",
input: quotedINIInput,
expected: expectedQuotedINIOutput,
scenarioType: "roundtrip",
},
}
func documentRoundtripINIScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
@@ -97,7 +70,7 @@ func documentRoundtripINIScenario(w *bufio.Writer, s formatScenario) {
}
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```ini\n%v```\n\n", mustProcessFormatScenario(s, NewINIDecoder(NewDefaultINIPreferences()), NewINIEncoder())))
writeOrPanic(w, fmt.Sprintf("```ini\n%v```\n\n", mustProcessFormatScenario(s, NewINIDecoder(), NewINIEncoder())))
}
func documentDecodeINIScenario(w *bufio.Writer, s formatScenario) {
@@ -121,7 +94,7 @@ func documentDecodeINIScenario(w *bufio.Writer, s formatScenario) {
}
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewINIDecoder(NewDefaultINIPreferences()), NewYamlEncoder(ConfiguredYamlPreferences))))
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewINIDecoder(), NewYamlEncoder(ConfiguredYamlPreferences))))
}
func testINIScenario(t *testing.T, s formatScenario) {
@@ -129,11 +102,11 @@ func testINIScenario(t *testing.T, s formatScenario) {
case "encode":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewINIEncoder()), s.description)
case "decode":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewINIDecoder(NewDefaultINIPreferences()), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewINIDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
case "roundtrip":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewINIDecoder(NewDefaultINIPreferences()), NewINIEncoder()), s.description)
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewINIDecoder(), NewINIEncoder()), s.description)
case "decode-error":
result, err := processFormatScenario(s, NewINIDecoder(NewDefaultINIPreferences()), NewINIEncoder())
result, err := processFormatScenario(s, NewINIDecoder(), NewINIEncoder())
if err == nil {
t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result)
} else {
@@ -202,21 +175,6 @@ func documentDecodeErrorINIScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("```\n%v\n```\n\n", s.expectedError))
}
func TestINIDecoderInitResetsFinished(t *testing.T) {
decoder := NewINIDecoder(NewDefaultINIPreferences())
firstDocuments, err := readDocuments(strings.NewReader("[first]\nkey = value\n"), "first.ini", 0, decoder)
if err != nil {
t.Fatal(err)
}
test.AssertResult(t, 1, firstDocuments.Len())
secondDocuments, err := readDocuments(strings.NewReader("[second]\nkey = value\n"), "second.ini", 1, decoder)
if err != nil {
t.Fatal(err)
}
test.AssertResult(t, 1, secondDocuments.Len())
}
func TestINIScenarios(t *testing.T) {
for _, tt := range iniScenarios {
testINIScenario(t, tt)
@@ -227,19 +185,3 @@ func TestINIScenarios(t *testing.T) {
}
documentScenarios(t, "usage", "convert", genericScenarios, documentINIScenario)
}
func testINIPreserveQuotesScenario(t *testing.T, s formatScenario) {
prefs := iniPreserveQuotesPrefs()
switch s.scenarioType {
case "roundtrip":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewINIDecoder(prefs), NewINIEncoder()), s.description)
default:
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
}
}
func TestINIPreserveQuotesScenarios(t *testing.T) {
for _, tt := range iniPreserveQuotesScenarios {
testINIPreserveQuotesScenario(t, tt)
}
}
-80
View File
@@ -189,38 +189,6 @@ var jsonScenarios = []formatScenario{
expected: "{\n \"cat\": \"meow\"\n}\n",
scenarioType: "encode",
},
{
description: "Encode json: signed hex int",
skipDoc: true,
input: `+0x12`,
indent: 0,
expected: "18\n",
scenarioType: "encode",
},
{
description: "Encode json: negative hex int",
skipDoc: true,
input: `-0x12`,
indent: 0,
expected: "-18\n",
scenarioType: "encode",
},
{
description: "Encode json: signed octal int",
skipDoc: true,
input: `+0o22`,
indent: 0,
expected: "18\n",
scenarioType: "encode",
},
{
description: "Encode json: negative octal int",
skipDoc: true,
input: `-0o22`,
indent: 0,
expected: "-18\n",
scenarioType: "encode",
},
{
description: "Encode json: simple - in one line",
input: `cat: meow # this is a comment, and it will be dropped.`,
@@ -252,54 +220,6 @@ var jsonScenarios = []formatScenario{
expected: "{\"stuff\":\"cool\"}\n{\"whatever\":\"cat\"}\n",
scenarioType: "encode",
},
{
description: "Encode json: preserve floats with trailing zero",
subdescription: "Whole-number floats keep their decimal point so downstream consumers see a JSON number with a fractional part (matches Go's encoding/json, Python's json, and jq).",
input: `percentiles: [50.0, 95.0, 99.0, 99.9]`,
indent: 0,
expected: "{\"percentiles\":[50.0,95.0,99.0,99.9]}\n",
scenarioType: "encode",
},
{
description: "Encode json: ints stay ints",
skipDoc: true,
input: `a: 50`,
indent: 0,
expected: "{\"a\":50}\n",
scenarioType: "encode",
},
{
description: "Encode json: !!float tagged whole number gets .0",
skipDoc: true,
input: `a: !!float 5`,
indent: 0,
expected: "{\"a\":5.0}\n",
scenarioType: "encode",
},
{
description: "Encode json: scientific notation float preserved",
skipDoc: true,
input: `a: 1.5e-3`,
indent: 0,
expected: "{\"a\":1.5e-3}\n",
scenarioType: "encode",
},
{
description: "Encode json: negative float preserved",
skipDoc: true,
input: `a: -7.0`,
indent: 0,
expected: "{\"a\":-7.0}\n",
scenarioType: "encode",
},
{
description: "Encode json: mixed int and float array",
skipDoc: true,
input: `a: [1, 2.0, 3, 4.5]`,
indent: 0,
expected: "{\"a\":[1,2.0,3,4.5]}\n",
scenarioType: "encode",
},
{
description: "Roundtrip JSON Lines / NDJSON",
input: sampleNdJson,
-2
View File
@@ -96,8 +96,6 @@ var participleYqRules = []*participleYqRule{
simpleOp("load_?str|str_?load", loadStringOpType),
{"LoadYaml", `load`, loadOp(NewYamlDecoder(LoadYamlPreferences)), 0},
simpleOp("system", systemOpType),
{"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0},
simpleOp("select", selectOpType),
+8 -27
View File
@@ -8,24 +8,14 @@ import (
"math"
"strconv"
"strings"
"sync"
)
var ExpressionParser ExpressionParserInterface
var expressionParserOnce sync.Once
// InitExpressionParser initialises the package level ExpressionParser, and is
// safe to call concurrently. The evaluators call it on every Evaluate, so
// without the guard two goroutines could construct a parser at the same time,
// and newParticipleLexer populates the shared participleYqRules entries as it
// goes, which a third goroutine could be reading through getYqDefinition.
func InitExpressionParser() {
expressionParserOnce.Do(func() {
if ExpressionParser == nil {
ExpressionParser = newExpressionParser()
}
})
if ExpressionParser == nil {
ExpressionParser = newExpressionParser()
}
}
var log = newLogger()
@@ -171,21 +161,12 @@ func parseInt64(numberString string) (string, int64, error) {
numberString = strings.ReplaceAll(numberString, "_", "")
}
// A leading +/- sign would hide the 0x/0o prefix below, so peel it off and
// hand it back to ParseInt with the digits.
sign := ""
digits := numberString
if len(digits) > 0 && (digits[0] == '+' || digits[0] == '-') {
sign = digits[:1]
digits = digits[1:]
}
if strings.HasPrefix(digits, "0x") ||
strings.HasPrefix(digits, "0X") {
num, err := strconv.ParseInt(sign+digits[2:], 16, 64)
if strings.HasPrefix(numberString, "0x") ||
strings.HasPrefix(numberString, "0X") {
num, err := strconv.ParseInt(numberString[2:], 16, 64)
return "0x%X", num, err
} else if strings.HasPrefix(digits, "0o") {
num, err := strconv.ParseInt(sign+digits[2:], 8, 64)
} else if strings.HasPrefix(numberString, "0o") {
num, err := strconv.ParseInt(numberString[2:], 8, 64)
return "0o%o", num, err
}
num, err := strconv.ParseInt(numberString, 10, 64)
+1 -48
View File
@@ -3,7 +3,6 @@ package yqlib
import (
"fmt"
"strings"
"sync"
"testing"
"github.com/mikefarah/yq/v4/test"
@@ -25,7 +24,7 @@ type parseSnippetScenario struct {
var parseSnippetScenarios = []parseSnippetScenario{
{
snippet: ":",
expectedError: "go-yaml load error in parser (while parsing a block mapping) at L1.C1: did not find expected key",
expectedError: "yaml: did not find expected key",
},
{
snippet: "",
@@ -144,16 +143,6 @@ var parseInt64Scenarios = []parseInt64Scenario{
numberString: "0o10",
expectedParsedNumber: 8,
},
{
numberString: "+0x12",
expectedParsedNumber: 18,
expectedFormatString: "0x12",
},
{
numberString: "+0o22",
expectedParsedNumber: 18,
expectedFormatString: "0o22",
},
}
func TestParseInt64(t *testing.T) {
@@ -556,39 +545,3 @@ func TestProcessEscapeCharacters(t *testing.T) {
test.AssertResultComplexWithContext(t, tt.expected, actual, fmt.Sprintf("Input: %q", tt.input))
}
}
// TestInitExpressionParserConcurrent covers the data race described in #2788.
// The evaluators call InitExpressionParser on every Evaluate, so before it was
// guarded two goroutines could build a parser at the same time while a third
// read the shared participleYqRules entries that newParticipleLexer fills in.
//
// To reproduce the original race the process must not have initialised the
// parser yet, so run this test on its own with the detector enabled:
//
// go test -race -run TestInitExpressionParserConcurrent ./pkg/yqlib/
func TestInitExpressionParserConcurrent(t *testing.T) {
const goroutines = 32
var wg sync.WaitGroup
errs := make(chan error, goroutines)
for range goroutines {
wg.Add(1)
go func() {
defer wg.Done()
InitExpressionParser()
if _, err := ExpressionParser.ParseExpression(".a.b"); err != nil {
errs <- err
}
}()
}
wg.Wait()
close(errs)
for err := range errs {
t.Fatalf("concurrent ParseExpression failed: %v", err)
}
if ExpressionParser == nil {
t.Fatal("ExpressionParser should be initialised after concurrent InitExpressionParser calls")
}
}
+1 -1
View File
@@ -2,7 +2,7 @@
package yqlib
func NewINIDecoder(prefs INIPreferences) Decoder {
func NewINIDecoder() Decoder {
return nil
}
-2
View File
@@ -164,8 +164,6 @@ var stringInterpolationOpType = &operationType{Type: "STRING_INT", NumArgs: 0, P
var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 52, Handler: loadOperator, CheckForPostTraverse: true}
var loadStringOpType = &operationType{Type: "LOAD_STRING", NumArgs: 1, Precedence: 52, Handler: loadStringOperator}
var systemOpType = &operationType{Type: "SYSTEM", NumArgs: 1, Precedence: 50, Handler: systemOperator}
var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 52, Handler: keysOperator, CheckForPostTraverse: true}
var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator}
+7 -12
View File
@@ -170,10 +170,6 @@ func fixedReconstructAliasedMap(node *CandidateNode) error {
if mergeNodeSeq.Kind == AliasNode {
mergeNodeSeq = mergeNodeSeq.Alias
}
mergeNodeSeq = mergeNodeSeq.Copy()
if err := explodeNode(mergeNodeSeq, Context{}); err != nil {
return err
}
if mergeNodeSeq.Kind != MappingNode {
return fmt.Errorf("can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing %v", mergeNodeSeq.Tag)
}
@@ -183,9 +179,12 @@ func fixedReconstructAliasedMap(node *CandidateNode) error {
})
for _, item := range itemsToAdd {
copied := item.Copy()
copied.Parent = node
newContent = append(newContent, copied)
// copy to ensure exploding doesn't modify the original node
itemCopy := item.Copy()
if err := explodeNode(itemCopy, Context{}); err != nil {
return err
}
newContent = append(newContent, itemCopy)
}
}
}
@@ -228,12 +227,8 @@ func reconstructAliasedMap(node *CandidateNode, context Context) error {
}
}
node.Content = make([]*CandidateNode, 0)
entries := make([]*CandidateNode, 0, newContent.Len())
for newEl := newContent.Front(); newEl != nil; newEl = newEl.Next() {
entries = append(entries, newEl.Value.(*CandidateNode))
}
for i := 0; i < len(entries); i += 2 {
node.AddKeyValueChild(entries[i], entries[i+1])
node.AddChild(newEl.Value.(*CandidateNode))
}
return nil
}
+1 -89
View File
@@ -31,7 +31,7 @@ thingOne:
value: false
thingTwo:
name: item_2
<<: *item_value
!!merge <<: *item_value
`
var explodeMergeAnchorsFixedExpected = `D0, P[], (!!map)::foo:
@@ -198,15 +198,6 @@ var fixedAnchorOperatorScenarios = []expressionScenario{
"D0, P[], (!!map)::{a: 42}\n",
},
},
{
skipDoc: true,
description: "Nested merge anchor with inline map",
document: `{<<: {<<: {a: 42}}}`,
expression: `explode(.)`,
expected: []string{
"D0, P[], (!!map)::{a: 42}\n",
},
},
{
skipDoc: true,
description: "Merge anchor with sequence with inline map",
@@ -234,29 +225,6 @@ var fixedAnchorOperatorScenarios = []expressionScenario{
"D0, P[], (!!map)::a:\n b: 42\nb: 42\n",
},
},
{
skipDoc: true,
description: "Merge after explode preserves correct parent references",
document: `opensearch: &opensearch-cluster
ip2geo:
enabled: false
opensearch-client:
<<: *opensearch-cluster
nodeGroup: client
opensearchJavaOpts: "-Xmx1024m -Xms1024m"`,
document2: `opensearch: &opensearch-cluster
ip2geo:
enabled: true
opensearch-client:
<<: *opensearch-cluster
opensearchJavaOpts: "-Xmx1536m -Xms1536m"`,
expression: `(select(fi == 0) | explode(.)) * (select(fi == 1) | explode(.))`,
expected: []string{
"D0, P[], (!!map)::opensearch:\n ip2geo:\n enabled: true\nopensearch-client:\n ip2geo:\n enabled: true\n nodeGroup: client\n opensearchJavaOpts: \"-Xmx1536m -Xms1536m\"\n",
},
},
}
var badAnchorOperatorScenarios = []expressionScenario{
@@ -311,63 +279,7 @@ var badAnchorOperatorScenarios = []expressionScenario{
},
}
var mixedMergeTagStyleDocument = `
constants:
errorResponse: &errorResponse
status: 200
endpoints:
- condition: true
!!merge <<: *errorResponse
- condition: false
<<: *errorResponse
other:
!!merge <<: *errorResponse
somethingElse:
<<: *errorResponse
`
var mixedMergeTagStyleExplodedDocument = `
constants:
errorResponse:
status: 200
endpoints:
- condition: true
status: 200
- condition: false
status: 200
other:
status: 200
somethingElse:
status: 200
`
var anchorOperatorScenarios = []expressionScenario{
{
// mergeObjects previously skipped all !!merge-tagged nodes. Since !!merge only appears on
// << map keys, this meant applyAssignment was never called for the << key. It was later
// autocreated by createStringScalarNode("<<") with tag !!str, silently dropping !!merge.
// DontFollowAlias:true already prevents aliases being followed, so the skip was redundant.
// Old (buggy) output: "D0, P[], (!!map)::base: &base\n x: 1\ndest:\n <<: *base\n"
skipDoc: true,
description: "direct *+ preserves explicit !!merge tag on << key (regression for issue 2677)",
document: "base: &base\n x: 1\ndest:\n !!merge <<: *base\n",
expression: `. as $d | {} *+ $d`,
expected: []string{"D0, P[], (!!map)::base: &base\n x: 1\ndest:\n !!merge <<: *base\n"},
},
{
skipDoc: true,
description: "explicit !!merge tag on << key is preserved through ireduce merge",
document: mixedMergeTagStyleDocument,
expression: `. as $item ireduce ({}; . *+ $item)`,
expected: []string{"D0, P[], (!!map)::" + mixedMergeTagStyleDocument},
},
{
skipDoc: true,
description: "explode expands << merge keys regardless of explicit tag style (!!merge or plain)",
document: mixedMergeTagStyleDocument,
expression: `explode(.)`,
expected: []string{"D0, P[], (!!map)::" + mixedMergeTagStyleExplodedDocument},
},
{
skipDoc: true,
description: "merge anchor to alias alias",
-25
View File
@@ -69,30 +69,6 @@ func deleteFromMap(node *CandidateNode, childPath interface{}) {
}
}
node.Content = newContents
normaliseEmptyCollectionMapKeyComment(node)
}
func normaliseEmptyCollectionMapKeyComment(node *CandidateNode) {
if (node.Kind != SequenceNode && node.Kind != MappingNode) || len(node.Content) != 0 || node.LineComment != "" {
return
}
key := node.Key
if node.Parent != nil && node.Parent.Kind == MappingNode {
for index := 0; index < len(node.Parent.Content)-1; index += 2 {
if node.Parent.Content[index+1] == node {
key = node.Parent.Content[index]
break
}
}
}
if key == nil || key.LineComment == "" {
return
}
node.LineComment = key.LineComment
key.LineComment = ""
node.Style = FlowStyle
}
func deleteFromArray(node *CandidateNode, childPath interface{}) {
@@ -111,5 +87,4 @@ func deleteFromArray(node *CandidateNode, childPath interface{}) {
}
}
node.Content = newContents
normaliseEmptyCollectionMapKeyComment(node)
}
-24
View File
@@ -119,30 +119,6 @@ var deleteOperatorScenarios = []expressionScenario{
"D0, P[], (!!map)::a: []\n",
},
},
{
skipDoc: true,
description: "Delete all entries from list with inline key comment",
document: `testList: # A comment
- name: test1
value: 123
- name: test2
value: 456`,
expression: `del(.testList[])`,
expected: []string{
"D0, P[], (!!map)::testList: [] # A comment\n",
},
},
{
skipDoc: true,
description: "Delete all entries from map with inline key comment",
document: `testMap: # A comment
name: test1
value: 123`,
expression: `del(.testMap[])`,
expected: []string{
"D0, P[], (!!map)::testMap: {} # A comment\n",
},
},
{
skipDoc: true,
description: "Delete entry appended to an array",
+1 -1
View File
@@ -45,7 +45,7 @@ var divideOperatorScenarios = []expressionScenario{
document: `{a: 1, b: -1}`,
expression: `.a = .a / 0 | .b = .b / 0`,
expected: []string{
"D0, P[], (!!map)::{a: +Inf, b: -Inf}\n",
"D0, P[], (!!map)::{a: !!float +Inf, b: !!float -Inf}\n",
},
},
{
+1 -1
View File
@@ -44,7 +44,7 @@ func hasOperator(d *dataTreeNavigator, context Context, expressionNode *Expressi
if errParsingInt != nil {
return Context{}, errParsingInt
}
candidateHasKey = number >= 0 && int64(len(contents)) > number
candidateHasKey = int64(len(contents)) > number
}
results.PushBack(createBooleanCandidate(candidate, candidateHasKey))
default:
-10
View File
@@ -71,16 +71,6 @@ var hasOperatorScenarios = []expressionScenario{
"D0, P[4], (!!bool)::true\n",
},
},
{
skipDoc: true,
description: "Negative array index is never present",
document: "[[1, 2, 3], []]",
expression: `.[] | has(-1)`,
expected: []string{
"D0, P[0], (!!bool)::false\n",
"D0, P[1], (!!bool)::false\n",
},
},
}
func TestHasOperatorScenarios(t *testing.T) {
+1 -1
View File
@@ -77,7 +77,7 @@ var loadScenarios = []expressionScenario{
document: `{something: {file: "thing.yml"}, over: {here: [{file: "thing.yml"}]}}`,
expression: `(.. | select(has("file"))) |= load("../../examples/" + .file)`,
expected: []string{
"D0, P[], (!!map)::{something: {a: apple is included, b: cool.}, over: {here: [{a: apple is included,\n b: cool.}]}}\n",
"D0, P[], (!!map)::{something: {a: apple is included, b: cool.}, over: {here: [{a: apple is included, b: cool.}]}}\n",
},
},
{
+3 -3
View File
@@ -37,7 +37,7 @@ var moduloOperatorScenarios = []expressionScenario{
document: `{a: 12, b: 2.5}`,
expression: `.a = .a % .b`,
expected: []string{
"D0, P[], (!!map)::{a: 2, b: 2.5}\n",
"D0, P[], (!!map)::{a: !!float 2, b: 2.5}\n",
},
},
{
@@ -53,7 +53,7 @@ var moduloOperatorScenarios = []expressionScenario{
document: `{a: 1.1, b: 0}`,
expression: `.a = .a % .b`,
expected: []string{
"D0, P[], (!!map)::{a: NaN, b: 0}\n",
"D0, P[], (!!map)::{a: !!float NaN, b: 0}\n",
},
},
{
@@ -70,7 +70,7 @@ var moduloOperatorScenarios = []expressionScenario{
document: "a: 2\nb: !goat 2.3",
expression: `.a = .a % .b`,
expected: []string{
"D0, P[], (!!map)::a: 2\nb: !goat 2.3\n",
"D0, P[], (!!map)::a: !!float 2\nb: !goat 2.3\n",
},
},
{
+4
View File
@@ -189,6 +189,10 @@ func mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs
log.Debugf("going to applied assignment to LHS: %v with RHS: %v", NodeToString(lhs), NodeToString(candidate))
if candidate.Tag == "!!merge" {
continue
}
err := applyAssignment(d, context, pathIndexToStartFrom, lhs, candidate, preferences)
if err != nil {
return nil, err
+8 -11
View File
@@ -2,7 +2,6 @@ package yqlib
import (
"fmt"
"math/bits"
"strings"
"testing"
)
@@ -121,7 +120,7 @@ var multiplyOperatorScenarios = []expressionScenario{
document: mergeArrayWithAnchors,
expression: `. * .`,
expected: []string{
"D0, P[], (!!map)::sample:\n - &a\n - <<: *a\n",
"D0, P[], (!!map)::sample:\n - &a\n - !!merge <<: *a\n",
},
},
{
@@ -514,7 +513,7 @@ var multiplyOperatorScenarios = []expressionScenario{
environmentVariables: map[string]string{"originalPath": ".myArray", "otherPath": ".newArray", "idPath": ".a"},
expression: mergeExpression,
expected: []string{
"D0, P[], (!!map)::{myArray: [{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC},\n {a: dingo, c: dingoC}], something: else}\n",
"D0, P[], (!!map)::{myArray: [{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC}, {a: dingo, c: dingoC}], something: else}\n",
},
},
{
@@ -546,7 +545,7 @@ var multiplyOperatorScenarios = []expressionScenario{
document: mergeDocSample,
expression: `.foobar * .foobarList`,
expected: []string{
"D0, P[foobar], (!!map)::c: foobarList_c\n<<: [*foo, *bar]\nthing: foobar_thing\nb: foobarList_b\n",
"D0, P[foobar], (!!map)::c: foobarList_c\n!!merge <<: [*foo, *bar]\nthing: foobar_thing\nb: foobarList_b\n",
},
},
{
@@ -581,7 +580,7 @@ var multiplyOperatorScenarios = []expressionScenario{
document: "a: 2\nb: !goat 3.5",
expression: ".a = .a * .b",
expected: []string{
"D0, P[], (!!map)::a: 7\nb: !goat 3.5\n",
"D0, P[], (!!map)::a: !!float 7\nb: !goat 3.5\n",
},
},
{
@@ -708,13 +707,11 @@ var multiplyOperatorScenarios = []expressionScenario{
expectedError: "result of repeating string (1 bytes) by 99999999 would exceed 10485760 bytes",
},
{
// Pick a count whose product with len("ab") overflows int on
// any architecture: 2^30 on 32-bit, 2^62 on 64-bit. Doubling
// either yields MaxInt+1, which wraps to MinInt and bypasses
// a naive len*count guard.
// The size guard must not overflow: len * count can wrap to
// a negative or small value on 64-bit, bypassing the check.
skipDoc: true,
expression: fmt.Sprintf(`"ab" * %d`, 1<<(bits.UintSize-2)),
expectedError: fmt.Sprintf("result of repeating string (2 bytes) by %d would exceed 10485760 bytes", 1<<(bits.UintSize-2)),
expression: `"ab" * 4611686018427387904`,
expectedError: "result of repeating string (2 bytes) by 4611686018427387904 would exceed 10485760 bytes",
},
}
+4 -4
View File
@@ -187,7 +187,7 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
document: mergeDocSample,
expression: `.foobar | [..]`,
expected: []string{
"D0, P[foobar], (!!seq)::- c: foobar_c\n <<: *foo\n thing: foobar_thing\n- foobar_c\n- *foo\n- foobar_thing\n",
"D0, P[foobar], (!!seq)::- c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n- foobar_c\n- *foo\n- foobar_thing\n",
},
},
{
@@ -195,7 +195,7 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
document: mergeDocSample,
expression: `.foobar | [...]`,
expected: []string{
"D0, P[foobar], (!!seq)::- c: foobar_c\n <<: *foo\n thing: foobar_thing\n- c\n- foobar_c\n- <<\n- *foo\n- thing\n- foobar_thing\n",
"D0, P[foobar], (!!seq)::- c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n- c\n- foobar_c\n- !!merge <<\n- *foo\n- thing\n- foobar_thing\n",
},
},
{
@@ -203,7 +203,7 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
document: mergeDocSample,
expression: `.foobarList | ..`,
expected: []string{
"D0, P[foobarList], (!!map)::b: foobarList_b\n<<: [*foo, *bar]\nc: foobarList_c\n",
"D0, P[foobarList], (!!map)::b: foobarList_b\n!!merge <<: [*foo, *bar]\nc: foobarList_c\n",
"D0, P[foobarList b], (!!str)::foobarList_b\n",
"D0, P[foobarList <<], (!!seq)::[*foo, *bar]\n",
"D0, P[foobarList << 0], (alias)::*foo\n",
@@ -216,7 +216,7 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
document: mergeDocSample,
expression: `.foobarList | ...`,
expected: []string{
"D0, P[foobarList], (!!map)::b: foobarList_b\n<<: [*foo, *bar]\nc: foobarList_c\n",
"D0, P[foobarList], (!!map)::b: foobarList_b\n!!merge <<: [*foo, *bar]\nc: foobarList_c\n",
"D0, P[foobarList b], (!!str)::b\n",
"D0, P[foobarList b], (!!str)::foobarList_b\n",
"D0, P[foobarList <<], (!!merge)::<<\n",
+1 -1
View File
@@ -98,7 +98,7 @@ var selectOperatorScenarios = []expressionScenario{
document: `[{animal: cat, legs: {cool: true}}, {animal: fish}]`,
expression: `(.[] | select(.legs.cool == true).canWalk) = true | (.[] | .alive.things) = "yes"`,
expected: []string{
"D0, P[], (!!seq)::[{animal: cat, legs: {cool: true}, canWalk: true, alive: {things: yes}}, {animal: fish,\n alive: {things: yes}}]\n",
"D0, P[], (!!seq)::[{animal: cat, legs: {cool: true}, canWalk: true, alive: {things: yes}}, {animal: fish, alive: {things: yes}}]\n",
},
},
{
+1 -6
View File
@@ -166,12 +166,7 @@ func (a sortableNodeArray) compare(lhs *CandidateNode, rhs *CandidateNode, dateT
if err != nil {
panic(err)
}
if lhsNum < rhsNum {
return -1
} else if lhsNum > rhsNum {
return 1
}
return 0
return int(lhsNum - rhsNum)
} else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") {
lhsNum, err := strconv.ParseFloat(lhs.Value, 64)
if err != nil {
+1 -1
View File
@@ -37,7 +37,7 @@ var sortKeysOperatorScenarios = []expressionScenario{
document: `{bParent: {c: dog, array: [3,1,2]}, aParent: {z: donkey, x: [{c: yum, b: delish}, {b: ew, a: apple}]}}`,
expression: `sort_keys(..)`,
expected: []string{
"D0, P[], (!!map)::{aParent: {x: [{b: delish, c: yum}, {a: apple, b: ew}], z: donkey}, bParent: {array: [\n 3, 1, 2], c: dog}}\n",
"D0, P[], (!!map)::{aParent: {x: [{b: delish, c: yum}, {a: apple, b: ew}], z: donkey}, bParent: {array: [3, 1, 2], c: dog}}\n",
},
},
}
-9
View File
@@ -171,15 +171,6 @@ var sortByOperatorScenarios = []expressionScenario{
"D0, P[], (!!seq)::# abc\n- def\n# ghi\n",
},
},
{
skipDoc: true,
description: "Sort large integers (no int64 subtraction overflow)",
document: "[5000000000000000000, -5000000000000000000]",
expression: `sort`,
expected: []string{
"D0, P[], (!!seq)::[-5000000000000000000, 5000000000000000000]\n",
},
},
}
func TestSortByOperatorScenarios(t *testing.T) {
+1 -1
View File
@@ -50,7 +50,7 @@ var styleOperatorScenarios = []expressionScenario{
document: "bing: &foo {x: z}\na:\n c: cat\n <<: [*foo]",
expression: `(... | select(tag=="!!str")) style="single"`,
expected: []string{
"D0, P[], (!!map)::'bing': &foo {'x': 'z'}\n'a':\n 'c': 'cat'\n <<: [*foo]\n",
"D0, P[], (!!map)::'bing': &foo {'x': 'z'}\n'a':\n 'c': 'cat'\n !!merge <<: [*foo]\n",
},
},
{
-146
View File
@@ -1,146 +0,0 @@
package yqlib
import (
"bytes"
"container/list"
"fmt"
"os/exec"
"strings"
)
func resolveSystemArgs(argsNode *CandidateNode) ([]string, error) {
if argsNode == nil {
return nil, nil
}
if argsNode.Kind == SequenceNode {
args := make([]string, 0, len(argsNode.Content))
for _, child := range argsNode.Content {
// Only non-null scalar children are valid arguments.
if child == nil {
continue
}
if child.Kind != ScalarNode || child.Tag == "!!null" {
return nil, fmt.Errorf("system operator: argument must be a non-null scalar; got kind=%v tag=%v", child.Kind, child.Tag)
}
args = append(args, child.Value)
}
if len(args) == 0 {
return nil, nil
}
return args, nil
}
// Single-argument case: only accept a non-null scalar node.
if argsNode.Tag == "!!null" {
return nil, nil
}
if argsNode.Kind != ScalarNode {
return nil, fmt.Errorf("system operator: args must be a non-null scalar or sequence of non-null scalars; got kind=%v tag=%v", argsNode.Kind, argsNode.Tag)
}
return []string{argsNode.Value}, nil
}
func resolveCommandNode(commandNodes Context) (string, error) {
if commandNodes.MatchingNodes.Front() == nil {
return "", fmt.Errorf("system operator: command expression returned no results")
}
if commandNodes.MatchingNodes.Len() > 1 {
log.Debugf("system operator: command expression returned %d results, using first", commandNodes.MatchingNodes.Len())
}
cmdNode := commandNodes.MatchingNodes.Front().Value.(*CandidateNode)
if cmdNode.Kind != ScalarNode || cmdNode.guessTagFromCustomType() != "!!str" {
return "", fmt.Errorf("system operator: command must be a string scalar")
}
if cmdNode.Value == "" {
return "", fmt.Errorf("system operator: command must be a non-empty string")
}
return cmdNode.Value, nil
}
func systemOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
if !ConfiguredSecurityPreferences.EnableSystemOps {
return Context{}, fmt.Errorf("system operations are disabled, use --security-enable-system-operator to enable")
}
// determine at parse time whether we have (command; args) or just (command)
hasArgs := expressionNode.RHS.Operation.OperationType == blockOpType
var results = list.New()
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
nodeContext := context.SingleReadonlyChildContext(candidate)
var command string
var args []string
if hasArgs {
block := expressionNode.RHS
commandNodes, err := d.GetMatchingNodes(nodeContext, block.LHS)
if err != nil {
return Context{}, err
}
command, err = resolveCommandNode(commandNodes)
if err != nil {
return Context{}, err
}
argsNodes, err := d.GetMatchingNodes(nodeContext, block.RHS)
if err != nil {
return Context{}, err
}
if argsNodes.MatchingNodes.Len() > 1 {
log.Debugf("system operator: args expression returned %d results, using first", argsNodes.MatchingNodes.Len())
}
if argsNodes.MatchingNodes.Front() != nil {
args, err = resolveSystemArgs(argsNodes.MatchingNodes.Front().Value.(*CandidateNode))
if err != nil {
return Context{}, err
}
}
} else {
commandNodes, err := d.GetMatchingNodes(nodeContext, expressionNode.RHS)
if err != nil {
return Context{}, err
}
command, err = resolveCommandNode(commandNodes)
if err != nil {
return Context{}, err
}
}
var stdin bytes.Buffer
encoded, err := encodeToYamlString(candidate)
if err != nil {
return Context{}, err
}
stdin.WriteString(encoded)
// #nosec G204 - intentional: user must explicitly enable this operator
cmd := exec.Command(command, args...)
cmd.Stdin = &stdin
var stderr bytes.Buffer
cmd.Stderr = &stderr
output, err := cmd.Output()
if err != nil {
stderrStr := strings.TrimSpace(stderr.String())
if stderrStr != "" {
return Context{}, fmt.Errorf("system command '%v' failed: %w\nstderr: %v", command, err, stderrStr)
}
return Context{}, fmt.Errorf("system command '%v' failed: %w", command, err)
}
result := string(output)
if strings.HasSuffix(result, "\r\n") {
result = result[:len(result)-2]
} else if strings.HasSuffix(result, "\n") {
result = result[:len(result)-1]
}
newNode := candidate.CreateReplacement(ScalarNode, "!!str", result)
results.PushBack(newNode)
}
return context.ChildContext(results), nil
}
-123
View File
@@ -1,123 +0,0 @@
package yqlib
import (
"os/exec"
"testing"
)
func findExec(t *testing.T, name string) string {
t.Helper()
path, err := exec.LookPath(name)
if err != nil {
t.Skipf("skipping: %v not found: %v", name, err)
}
return path
}
var systemOperatorDisabledScenarios = []expressionScenario{
{
description: "system operator returns error when disabled",
subdescription: "Use `--security-enable-system-operator` to enable the system operator.",
document: "country: Australia",
expression: `.country = system("/usr/bin/echo"; "test")`,
expectedError: "system operations are disabled, use --security-enable-system-operator to enable",
},
}
func TestSystemOperatorDisabledScenarios(t *testing.T) {
originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps
defer func() {
ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps
}()
ConfiguredSecurityPreferences.EnableSystemOps = false
for _, tt := range systemOperatorDisabledScenarios {
testScenario(t, &tt)
}
documentOperatorScenarios(t, "system-operators", systemOperatorDisabledScenarios)
}
func TestSystemOperatorEnabledScenarios(t *testing.T) {
echoPath := findExec(t, "echo")
falsePath := findExec(t, "false")
originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps
defer func() {
ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps
}()
ConfiguredSecurityPreferences.EnableSystemOps = true
scenarios := []expressionScenario{
{
description: "Run a command with an argument",
subdescription: "Use `--security-enable-system-operator` to enable the system operator.",
yqFlags: "--security-enable-system-operator",
document: "country: Australia",
expression: `.country = system("` + echoPath + `"; "test")`,
expected: []string{
"D0, P[], (!!map)::country: test\n",
},
},
{
description: "Run a command without arguments",
subdescription: "Omit the semicolon and args to run the command with no extra arguments.",
yqFlags: "--security-enable-system-operator",
document: "a: hello",
expression: `.a = system("` + echoPath + `")`,
expected: []string{
"D0, P[], (!!map)::a: \"\"\n",
},
},
{
description: "Run a command with multiple arguments",
subdescription: "Pass an array of arguments.",
skipDoc: true,
document: "a: hello",
expression: `.a = system("` + echoPath + `"; ["foo", "bar"])`,
expected: []string{
"D0, P[], (!!map)::a: foo bar\n",
},
},
{
description: "Command and args are evaluated per matched node",
skipDoc: true,
document: "cmd: " + echoPath + "\narg: hello",
expression: `.result = system(.cmd; .arg)`,
expected: []string{
"D0, P[], (!!map)::cmd: " + echoPath + "\narg: hello\nresult: hello\n",
},
},
{
description: "Command failure returns error",
skipDoc: true,
document: "a: hello",
expression: `.a = system("` + falsePath + `")`,
expectedError: "system command '" + falsePath + "' failed: exit status 1",
},
{
description: "Null command returns error",
skipDoc: true,
document: "a: hello",
expression: `.a = system(null)`,
expectedError: "system operator: command must be a string scalar",
},
{
description: "System operator processes multiple matched nodes",
skipDoc: true,
document: "a: first",
document2: "a: second",
expression: `.a = system("` + echoPath + `"; "replaced")`,
expected: []string{
"D0, P[], (!!map)::a: replaced\n",
"D0, P[], (!!map)::a: replaced\n",
},
},
}
for _, tt := range scenarios {
testScenario(t, &tt)
}
appendOperatorDocumentScenario(t, "system-operators", scenarios)
}
+2 -2
View File
@@ -494,7 +494,7 @@ var traversePathOperatorScenarios = []expressionScenario{
document: mergeDocSample,
expression: `.foobar`,
expected: []string{
"D0, P[foobar], (!!map)::c: foobar_c\n<<: *foo\nthing: foobar_thing\n",
"D0, P[foobar], (!!map)::c: foobar_c\n!!merge <<: *foo\nthing: foobar_thing\n",
},
},
{
@@ -518,7 +518,7 @@ var traversePathOperatorScenarios = []expressionScenario{
document: mergeDocSample,
expression: `.foobarList`,
expected: []string{
"D0, P[foobarList], (!!map)::b: foobarList_b\n<<: [*foo, *bar]\nc: foobarList_c\n",
"D0, P[foobarList], (!!map)::b: foobarList_b\n!!merge <<: [*foo, *bar]\nc: foobarList_c\n",
},
},
{
+3 -12
View File
@@ -31,7 +31,6 @@ type expressionScenario struct {
dontFormatInputForDoc bool // dont format input doc for documentation generation
requiresFormat string
skipForGoccy bool
yqFlags string // extra yq flags to include in generated doc command snippets
}
var goccyTesting = false
@@ -357,22 +356,14 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
writeOrPanic(w, "then\n")
flagsPrefix := ""
if s.yqFlags != "" {
flagsPrefix = s.yqFlags + " "
}
if s.expression != "" {
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v'%v' %v\n```\n", envCommand, flagsPrefix, command, strings.ReplaceAll(s.expression, "'", `'\''`), files))
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v'%v' %v\n```\n", envCommand, command, strings.ReplaceAll(s.expression, "'", `'\''`), files))
} else {
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v%v\n```\n", envCommand, flagsPrefix, command, files))
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v\n```\n", envCommand, command, files))
}
} else {
writeOrPanic(w, "Running\n")
flagsPrefix := ""
if s.yqFlags != "" {
flagsPrefix = s.yqFlags + " "
}
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v--null-input '%v'\n```\n", envCommand, flagsPrefix, command, s.expression))
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v--null-input '%v'\n```\n", envCommand, command, s.expression))
}
return formattedDoc, formattedDoc2
}
-37
View File
@@ -202,37 +202,6 @@ var propertyScenarios = []formatScenario{
expected: expectedDecodedYaml,
scenarioType: "decode",
},
{
skipDoc: true,
description: "Decode properties with array brackets",
input: `user.credentials[0].username=user1
user.credentials[0].password=$2b$08$...
user.credentials[1].username=user2
user.credentials[1].password=$2b$08$...
user.credentials[2].username=user3
user.credentials[2].password=$2b$10$...`,
expected: `user:
credentials:
- username: user1
password: $2b$08$...
- username: user2
password: $2b$08$...
- username: user3
password: $2b$10$...
`,
scenarioType: "decode-array-brackets",
},
{
skipDoc: true,
description: "Decode properties with nested array brackets",
input: `user.clowns[0][1] = "cool"`,
expected: `user:
clowns:
- - null
- '"cool"'
`,
scenarioType: "decode-array-brackets",
},
{
skipDoc: true,
@@ -473,12 +442,6 @@ func TestPropertyScenarios(t *testing.T) {
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(ConfiguredPropertiesPreferences)), s.description)
case "decode":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
case "decode-array-brackets":
previousPreferences := ConfiguredPropertiesPreferences.Copy()
ConfiguredPropertiesPreferences.UseArrayBrackets = true
actual := mustProcessFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(ConfiguredYamlPreferences))
ConfiguredPropertiesPreferences = previousPreferences
test.AssertResultWithContext(t, s.expected, actual, s.description)
case "encode-wrapped":
prefs := ConfiguredPropertiesPreferences.Copy()
prefs.UnwrapScalar = false
+4 -6
View File
@@ -1,13 +1,11 @@
package yqlib
type SecurityPreferences struct {
DisableEnvOps bool
DisableFileOps bool
EnableSystemOps bool
DisableEnvOps bool
DisableFileOps bool
}
var ConfiguredSecurityPreferences = SecurityPreferences{
DisableEnvOps: false,
DisableFileOps: false,
EnableSystemOps: false,
DisableEnvOps: false,
DisableFileOps: false,
}
+7 -2
View File
@@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"io"
"os"
)
// A yaml expression evaluator that runs the expression multiple times for each given yaml document.
@@ -49,17 +50,21 @@ func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, p
}
for _, filename := range filenames {
reader, cleanup, err := readStream(filename)
reader, err := readStream(filename)
if err != nil {
return err
}
processedDocs, err := s.Evaluate(filename, reader, node, printer, decoder)
cleanup()
if err != nil {
return err
}
totalProcessDocs = totalProcessDocs + processedDocs
switch reader := reader.(type) {
case *os.File:
safelyCloseFile(reader)
}
}
if totalProcessDocs == 0 {
+1 -243
View File
@@ -209,50 +209,6 @@ address = "12 cat st"
var rtEmptyArray = `A = []
`
var rtEmptyArrayInTable = `[features]
my-feature = []
`
var rtMixedEmptyArraysInTable = `[features]
my-other-feature = []
my-feature = ["my-other-feature"]
`
var yamlEmptyArrayInTable = `features:
my-feature: []
`
var expectedTomlEmptyArrayInTable = `[features]
my-feature = []
`
var yamlMixedEmptyArraysInTable = `features:
my-other-feature: []
my-feature:
- my-other-feature
`
var expectedTomlMixedEmptyArraysInTable = `[features]
my-other-feature = []
my-feature = ["my-other-feature"]
`
var issue2688SampleToml = `[project]
name = "some-project"
version = "0.5.1"
authors = [{name = "Author", email = "author@example.com"}]
license = { file = "LICENSE" }
readme = "README.md"
`
var issue2688SampleExpected = `[project]
name = "some-project"
version = "0.5.2"
authors = [{ name = "Author", email = "author@example.com" }]
license = { file = "LICENSE" }
readme = "README.md"
`
var rtSampleTable = `var = "x"
[owner.contact]
@@ -331,18 +287,6 @@ var expectedSubArrays = `array:
- {}
`
// Keys with special characters that require quoting in TOML
var rtSpecialKeyInlineTable = `host = { "http://sealos.hub:5000" = { capabilities = ["pull", "resolve", "push"], skip_verify = true } }
`
var rtSpecialKeyTableSection = `["/tmp/blah"]
value = "hello"
`
var rtSpecialKeyDottedTableSection = `[servers."http://localhost:8080"]
ip = "127.0.0.1"
`
var tomlScenarios = []formatScenario{
{
skipDoc: true,
@@ -369,7 +313,7 @@ var tomlScenarios = []formatScenario{
skipDoc: true,
description: "blank",
input: `A = "hello`,
expectedError: `bad file 'sample.yml': unterminated basic string`,
expectedError: `bad file 'sample.yml': basic string not terminated by "`,
scenarioType: "decode-error",
},
{
@@ -607,36 +551,6 @@ var tomlScenarios = []formatScenario{
expected: rtEmptyArray,
scenarioType: "roundtrip",
},
{
skipDoc: true,
description: "Issue #2674: roundtrip empty array in table",
input: rtEmptyArrayInTable,
expression: ".",
expected: rtEmptyArrayInTable,
scenarioType: "roundtrip",
},
{
skipDoc: true,
description: "Issue #2674: roundtrip mixed empty and non-empty arrays in table",
input: rtMixedEmptyArraysInTable,
expression: ".",
expected: rtMixedEmptyArraysInTable,
scenarioType: "roundtrip",
},
{
skipDoc: true,
description: "Issue #2674: encode empty array in table",
input: yamlEmptyArrayInTable,
expected: expectedTomlEmptyArrayInTable,
scenarioType: "encode",
},
{
skipDoc: true,
description: "Issue #2674: encode mixed empty and non-empty arrays in table",
input: yamlMixedEmptyArraysInTable,
expected: expectedTomlMixedEmptyArraysInTable,
scenarioType: "encode",
},
{
description: "Roundtrip: sample table",
input: rtSampleTable,
@@ -700,80 +614,6 @@ var tomlScenarios = []formatScenario{
expected: tomlTableWithComments,
scenarioType: "roundtrip",
},
// Encode (YAML → TOML) scenarios - verify readable table sections are produced
{
description: "Encode: Simple mapping produces table section",
input: "arg:\n hello: foo\n",
expected: "[arg]\nhello = \"foo\"\n",
scenarioType: "encode",
},
{
skipDoc: true,
description: "Encode: Nested mappings produce nested table sections",
input: "a:\n b:\n c: val\n",
expected: "[a.b]\nc = \"val\"\n",
scenarioType: "encode",
},
{
skipDoc: true,
description: "Encode: Mixed scalars and nested mapping",
input: "a:\n hello: foo\n nested:\n key: val\n",
expected: "[a]\nhello = \"foo\"\n\n[a.nested]\nkey = \"val\"\n",
scenarioType: "encode",
},
{
skipDoc: true,
description: "Encode: YAML flow mapping produces table section (same as block mapping)",
input: "arg: {hello: foo}\n",
expected: "[arg]\nhello = \"foo\"\n",
scenarioType: "encode",
},
{
skipDoc: true,
description: "Issue: JSON auto-detected via YAML decoder produces table sections",
input: `{"arg":{"hello": "foo"}}`,
expected: "[arg]\nhello = \"foo\"\n",
scenarioType: "encode",
},
{
skipDoc: true,
description: "Issue: JSON via JSON decoder produces table sections",
input: `{"arg":{"hello": "foo"}}`,
expected: "[arg]\nhello = \"foo\"\n",
scenarioType: "encode-json",
},
{
skipDoc: true,
description: "Issue 2688: inline table arrays do not change following table scope",
input: issue2688SampleToml,
expression: `.project.version = "0.5.2"`,
expected: issue2688SampleExpected,
scenarioType: "roundtrip",
},
{
skipDoc: true,
description: "Roundtrip: key with special characters in inline table",
input: rtSpecialKeyInlineTable,
expression: ".",
expected: rtSpecialKeyInlineTable,
scenarioType: "roundtrip",
},
{
skipDoc: true,
description: "Roundtrip: key with special characters in table section",
input: rtSpecialKeyTableSection,
expression: ".",
expected: rtSpecialKeyTableSection,
scenarioType: "roundtrip",
},
{
skipDoc: true,
description: "Roundtrip: special character key in dotted table section header",
input: rtSpecialKeyDottedTableSection,
expression: ".",
expected: rtSpecialKeyDottedTableSection,
scenarioType: "roundtrip",
},
}
func testTomlScenario(t *testing.T, s formatScenario) {
@@ -789,10 +629,6 @@ func testTomlScenario(t *testing.T, s formatScenario) {
}
case "roundtrip":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewTomlDecoder(), NewTomlEncoder()), s.description)
case "encode":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewTomlEncoder()), s.description)
case "encode-json":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewTomlEncoder()), s.description)
}
}
@@ -818,28 +654,6 @@ func documentTomlDecodeScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(ConfiguredYamlPreferences))))
}
func documentTomlEncodeScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
if s.subdescription != "" {
writeOrPanic(w, s.subdescription)
writeOrPanic(w, "\n\n")
}
writeOrPanic(w, "Given a sample.yml file of:\n")
writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input))
writeOrPanic(w, "then\n")
expression := s.expression
if expression == "" {
expression = "."
}
writeOrPanic(w, fmt.Sprintf("```bash\nyq -o toml '%v' sample.yml\n```\n", expression))
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```toml\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewTomlEncoder())))
}
func documentTomlRoundtripScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
@@ -873,8 +687,6 @@ func documentTomlScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
documentTomlDecodeScenario(w, s)
case "roundtrip":
documentTomlRoundtripScenario(w, s)
case "encode", "encode-json":
documentTomlEncodeScenario(w, s)
default:
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
@@ -892,60 +704,6 @@ func TestTomlScenarios(t *testing.T) {
documentScenarios(t, "usage", "toml", genericScenarios, documentTomlScenario)
}
func TestTomlEncodeJsonKeepsRootArrayBeforeTables(t *testing.T) {
scenario := formatScenario{
description: "Encode: JSON root array stays outside later tables",
input: `{
"_source": {
"cookie": [
{
"Domain": "",
"Expires": "0001-01-01T00:00:00Z",
"HttpOnly": false,
"MaxAge": 0,
"Name": "name",
"Path": "",
"Raw": "",
"RawExpires": "",
"SameSite": 0,
"Secure": false,
"Unparsed": null,
"Value": "value"
}
]
},
"highlight": {
"did": [
"did"
]
},
"sort": [
1
]
}`,
expected: `sort = [1]
[[_source.cookie]]
Domain = ""
Expires = "0001-01-01T00:00:00Z"
HttpOnly = false
MaxAge = 0
Name = "name"
Path = ""
Raw = ""
RawExpires = ""
SameSite = 0
Secure = false
Value = "value"
[highlight]
did = ["did"]
`,
}
test.AssertResultWithContext(t, scenario.expected, mustProcessFormatScenario(scenario, NewJSONDecoder(), NewTomlEncoder()), scenario.description)
}
// TestTomlColourization tests that colourization correctly distinguishes
// between table section headers and inline arrays
func TestTomlColourization(t *testing.T) {
+14 -13
View File
@@ -32,24 +32,25 @@ func resolveFilename(filename string) string {
return filename
}
// readStream returns a reader for the given file, along with a cleanup function
// that must be called once the reader is no longer needed. The cleanup is a no-op
// for stdin.
func readStream(filename string) (io.Reader, func(), error) {
func readStream(filename string) (io.Reader, error) {
var reader *bufio.Reader
if filename == "-" {
return bufio.NewReader(os.Stdin), func() {}, nil
reader = bufio.NewReader(os.Stdin)
} else {
// ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory,
// and ensuring that it's not possible to give a path to a file outside that directory.
file, err := os.Open(filename) // #nosec
if err != nil {
return nil, err
}
reader = bufio.NewReader(file)
}
// ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory,
// and ensuring that it's not possible to give a path to a file outside that directory.
file, err := os.Open(filename) // #nosec
if err != nil {
return nil, nil, err
}
return bufio.NewReader(file), func() { safelyCloseFile(file) }, nil
return reader, nil
}
func writeString(writer io.Writer, txt string) error {
_, errorWriting := io.WriteString(writer, txt)
_, errorWriting := writer.Write([]byte(txt))
return errorWriting
}
-119
View File
@@ -1,119 +0,0 @@
package yqlib
import (
"bufio"
"bytes"
"io"
"os"
"path/filepath"
"runtime/debug"
"strconv"
"testing"
"github.com/mikefarah/yq/v4/test"
)
// countOpenFileDescriptors returns the number of file descriptors this process
// currently holds open, or -1 if the platform does not expose them.
func countOpenFileDescriptors() int {
for _, dir := range []string{"/proc/self/fd", "/dev/fd"} {
// Readdirnames avoids stat-ing each entry, which races with descriptors
// (including this directory handle) being closed underneath us.
handle, err := os.Open(dir)
if err != nil {
continue
}
names, err := handle.Readdirnames(-1)
safelyCloseFile(handle)
if err != nil {
continue
}
// discount the directory handle itself
return len(names) - 1
}
return -1
}
func writeSampleFiles(t *testing.T, count int) []string {
t.Helper()
dir := t.TempDir()
filenames := make([]string, count)
for i := 0; i < count; i++ {
filename := filepath.Join(dir, "sample-"+strconv.Itoa(i)+".yml")
if err := os.WriteFile(filename, []byte("a: apple\n"), 0600); err != nil {
t.Fatalf("failed to write sample file: %v", err)
}
filenames[i] = filename
}
return filenames
}
func discardingPrinter() Printer {
return NewPrinter(NewYamlEncoder(ConfiguredYamlPreferences), NewSinglePrinterWriter(bufio.NewWriter(io.Discard)))
}
func assertNoLeakedFileDescriptors(t *testing.T, evaluate func(filenames []string) error) {
t.Helper()
InitExpressionParser()
// os.File finalisers close leaked descriptors on collection, which would
// let a genuine leak pass unnoticed.
defer debug.SetGCPercent(debug.SetGCPercent(-1))
before := countOpenFileDescriptors()
if before < 0 {
t.Skip("file descriptors are not observable on this platform")
}
if err := evaluate(writeSampleFiles(t, 50)); err != nil {
t.Fatalf("failed to evaluate files: %v", err)
}
after := countOpenFileDescriptors()
if after > before {
t.Errorf("expected no additional open file descriptors, had %d before and %d after", before, after)
}
}
func TestStreamEvaluatorClosesInputFiles(t *testing.T) {
assertNoLeakedFileDescriptors(t, func(filenames []string) error {
return NewStreamEvaluator().EvaluateFiles(".a", filenames, discardingPrinter(), NewYamlDecoder(ConfiguredYamlPreferences))
})
}
func TestAllAtOnceEvaluatorClosesInputFiles(t *testing.T) {
assertNoLeakedFileDescriptors(t, func(filenames []string) error {
return NewAllAtOnceEvaluator().EvaluateFiles(".a", filenames, discardingPrinter(), NewYamlDecoder(ConfiguredYamlPreferences))
})
}
// plainWriter only implements io.Writer, so io.WriteString must fall back to Write.
type plainWriter struct {
buf bytes.Buffer
}
func (w *plainWriter) Write(p []byte) (int, error) {
return w.buf.Write(p)
}
func TestWriteStringToStringWriter(t *testing.T) {
var buf bytes.Buffer
writer := bufio.NewWriter(&buf)
test.AssertResult(t, nil, writeString(writer, "hello world"))
test.AssertResult(t, nil, writer.Flush())
test.AssertResult(t, "hello world", buf.String())
}
func TestWriteStringToPlainWriter(t *testing.T) {
writer := &plainWriter{}
test.AssertResult(t, nil, writeString(writer, "hello world"))
test.AssertResult(t, "hello world", writer.buf.String())
}
func TestWriteStringDoesNotAllocate(t *testing.T) {
writer := bufio.NewWriter(io.Discard)
allocations := testing.AllocsPerRun(100, func() {
_ = writeString(writer, "hello world")
})
test.AssertResult(t, 0.0, allocations)
}
+306
View File
@@ -0,0 +1,306 @@
abxbbxdbxebxczzx
abxbbxdbxebxczzy
accum
Accum
adithyasunil
AEDT
água
ÁGUA
alecthomas
appleapple
Astuff
autocreating
autoparse
AWST
axbxcxdxe
axbxcxdxexxx
bananabanana
barp
nbaz
bitnami
blarp
blddir
Bobo
BODMAS
bonapite
Brien
Bstuff
BUILDKIT
buildpackage
catmeow
CATYPE
CBVVE
chardata
chillum
choco
chomper
cleanup
cmlu
colorise
colors
Colors
colourize
compinit
coolioo
coverprofile
createmap
csvd
CSVUTF
currentlabel
cygpath
czvf
datestring
datetime
Datetime
datetimes
DEBEMAIL
debhelper
Debugf
debuild
delish
delpaths
DELPATHS
devorbitus
devscripts
dimchansky
Dont
dput
elliotchance
endhint
endofname
Entriesfrom
envsubst
errorlevel
Escandón
Evalall
fakefilename
fakeroot
Farah
fatih
Fifi
filebytes
Fileish
foobar
foobaz
foof
frood
fullpath
gitbook
githubactions
gnupg
goccy
gofmt
gogo
golangci
goreleaser
GORELEASER
GOMODCACHE
GOPATH
gosec
gota
goversion
GOVERSION
haha
hellno
herbygillot
hexdump
Hoang
hostpath
hotdog
howdy
incase
Infof
inlinetables
inplace
ints
ireduce
iwatch
jinzhu
jq's
jsond
keygrip
Keygrip
KEYGRIP
KEYID
keyvalue
kwak
lalilu
ldflags
LDFLAGS
lexer
Lexer
libdistro
lindex
linecomment
LVAs
magiconair
mapvalues
Mier
mikefarah
minideb
minishift
mipsle
mitchellh
mktemp
Mult
multidoc
multimaint
myenv
myenvnonexisting
myfile
myformat
ndjson
NDJSON
NFKD
nixpkgs
nojson
nonascii
nonempty
noninteractive
Nonquoting
nosec
notoml
noxml
nolua
nullinput
onea
Oneshot
opencollect
opstack
orderedmap
osarch
overridign
pacman
Padder
pandoc
parsechangelog
pcsv
pelletier
pflag
prechecking
Prerelease
proc
propsd
qylib
readline
realnames
realpath
repr
rhash
rindex
risentveber
rmescandon
Rosey
roundtrip
roundtrips
Roundtrip
roundtripping
Interp
interp
runningvms
sadface
selfupdate
setpath
sharedfolder
Sharedfolder
shellvariables
shellvars
shortfunc
shortpipe
shunit
snapcraft
somevalue
splt
srcdir
stackoverflow
stiched
Strc
strenv
strload
stylig
subarray
subchild
subdescription
submatch
submatches
SUBSTR
tempfile
tfstate
Tfstate
thar
timezone
Timezone
timezones
Timezones
tojson
Tokenvalue
tsvd
Tuan
tzdata
Uhoh
updateassign
urid
utfbom
Warningf
Wazowski
webi
Webi
wherever
winget
withdots
wizz
woop
workdir
Writable
xmld
xyzzy
yamld
yqlib
yuin
zabbix
tonumber
noyaml
nolint
shortfile
Unmarshalling
noini
nocsv
nobase64
nouri
noprops
nosh
noshell
tinygo
nonexistent
hclsyntax
hclwrite
nohcl
zclconf
cty
go-cty
Colorisation
goimports
errorlint
RDBMS
expeñded
bananabananabananabanana
edwinjhlee
flox
unlabelled
kyaml
KYAML
nokyaml
buildvcs
behaviour
GOFLAGS
gocache
subsubarray
Ffile
Fquery
coverpkg
gsub
ralia
Austr
ustrali
héllo
alia
-16
View File
@@ -8,22 +8,6 @@
- git push --tags
- use github actions to publish docker and make github release
- check github updated yq action in marketplace
- update github-action/Dockerfile to pin the newly published docker image digest (must match the mikefarah/yq:4 manifest digest):
docker buildx imagetools inspect docker.io/mikefarah/yq:4 --format '{{printf "%s" .Manifest.Digest}}'
then update the FROM line in github-action/Dockerfile with the new digest:
FROM mikefarah/yq:4@<digest-from-above>
- commit the Dockerfile change, then manually run the "Release Docker GitHub Action" workflow
(Actions -> Release Docker GitHub Action -> Run workflow)
- update action.yml to pin the newly published github-action image digest (must match the mikefarah/yq:4-githubaction manifest digest):
docker buildx imagetools inspect docker.io/mikefarah/yq:4-githubaction --format '{{printf "%s" .Manifest.Digest}}'
then update the image line in action.yml with the new digest:
image: 'docker://mikefarah/yq:4-githubaction@<digest-from-above>'
- commit the action.yml change and push
// release artifacts are signed with cosign keyless signing (Sigstore)
// users can verify with:
// cosign verify-blob --bundle checksums.bundle checksums
// install cosign: brew install cosign OR go install github.com/sigstore/cosign/v2/cmd/cosign@v2.6.1
- snapcraft
-47
View File
@@ -1,50 +1,3 @@
4.53.4:
- Close input files after processing each one (#2796) (#2808) Thanks @MsfPablo
- Fixed non string HCL string keys (#2795)
- Fixed heap allocation bug (#2809) Thanks @MsfPablo
- Fix deleting commented empty list YAML output (#2765) Thanks @dpersek
- Fix (has) return false for negative array indices (#2769) Thanks @maximilize
- Fix (sort): avoid int64 overflow comparing large integers (#2771) Thanks @maximilize
- Fix: preserve correct parent references in explode merge anchor reconstruction (#2730) Thanks @vomba
- Improve Guard ExpressionParser initialization with sync.Once (#2789) Thanks @arcaven
- Fix: reject negative indent instead of panicking (#2746) Thanks @StressTestor
- Fix: parse signed hex and octal integers (#2749) Thanks @StressTestor
- Fix: skip UTF-8 BOM when processing front matter (#2751) Thanks @StressTestor
- Fix !!merge tag regression for yq (#2705) Thanks @W-Floyd
- Default to yaml when a file's extension is not a recognised format (#2785) Thanks @devthedevil
- Bumped dependencies
4.53.3:
- Add `--ini-preserve-quotes` flag for INI round-trip quote preservation (#2728) Thanks @toller892!
- Fix: reset INI decoder state on init (#2719) Thanks @xieby1!
- Fix: decode properties array bracket paths (#2693) Thanks @cyphercodes!
- Fix: preserve floats with trailing zero when encoding YAML to JSON (#2701) Thanks @ChrisJr404!
- Fix: JSON to TOML root scope and null handling (#2689) Thanks @LovesAsuna!
- Fix: reset TOML decoder finished flag on Init for multi-doc evaluation (#2704) Thanks @terminalchai!
- Fix: reset TOML decoder between files when evaluating all at once (#2685) Thanks @terminalchai!
- Fix: preserve TOML inline table array scope (#2694) Thanks @cyphercodes!
- Fix: preserve empty TOML arrays in tables (#2686) Thanks @cyphercodes!
- Fix: TOML encoder uses inline tables for YAML FlowStyle mappings (#2687)
- Fix nested inline YAML merge explode (#2699) Thanks @cyphercodes!
- Fix repeatString overflow test on 32-bit platforms (#2680) Thanks @jandubois!
- Bumped dependencies
4.53.2:
- Fixing release process
4.53.1:
- Releases and tags now signed and immutable!
- Add system(command; args) operator (disabled by default) (#2640)
- TOML encoder: prefer readable table sections over inline tables (#2649)
- Fix TOML encoder to quote keys containing special characters (#2648)
- Add string slicing support (#2639)
- Fix findInArray misuse on MappingNodes in equality and contains (#2645) Thanks @jandubois!
- Fix panic on negative slice indices that underflow after adjustment (#2646) Thanks @jandubois!
- Fix stack overflow from circular alias in traverse (#2647) Thanks @jandubois!
- Fix panic and OOM in repeatString for large repeat counts (#2644) Thanks @jandubois!
- Bumped dependencies
4.52.5:
- Fix: reset TOML decoder state between files (#2634) thanks @terminalchai
- Fix: preserve original filename when using --front-matter (#2613) thanks @cobyfrombrooklyn-bot
+5 -8
View File
@@ -8,17 +8,14 @@ fi
version=$1
# validate version is in the right format (bash regex — portable; GNU sed's q1 is not on macOS)
if [[ ! $version =~ ^v4\.[0-9][0-9]\.[0-9][0-9]?$ ]]; then
echo "Please specify a valid version (e.g. v4.53.3)"
exit 1
fi
# validate version is in the right format
echo $version | sed -r '/v4\.[0-9][0-9]\.[0-9][0-9]?$/!{q1}'
previousVersion=$(cat cmd/version.go| sed -n 's/.*Version = "\([^"]*\)"/\1/p')
echo "Updating from $previousVersion to $version"
sed "s/\(.*Version =\).*/\1 \"$version\"/" cmd/version.go > cmd/version.go.tmp && mv cmd/version.go.tmp cmd/version.go
sed -i "s/\(.*Version =\).*/\1 \"$version\"/" cmd/version.go
go build .
actualVersion=$(./yq --version)
@@ -52,5 +49,5 @@ fi
git add cmd/version.go snap/snapcraft.yaml
git commit -m 'Bumping version'
git tag $version -m "releasing"
git tag -f v4 -m "releasing $version"
git tag $version
git tag -f v4
+3 -48
View File
@@ -1,50 +1,5 @@
#!/bin/sh
set -ex
go mod download golang.org/x/tools@v0.44.0
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/6008b81b81c690c046ffc3fd5bce896da715d5fd/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.11.3
curl -sSfL https://raw.githubusercontent.com/securego/gosec/424fc4cd9c82ea0fd6bee9cd49c2db2c3cc0c93f/install.sh | sh -s v2.22.11
TYPOS_VERSION=v1.47.2
TYPOS_OS=$(uname -s | tr '[:upper:]' '[:lower:]')
TYPOS_ARCH=$(uname -m)
case "${TYPOS_ARCH}" in
x86_64) TYPOS_ARCH=x86_64 ;;
aarch64|arm64) TYPOS_ARCH=aarch64 ;;
*)
echo "unsupported architecture for typos: ${TYPOS_ARCH}"
exit 1
;;
esac
case "${TYPOS_OS}" in
linux) TYPOS_TARGET="${TYPOS_ARCH}-unknown-linux-musl" ;;
darwin) TYPOS_TARGET="${TYPOS_ARCH}-apple-darwin" ;;
*)
echo "unsupported OS for typos: ${TYPOS_OS}"
exit 1
;;
esac
TYPOS_ARCHIVE="typos-${TYPOS_VERSION}-${TYPOS_TARGET}.tar.gz"
TYPOS_URL="https://github.com/crate-ci/typos/releases/download/${TYPOS_VERSION}/${TYPOS_ARCHIVE}"
case "${TYPOS_TARGET}" in
aarch64-apple-darwin) TYPOS_SHA256=23ca24a9186b5cb395b5f6c8eea8cdb02911c8980833e016454b56e90c3bd474 ;;
aarch64-unknown-linux-musl) TYPOS_SHA256=596d5c6b9ecf34307f68bea649178c5b45a4398fe3a1fcef9598e85aa2ccb742 ;;
x86_64-apple-darwin) TYPOS_SHA256=469a2d9fc894b0cdcec6e4fa3719b4c4638e195feee6517d4845450f8e8985c6 ;;
x86_64-unknown-linux-musl) TYPOS_SHA256=7aef58932fc123b4cf4b40d86468e89a3297d80169051d7cfd13a235e05fc426 ;;
*)
echo "unsupported typos target: ${TYPOS_TARGET}"
exit 1
;;
esac
TYPOS_TMPDIR=$(mktemp -d)
curl -sSfL "${TYPOS_URL}" -o "${TYPOS_TMPDIR}/${TYPOS_ARCHIVE}"
TYPOS_ACTUAL_SHA256=$(sha256sum "${TYPOS_TMPDIR}/${TYPOS_ARCHIVE}" 2>/dev/null | cut -d' ' -f1)
if [ -z "${TYPOS_ACTUAL_SHA256}" ]; then
TYPOS_ACTUAL_SHA256=$(shasum -a 256 "${TYPOS_TMPDIR}/${TYPOS_ARCHIVE}" | cut -d' ' -f1)
fi
if [ "${TYPOS_ACTUAL_SHA256}" != "${TYPOS_SHA256}" ]; then
echo "typos archive checksum mismatch: expected ${TYPOS_SHA256}, got ${TYPOS_ACTUAL_SHA256}"
exit 1
fi
tar xzf "${TYPOS_TMPDIR}/${TYPOS_ARCHIVE}" -C "${TYPOS_TMPDIR}"
install -m 755 "${TYPOS_TMPDIR}/typos" "$(go env GOPATH)/bin/typos"
rm -rf "${TYPOS_TMPDIR}"
go mod download golang.org/x/tools@latest
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.11.3
curl -sSfL https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.22.11
+1 -16
View File
@@ -1,18 +1,3 @@
#!/bin/bash
set -euo pipefail
GOPATH_TYPOS="$(go env GOPATH)/bin/typos"
TYPOS_CMD=""
if [ -f "${GOPATH_TYPOS}" ]; then
TYPOS_CMD="${GOPATH_TYPOS}"
elif command -v typos >/dev/null 2>&1; then
TYPOS_CMD="typos"
else
echo "Error: typos not found in $(go env GOPATH)/bin or PATH."
echo "Please run scripts/devtools.sh or ensure typos is installed correctly."
exit 1
fi
git ls-files '*.go' '*.sh' '*.md' | "${TYPOS_CMD}" --file-list -
npx cspell --no-progress "**/*.{sh,go,md}"
-4
View File
@@ -1,7 +1,3 @@
#!/bin/bash
go test $(go list ./... | grep -v -E 'examples' | grep -v -E 'test')
# Run after the main test suite: TestGoInstallCompatibility zips the module tree and
# must not run in parallel with pkg/yqlib tests that rewrite doc/*.md files.
go test -tags goinstall -run TestGoInstallCompatibility .
+2 -2
View File
@@ -2,9 +2,9 @@
set -eo pipefail
# You may need to go install github.com/goreleaser/goreleaser/v2@v2.16.0 first
# You may need to go install github.com/goreleaser/goreleaser/v2@latest first
GORELEASER="goreleaser build --clean"
if [ -z "$CI" ] || [[ "${GITHUB_REF_NAME:-}" == draft-* ]]; then
if [ -z "$CI" ]; then
GORELEASER+=" --snapshot"
fi
+2 -2
View File
@@ -1,5 +1,5 @@
name: yq
version: 'v4.53.4'
version: 'v4.52.5'
summary: A lightweight and portable command-line data file processor
description: |
`yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml, json, xml, csv, properties and TOML files.
@@ -32,6 +32,6 @@ parts:
build-environment:
- CGO_ENABLED: 0
source: https://github.com/mikefarah/yq.git
source-tag: v4.53.4
source-tag: v4.52.5
build-snaps:
- go/latest/stable