Compare commits

..

1 Commits

Author SHA1 Message Date
Mike Farah
1d0736f956
Securing release workflow wip 2026-04-13 10:56:23 +10:00
69 changed files with 589 additions and 1235 deletions

View File

@ -20,8 +20,6 @@ on:
schedule:
- cron: '24 3 * * 1'
permissions: {}
jobs:
analyze:
name: Analyze
@ -40,11 +38,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
uses: github/codeql-action/init@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
@ -55,7 +53,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
uses: github/codeql-action/autobuild@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1
# Command-line programs to run using the OS shell.
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@ -69,4 +67,4 @@ jobs:
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
uses: github/codeql-action/analyze@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1

View File

@ -1,107 +0,0 @@
name: Release Docker GitHub Action
on:
workflow_dispatch:
permissions: {}
jobs:
publishGithubActionDocker:
environment: dockerhub
env:
IMAGE_NAME: mikefarah/yq
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
- name: Set up QEMU
uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # v4.1.0
with:
platforms: all
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
with:
version: latest
- name: Verify Dockerfile base image digest matches yq:4
run: |
PINNED_DIGEST=$(grep -oE 'sha256:[a-f0-9]{64}' github-action/Dockerfile | head -1)
if [ -z "${PINNED_DIGEST}" ]; then
echo "::error::Could not find a sha256 digest in github-action/Dockerfile"
exit 1
fi
LATEST_DIGEST=$(docker buildx imagetools inspect "${IMAGE_NAME}:4" --format '{{printf "%s" .Manifest.Digest}}')
echo "Dockerfile pins: ${PINNED_DIGEST}"
echo "mikefarah/yq:4 is: ${LATEST_DIGEST}"
if [ "${PINNED_DIGEST}" != "${LATEST_DIGEST}" ]; then
echo "::error::github-action/Dockerfile digest does not match the current mikefarah/yq:4 image"
echo "Update the FROM line in github-action/Dockerfile to:"
echo " FROM mikefarah/yq:4@${LATEST_DIGEST}"
exit 1
fi
- name: Resolve version from yq:4
run: |
IMAGE_VERSION=$(docker run --rm "${IMAGE_NAME}:4" --version | awk '{print $NF}' | sed 's/^v//')
if [ -z "${IMAGE_VERSION}" ]; then
echo "::error::Could not determine yq version from ${IMAGE_NAME}:4"
exit 1
fi
echo "Resolved yq version: ${IMAGE_VERSION}"
echo "IMAGE_VERSION=${IMAGE_VERSION}" >> "${GITHUB_ENV}"
- name: Login to Docker Hub
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push github-action image
working-directory: github-action
run: |
PLATFORMS="linux/amd64,linux/ppc64le,linux/arm64,linux/arm/v7,linux/s390x"
echo "Building and pushing github-action image for version ${IMAGE_VERSION}"
docker buildx build \
--label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
--label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
--label "org.opencontainers.image.description=yq is a portable command-line data file processor" \
--label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
--label "org.opencontainers.image.licenses=MIT" \
--label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \
--label "org.opencontainers.image.source=https://github.com/mikefarah/yq" \
--label "org.opencontainers.image.title=yq" \
--label "org.opencontainers.image.url=https://mikefarah.gitbook.io/yq/" \
--label "org.opencontainers.image.version=${IMAGE_VERSION}" \
--platform "${PLATFORMS}" \
--pull \
--push \
-t "${IMAGE_NAME}:${IMAGE_VERSION}-githubaction" \
-t "${IMAGE_NAME}:4-githubaction" \
-t "${IMAGE_NAME}:latest-githubaction" \
-t "ghcr.io/${IMAGE_NAME}:${IMAGE_VERSION}-githubaction" \
-t "ghcr.io/${IMAGE_NAME}:4-githubaction" \
-t "ghcr.io/${IMAGE_NAME}:latest-githubaction" \
.
- name: Report action.yml digest to pin
run: |
GITHUBACTION_DIGEST=$(docker buildx imagetools inspect "${IMAGE_NAME}:4-githubaction" --format '{{printf "%s" .Manifest.Digest}}')
echo "Published ${IMAGE_NAME}:4-githubaction at ${GITHUBACTION_DIGEST}"
echo "Update action.yml image to:"
echo " docker://${IMAGE_NAME}:4-githubaction@${GITHUBACTION_DIGEST}"

View File

@ -7,7 +7,9 @@ on:
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
permissions: {}
permissions:
contents: read
packages: write
jobs:
publishDocker:
@ -15,20 +17,17 @@ jobs:
env:
IMAGE_NAME: mikefarah/yq
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Set up QEMU
uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # v4.1.0
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0
with:
platforms: all
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
with:
version: latest
@ -36,13 +35,13 @@ jobs:
run: echo ${{ steps.buildx.outputs.platforms }} && docker version
- name: Login to Docker Hub
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
with:
registry: ghcr.io
username: ${{ github.actor }}
@ -80,3 +79,26 @@ jobs:
-t "ghcr.io/${IMAGE_NAME}:4" \
-t "ghcr.io/${IMAGE_NAME}:latest" \
.
cd github-action
docker buildx build \
--label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
--label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
--label "org.opencontainers.image.description=yq is a portable command-line data file processor" \
--label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
--label "org.opencontainers.image.licenses=MIT" \
--label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \
--label "org.opencontainers.image.source=https://github.com/mikefarah/yq" \
--label "org.opencontainers.image.title=yq" \
--label "org.opencontainers.image.url=https://mikefarah.gitbook.io/yq/" \
--label "org.opencontainers.image.version=${IMAGE_VERSION}" \
--platform "${PLATFORMS}" \
--pull \
--push \
-t "${IMAGE_NAME}:${IMAGE_VERSION}-githubaction" \
-t "${IMAGE_NAME}:4-githubaction" \
-t "${IMAGE_NAME}:latest-githubaction" \
-t "ghcr.io/${IMAGE_NAME}:${IMAGE_VERSION}-githubaction" \
-t "ghcr.io/${IMAGE_NAME}:4-githubaction" \
-t "ghcr.io/${IMAGE_NAME}:latest-githubaction" \
.

View File

@ -5,53 +5,23 @@ permissions:
jobs:
verify-action-digest:
name: Verify action.yml image digest
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
- name: Verify action.yml digest matches published image
run: |
PINNED_DIGEST=$(grep -oE 'sha256:[a-f0-9]{64}' action.yml | head -1)
if [ -z "${PINNED_DIGEST}" ]; then
echo "::error::action.yml does not pin the runtime image by digest"
exit 1
fi
LATEST_DIGEST=$(docker buildx imagetools inspect docker.io/mikefarah/yq:4-githubaction --format '{{printf "%s" .Manifest.Digest}}')
echo "action.yml pins: ${PINNED_DIGEST}"
echo "mikefarah/yq:4-githubaction: ${LATEST_DIGEST}"
if [ "${PINNED_DIGEST}" != "${LATEST_DIGEST}" ]; then
echo "::error::action.yml digest does not match the current mikefarah/yq:4-githubaction image"
echo "Update the image line in action.yml to:"
echo " docker://mikefarah/yq:4-githubaction@${LATEST_DIGEST}"
exit 1
fi
build:
name: Build
runs-on: ubuntu-latest
steps:
- name: Set up Go
uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
with:
go-version: '^1.20'
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Get dependencies
run: |
go get -v -t -d ./...
if [ -f Gopkg.toml ]; then
curl -sSfL https://raw.githubusercontent.com/golang/dep/1f7c19e5f52f49ffb9f956f64c010be14683468b/install.sh | env DEP_RELEASE_TAG=v0.5.4 sh
dep ensure
fi
- name: Check the build
shell: bash -l {0}

View File

@ -5,8 +5,6 @@ on:
- 'v4.*'
- 'draft-*'
permissions: {}
jobs:
publishGitRelease:
runs-on: ubuntu-latest
@ -14,8 +12,8 @@ jobs:
contents: write
id-token: write
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
- uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
with:
go-version: '^1.20'
check-latest: true
@ -29,7 +27,7 @@ jobs:
run: echo "VERSION=${GITHUB_REF##*/}" >> "${GITHUB_OUTPUT}"
- name: Generate man page
uses: docker://pandoc/core:2.14.2@sha256:04e127c6642a2b9d447c26fe0ac6a5932efa8f508eda9f07da51b6e621dd7c19
uses: docker://pandoc/core:2.14.2
id: gen-man-page
with:
args: >-
@ -43,12 +41,12 @@ jobs:
man.md
- name: Install cosign
uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
uses: sigstore/cosign-installer@cad07c2e89fa2edd6e2d7bab4c1aa38e53f76003 # v4.1.1
- name: Cross compile
run: |
sudo apt-get install rhash -y
go install github.com/goreleaser/goreleaser/v2@v2.16.0
go install github.com/goreleaser/goreleaser/v2@v2.15.2
./scripts/xcompile.sh
- name: Sign checksums
@ -57,7 +55,7 @@ jobs:
cosign sign-blob --yes --bundle build/checksums-bsd.bundle build/checksums-bsd
- name: Release
uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v3.0.0
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2.6.2
with:
files: build/*
draft: true

View File

@ -1,78 +0,0 @@
# This workflow uses actions that are not certified by GitHub. They are provided
# by a third-party and are governed by separate terms of service, privacy
# policy, and support documentation.
name: Scorecard supply-chain security
on:
# For Branch-Protection check. Only the default branch is supported. See
# https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
branch_protection_rule:
# To guarantee Maintained check is occasionally updated. See
# https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
schedule:
- cron: '39 7 * * 2'
push:
branches: [ "master" ]
# Declare default permissions as read only.
permissions: read-all
jobs:
analysis:
name: Scorecard analysis
runs-on: ubuntu-latest
# `publish_results: true` only works when run from the default branch. conditional can be removed if disabled.
if: github.event.repository.default_branch == github.ref_name || github.event_name == 'pull_request'
permissions:
# Needed to upload the results to code-scanning dashboard.
security-events: write
# Needed to publish results and get a badge (see publish_results below).
id-token: write
# Uncomment the permissions below if installing in a private repository.
# contents: read
# actions: read
steps:
- name: "Checkout code"
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
with:
persist-credentials: false
- name: "Run analysis"
uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3
with:
results_file: results.sarif
results_format: sarif
# (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
# - you want to enable the Branch-Protection check on a *public* repository, or
# - you are installing Scorecard on a *private* repository
# To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional.
# repo_token: ${{ secrets.SCORECARD_TOKEN }}
# Public repositories:
# - Publish results to OpenSSF REST API for easy access by consumers
# - Allows the repository to include the Scorecard badge.
# - See https://github.com/ossf/scorecard-action#publishing-results.
# For private repositories:
# - `publish_results` will always be set to `false`, regardless
# of the value entered here.
publish_results: true
# (Optional) Uncomment file_mode if you have a .gitattributes with files marked export-ignore
# file_mode: git
# Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
# format to the repository Actions tab.
- name: "Upload artifact"
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: SARIF file
path: results.sarif
retention-days: 5
# Upload the results to GitHub's code scanning dashboard (optional).
# Commenting out will disable upload of results to your repo's Code Scanning dashboard
- name: "Upload to code-scanning"
uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
with:
sarif_file: results.sarif

View File

@ -7,16 +7,15 @@ on:
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
permissions: {}
permissions:
contents: read
jobs:
buildSnap:
environment: snap
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: snapcore/action-build@3bdaa03e1ba6bf59a65f84a751d943d549a54e79 # v1.3.0
id: build
env:

View File

@ -13,7 +13,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: Get test
id: get_value
uses: mikefarah/yq@master

View File

@ -1,4 +1,4 @@
FROM golang:1.26.4@sha256:792443b89f65105abba56b9bd5e97f680a80074ac62fc844a584212f8c8102c3 AS builder
FROM golang:1.26.2@sha256:2a2b4b5791cea8ae09caecba7bad0bd9631def96e5fe362e4a5e67009fe4ae61 AS builder
WORKDIR /go/src/mikefarah/yq
@ -10,7 +10,7 @@ RUN ./scripts/acceptance.sh
# Choose alpine as a base image to make this useful for CI, as many
# CI tools expect an interactive shell inside the container
FROM alpine:3@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b AS production
FROM alpine:3@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659 AS production
LABEL maintainer="Mike Farah <mikefarah@users.noreply.github.com>"
COPY --from=builder /go/src/mikefarah/yq/yq /usr/bin/yq

View File

@ -1,4 +1,8 @@
FROM golang:1.26.4@sha256:792443b89f65105abba56b9bd5e97f680a80074ac62fc844a584212f8c8102c3
FROM golang:1.26.2@sha256:2a2b4b5791cea8ae09caecba7bad0bd9631def96e5fe362e4a5e67009fe4ae61
RUN apt-get update && \
apt-get install -y npm && \
npm install -g npx cspell@latest
COPY scripts/devtools.sh /opt/devtools.sh

View File

@ -42,7 +42,7 @@ quiet: # this is silly but shuts up 'Nothing to be done for `local`'
@:
prepare: tmp/dev_image_id
tmp/dev_image_id: Dockerfile.dev scripts/devtools.sh _typos.toml
tmp/dev_image_id: Dockerfile.dev scripts/devtools.sh
@mkdir -p tmp
@${ENGINE} rmi -f ${DEV_IMAGE} > /dev/null 2>&1 || true
@${ENGINE} build -t ${DEV_IMAGE} -f Dockerfile.dev .

View File

@ -1,20 +0,0 @@
[files]
extend-exclude = ["vendor", "bin"]
[default]
locale = "en"
extend-ignore-identifiers-re = [
"NdJson",
]
[default.extend-identifiers]
AttributeIDSupressMenu = "AttributeIDSupressMenu"
[default.extend-words]
Teh = "Teh"
teh = "teh"
Supress = "Supress"
HashiCorp = "HashiCorp"
Hashi = "Hashi"
fot = "fot"
nd = "nd"

View File

@ -12,6 +12,6 @@ outputs:
description: "The complete result from the yq command being run"
runs:
using: 'docker'
image: 'docker://mikefarah/yq:4-githubaction@sha256:e1b8c865f299ea6b02910a7ddf147d5d431244d4cc116f89c2148c9f53822906'
image: 'docker://mikefarah/yq:4-githubaction'
args:
- ${{ inputs.cmd }}

View File

@ -1,79 +1,3 @@
# yq — agent instructions
## ⚠️ MANDATORY: GitHub agent disclosure
**Always required. No exceptions.**
Whenever you perform **any** GitHub action on behalf of the user, you **must** disclose that an AI agent (Cursor) wrote the content and is acting on the user's behalf — **not the user personally**. Do this **before** submitting; never post first and add the disclosure later.
Applies to **all** GitHub interactions, including:
- Pull requests (titles, descriptions, and reviews)
- PR comments and inline review comments
- Issues (new issues, comments, and updates)
- Any other post or reply on GitHub
**How to disclose:** Put it prominently at the **top** of every PR description, review body, comment, or issue. Use wording like:
Inline review comments must include a short disclosure too (e.g. `> Generated by Cursor acting on the user's behalf, not the user personally.`).
**Never** submit a GitHub action without this disclosure.
---
Always run the spellcheck before raising a PR:
```bash
bash scripts/spelling.sh
```
This is also included in the full CI pipeline via `make local test`.
## Cursor Cloud specific instructions
### Overview
**yq** is a Go CLI for querying and transforming YAML, JSON, XML, INI, and other structured formats. There are no long-running services — development is build-and-test against a local `./yq` binary.
### Prerequisites
- **Go ≥ 1.25** (see `go.mod`)
- **Bash** (acceptance tests)
- **Docker/Podman** is optional; use `make local <target>` to run natively when containers are unavailable
### PATH
After `scripts/devtools.sh`, add Go tool binaries to PATH:
```bash
export PATH="$HOME/go/bin:$PATH"
```
`golangci-lint` and `typos` install to `$HOME/go/bin`; `gosec` installs to `./bin/gosec` in the repo root.
### Common commands (local, no Docker)
| Task | Command |
|------|---------|
| Install dev tools | `bash scripts/devtools.sh` |
| Vendor dependencies | `make local vendor` |
| Build binary | `go build -o yq .` or `make local build` |
| Format | `make local format` |
| Lint | `make local check` |
| Unit tests | `make local test` or `bash scripts/test.sh` |
| Acceptance (E2E) | `bash scripts/acceptance.sh` (requires `./yq` built first) |
`make local build` runs the full CI chain (format → spelling → gosec → lint → unit tests → build → acceptance). For a faster loop, build with `go build -o yq .` and run `bash scripts/acceptance.sh`.
### Caveats
- **`make` without `local`** tries Docker/Podman (`Dockerfile.dev`). In Cloud Agent VMs without Docker, always prefix with `make local`.
- **Spelling step** uses `typos` (installed by `scripts/devtools.sh`).
- **`make local test` / `scripts/check.sh`** require `golangci-lint` on PATH (`devtools.sh`).
---
# General rules
✅ **DO:**
- You can use ./yq with the `--debug-node-info` flag to get a deeper understanding of the ast.
@ -81,12 +5,10 @@ export PATH="$HOME/go/bin:$PATH"
- Add comprehensive tests to cover the changes
- Run test suite to ensure there is no regression
- Use UK english spelling
- **Follow the mandatory GitHub agent disclosure rule above** on every GitHub action — no exceptions
❌ **DON'T:**
- Git add or commit
- Add comments to functions that are self-explanatory
- **Post to GitHub without the mandatory agent disclosure** (PRs, reviews, comments, issues, or any other GitHub interaction)

View File

@ -156,8 +156,6 @@ yq -P -oy sample.json
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.UnquotedKeys, "lua-unquoted", yqlib.ConfiguredLuaPreferences.UnquotedKeys, "output unquoted string keys (e.g. {foo=\"bar\"})")
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.Globals, "lua-globals", yqlib.ConfiguredLuaPreferences.Globals, "output keys as top-level global variables")
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredINIPreferences.PreserveSurroundedQuote, "ini-preserve-quotes", yqlib.ConfiguredINIPreferences.PreserveSurroundedQuote, "preserve surrounding quotes on INI values during round-trip")
rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, "properties-separator", yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, "separator to use between keys and values")
rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredPropertiesPreferences.UseArrayBrackets, "properties-array-brackets", yqlib.ConfiguredPropertiesPreferences.UseArrayBrackets, "use [x] in array paths (e.g. for SpringBoot)")

View File

@ -11,7 +11,7 @@ var (
GitDescribe string
// Version is main version number that is being run at the moment.
Version = "v4.53.3"
Version = "v4.52.5"
// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release

14
cspell.config.yaml Normal file
View File

@ -0,0 +1,14 @@
---
$schema: https://raw.githubusercontent.com/streetsidesoftware/cspell/main/cspell.schema.json
version: '0.2'
language: en-GB
dictionaryDefinitions:
- name: project-words
path: './project-words.txt'
addWords: true
dictionaries:
- project-words
ignorePaths:
- 'vendor'
- 'bin'
- '/project-words.txt'

View File

@ -1,4 +1,4 @@
FROM mikefarah/yq:4@sha256:11a1f0b604b13dbbdc662260d8db6f644b22d8553122a25c1b5b2e8713ca6977
FROM mikefarah/yq:4@sha256:603ebff15eb308a05f1c5b8b7613179cad859aed3ec9fdd04f2ef5d32345950e
COPY entrypoint.sh /entrypoint.sh

18
go.mod
View File

@ -13,16 +13,16 @@ require (
github.com/hashicorp/hcl/v2 v2.24.0
github.com/jinzhu/copier v0.4.0
github.com/magiconair/properties v1.8.10
github.com/pelletier/go-toml/v2 v2.4.2
github.com/pelletier/go-toml/v2 v2.3.0
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
github.com/spf13/cobra v1.10.2
github.com/spf13/pflag v1.0.10
github.com/yuin/gopher-lua v1.1.2
github.com/zclconf/go-cty v1.18.1
go.yaml.in/yaml/v4 v4.0.0-rc.6
golang.org/x/mod v0.37.0
golang.org/x/net v0.56.0
golang.org/x/text v0.38.0
github.com/zclconf/go-cty v1.18.0
go.yaml.in/yaml/v4 v4.0.0-rc.4
golang.org/x/mod v0.34.0
golang.org/x/net v0.52.0
golang.org/x/text v0.35.0
)
require (
@ -33,9 +33,9 @@ require (
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
golang.org/x/sync v0.21.0 // indirect
golang.org/x/sys v0.46.0 // indirect
golang.org/x/tools v0.45.0 // indirect
golang.org/x/sync v0.20.0 // indirect
golang.org/x/sys v0.42.0 // indirect
golang.org/x/tools v0.42.0 // indirect
)
go 1.25.0

36
go.sum
View File

@ -46,8 +46,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
github.com/pelletier/go-toml/v2 v2.4.2 h1:M2fKKbmyvI+hGId/D0W64qDBMVhJnNR10O5gIbMc//Q=
github.com/pelletier/go-toml/v2 v2.4.2/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pelletier/go-toml/v2 v2.3.0 h1:k59bC/lIZREW0/iVaQR8nDHxVq8OVlIzYCOJf421CaM=
github.com/pelletier/go-toml/v2 v2.3.0/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@ -63,26 +63,26 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/yuin/gopher-lua v1.1.2 h1:yF/FjE3hD65tBbt0VXLE13HWS9h34fdzJmrWRXwobGA=
github.com/yuin/gopher-lua v1.1.2/go.mod h1:7aRmXIWl37SqRf0koeyylBEzJ+aPt8A+mmkQ4f1ntR8=
github.com/zclconf/go-cty v1.18.1 h1:yEGE8M4iIZlyKQURZNb2SnEyZlZHUcBCnx6KF81KuwM=
github.com/zclconf/go-cty v1.18.1/go.mod h1:qpnV6EDNgC1sns/AleL1fvatHw72j+S+nS+MJ+T2CSg=
github.com/zclconf/go-cty v1.18.0 h1:pJ8+HNI4gFoyRNqVE37wWbJWVw43BZczFo7KUoRczaA=
github.com/zclconf/go-cty v1.18.0/go.mod h1:qpnV6EDNgC1sns/AleL1fvatHw72j+S+nS+MJ+T2CSg=
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo=
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ=
golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0=
golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
go.yaml.in/yaml/v4 v4.0.0-rc.4 h1:UP4+v6fFrBIb1l934bDl//mmnoIZEDK0idg1+AIvX5U=
go.yaml.in/yaml/v4 v4.0.0-rc.4/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@ -1,5 +1,3 @@
//go:build goinstall
package main
import (
@ -13,10 +11,6 @@ import (
// TestGoInstallCompatibility ensures the module can be zipped for go install.
// This is an integration test that uses the same zip.CreateFromDir function
// that go install uses internally. If this test fails, go install will fail.
//
// Built with the goinstall tag and run after the main test suite (see scripts/test.sh)
// so it does not race with pkg/yqlib tests that rewrite doc/*.md during execution.
//
// See: https://github.com/mikefarah/yq/issues/2587
func TestGoInstallCompatibility(t *testing.T) {
mod := module.Version{

View File

@ -46,11 +46,7 @@ const (
func createStringScalarNode(stringValue string) *CandidateNode {
var node = &CandidateNode{Kind: ScalarNode}
node.Value = stringValue
if stringValue == "<<" {
node.Tag = "!!merge"
} else {
node.Tag = "!!str"
}
node.Tag = "!!str"
return node
}

View File

@ -7,9 +7,6 @@ import (
"errors"
"fmt"
"io"
"math"
"strconv"
"strings"
"github.com/goccy/go-json"
)
@ -143,12 +140,6 @@ func (o *CandidateNode) MarshalJSON() ([]byte, error) {
return buf.Bytes(), err
case ScalarNode:
log.Debugf("MarshalJSON ScalarNode")
if o.guessTagFromCustomType() == "!!float" {
if raw, ok := jsonFloatLiteral(o.Value); ok {
buf.WriteString(raw)
return buf.Bytes(), nil
}
}
value, err := o.GetValueRep()
if err != nil {
return buf.Bytes(), err
@ -186,85 +177,3 @@ func (o *CandidateNode) MarshalJSON() ([]byte, error) {
return buf.Bytes(), err
}
}
// jsonFloatLiteral returns a JSON-shaped representation of a YAML !!float scalar
// value, preserving the original textual form (e.g. "50.0" stays "50.0") whenever
// possible. The second return value is false when the value cannot be safely
// rendered as a JSON number (e.g. ".inf", ".nan", or anything that parses to a
// non-finite float); callers should fall back to the normal encoding path in
// that case, which preserves the existing behaviour for those inputs.
func jsonFloatLiteral(raw string) (string, bool) {
if raw == "" {
return "", false
}
f, err := strconv.ParseFloat(raw, 64)
if err != nil {
return "", false
}
if math.IsInf(f, 0) || math.IsNaN(f) {
return "", false
}
if isJSONNumberLiteral(raw) {
return raw, true
}
formatted := strconv.FormatFloat(f, 'f', -1, 64)
if !strings.ContainsAny(formatted, ".eE") {
formatted += ".0"
}
return formatted, true
}
// isJSONNumberLiteral reports whether s is already a valid JSON number literal
// representing a fractional value (i.e. contains a "." or an exponent), so it
// can be emitted verbatim without round-tripping through a float64.
func isJSONNumberLiteral(s string) bool {
if s == "" {
return false
}
i := 0
if s[i] == '-' {
i++
if i == len(s) {
return false
}
}
// integer part: 0 or [1-9][0-9]*
if s[i] == '0' {
i++
} else if s[i] >= '1' && s[i] <= '9' {
for i < len(s) && s[i] >= '0' && s[i] <= '9' {
i++
}
} else {
return false
}
hasFraction := false
if i < len(s) && s[i] == '.' {
hasFraction = true
i++
if i == len(s) || s[i] < '0' || s[i] > '9' {
return false
}
for i < len(s) && s[i] >= '0' && s[i] <= '9' {
i++
}
}
hasExponent := false
if i < len(s) && (s[i] == 'e' || s[i] == 'E') {
hasExponent = true
i++
if i < len(s) && (s[i] == '+' || s[i] == '-') {
i++
}
if i == len(s) || s[i] < '0' || s[i] > '9' {
return false
}
for i < len(s) && s[i] >= '0' && s[i] <= '9' {
i++
}
}
if i != len(s) {
return false
}
return hasFraction || hasExponent
}

View File

@ -106,31 +106,6 @@ func TestCreateScalarNodeScenarios(t *testing.T) {
}
}
type createStringScalarNodeScenario struct {
stringValue string
expectedTag string
}
var createStringScalarNodeScenarios = []createStringScalarNodeScenario{
{
stringValue: "yourKey",
expectedTag: "!!str",
},
{
stringValue: "<<",
expectedTag: "!!merge",
},
}
func TestCreateStringScalarNodeScenarios(t *testing.T) {
for _, tt := range createStringScalarNodeScenarios {
actual := createStringScalarNode(tt.stringValue)
test.AssertResultWithContext(t, tt.stringValue, actual.Value, fmt.Sprintf("Value for: %v", tt.stringValue))
test.AssertResultWithContext(t, tt.expectedTag, actual.Tag, fmt.Sprintf("Tag for: %v", tt.stringValue))
test.AssertResultWithContext(t, ScalarNode, actual.Kind, fmt.Sprintf("Kind for: %v", tt.stringValue))
}
}
func TestGetKeyForMapValue(t *testing.T) {
key := createStringScalarNode("yourKey")
n := CandidateNode{Key: key, Value: "meow", document: 3}

View File

@ -12,20 +12,17 @@ import (
type iniDecoder struct {
reader io.Reader
finished bool // Flag to signal completion of processing
prefs INIPreferences
}
func NewINIDecoder(prefs INIPreferences) Decoder {
func NewINIDecoder() Decoder {
return &iniDecoder{
finished: false, // Initialise the flag as false
prefs: prefs,
}
}
func (dec *iniDecoder) Init(reader io.Reader) error {
// Store the reader for use in Decode
dec.reader = reader
dec.finished = false
return nil
}
@ -42,10 +39,7 @@ func (dec *iniDecoder) Decode() (*CandidateNode, error) {
}
// Parse the INI content
loadOpts := ini.LoadOptions{
PreserveSurroundedQuote: dec.prefs.PreserveSurroundedQuote,
}
cfg, err := ini.LoadSources(loadOpts, content)
cfg, err := ini.Load(content)
if err != nil {
return nil, fmt.Errorf("failed to parse INI content: %w", err)
}

View File

@ -16,11 +16,10 @@ type propertiesDecoder struct {
reader io.Reader
finished bool
d DataTreeNavigator
prefs PropertiesPreferences
}
func NewPropertiesDecoder() Decoder {
return &propertiesDecoder{d: NewDataTreeNavigator(), finished: false, prefs: ConfiguredPropertiesPreferences.Copy()}
return &propertiesDecoder{d: NewDataTreeNavigator(), finished: false}
}
func (dec *propertiesDecoder) Init(reader io.Reader) error {
@ -29,56 +28,20 @@ func (dec *propertiesDecoder) Init(reader io.Reader) error {
return nil
}
func parsePropKey(key string, prefs PropertiesPreferences) []interface{} {
func parsePropKey(key string) []interface{} {
pathStrArray := strings.Split(key, ".")
path := make([]interface{}, 0, len(pathStrArray))
for _, pathStr := range pathStrArray {
path = appendPropKeySegment(path, pathStr, prefs.UseArrayBrackets)
path := make([]interface{}, len(pathStrArray))
for i, pathStr := range pathStrArray {
num, err := strconv.ParseInt(pathStr, 10, 32)
if err == nil {
path[i] = num
} else {
path[i] = pathStr
}
}
return path
}
func appendPropKeySegment(path []interface{}, segment string, useArrayBrackets bool) []interface{} {
if useArrayBrackets && strings.Contains(segment, "[") {
bracketPath, ok := parsePropKeyArrayBracketSegment(segment)
if ok {
return append(path, bracketPath...)
}
}
num, err := strconv.ParseInt(segment, 10, 32)
if err == nil {
return append(path, num)
}
return append(path, segment)
}
func parsePropKeyArrayBracketSegment(segment string) ([]interface{}, bool) {
path := []interface{}{}
bracketIndex := strings.Index(segment, "[")
if bracketIndex > 0 {
path = append(path, segment[:bracketIndex])
}
remaining := segment[bracketIndex:]
for remaining != "" {
if !strings.HasPrefix(remaining, "[") {
return nil, false
}
closingBracket := strings.Index(remaining, "]")
if closingBracket < 0 {
return nil, false
}
arrayIndex, err := strconv.ParseInt(remaining[1:closingBracket], 10, 32)
if err != nil {
return nil, false
}
path = append(path, arrayIndex)
remaining = remaining[closingBracket+1:]
}
return path, true
}
func (dec *propertiesDecoder) processComment(c string) string {
if c == "" {
return ""
@ -112,7 +75,7 @@ func (dec *propertiesDecoder) applyPropertyComments(context Context, path []inte
func (dec *propertiesDecoder) applyProperty(context Context, properties *properties.Properties, key string) error {
value, _ := properties.Get(key)
path := parsePropKey(key, dec.prefs)
path := parsePropKey(key)
propertyComments := properties.GetComments(key)
if len(propertyComments) > 0 {

View File

@ -2,7 +2,7 @@
Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names).
`yq` supports merge keys (like `<<: *blah`) from YAML 1.1. These are no longer part of the YAML 1.2 standard, but remain common in practice. Plain `<<:` keys are recognised as merge keys and round-trip as `<<:` without an explicit `!!merge` tag. When the source uses an explicit `!!merge` tag, that is preserved on output. Internally, when `yq` synthesises a `<<` map key (for example during merge operations), it tags the key as `!!merge` rather than `!!str`.
`yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.
## NOTE --yaml-fix-merge-anchor-to-spec flag
@ -32,7 +32,7 @@ Given a sample.yml file of:
r: 10
- &SMALL
r: 1
- <<: *CENTRE
- !!merge <<: *CENTRE
r: 10
```
then
@ -213,10 +213,10 @@ item_value: &item_value
value: true
thingOne:
name: item_1
<<: *item_value
!!merge <<: *item_value
thingTwo:
name: item_2
<<: *item_value
!!merge <<: *item_value
```
then
```bash
@ -231,7 +231,7 @@ thingOne:
value: false
thingTwo:
name: item_2
<<: *item_value
!!merge <<: *item_value
```
## LEGACY: Explode with merge anchors
@ -249,13 +249,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@ -298,7 +298,7 @@ Given a sample.yml file of:
r: 10
- &SMALL
r: 1
- <<:
- !!merge <<:
- *CENTRE
- *BIG
```
@ -328,7 +328,7 @@ Given a sample.yml file of:
r: 10
- &SMALL
r: 1
- <<:
- !!merge <<:
- *BIG
- *LEFT
- *SMALL
@ -361,13 +361,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@ -411,7 +411,7 @@ Given a sample.yml file of:
r: 10
- &SMALL
r: 1
- <<:
- !!merge <<:
- *CENTRE
- *BIG
```
@ -442,7 +442,7 @@ Given a sample.yml file of:
r: 10
- &SMALL
r: 1
- <<:
- !!merge <<:
- *BIG
- *LEFT
- *SMALL
@ -467,7 +467,7 @@ Given a sample.yml file of:
```yaml
a:
b: &b 42
<<:
!!merge <<:
c: *b
```
then

View File

@ -55,7 +55,7 @@ yq '.a = .a / 0 | .b = .b / 0' sample.yml
```
will output
```yaml
a: +Inf
b: -Inf
a: !!float +Inf
b: !!float -Inf
```

View File

@ -2,7 +2,7 @@
Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names).
`yq` supports merge keys (like `<<: *blah`) from YAML 1.1. These are no longer part of the YAML 1.2 standard, but remain common in practice. Plain `<<:` keys are recognised as merge keys and round-trip as `<<:` without an explicit `!!merge` tag. When the source uses an explicit `!!merge` tag, that is preserved on output. Internally, when `yq` synthesises a `<<` map key (for example during merge operations), it tags the key as `!!merge` rather than `!!str`.
`yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.
## NOTE --yaml-fix-merge-anchor-to-spec flag

View File

@ -34,7 +34,7 @@ yq '.a = .a % .b' sample.yml
```
will output
```yaml
a: 2
a: !!float 2
b: 2.5
```
@ -69,7 +69,7 @@ yq '.a = .a % .b' sample.yml
```
will output
```yaml
a: NaN
a: !!float NaN
b: 0
```

View File

@ -471,13 +471,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@ -487,7 +487,7 @@ yq '.foobar * .foobarList' sample.yml
will output
```yaml
c: foobarList_c
<<:
!!merge <<:
- *foo
- *bar
thing: foobar_thing

View File

@ -131,13 +131,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@ -147,7 +147,7 @@ yq '.foobar | [..]' sample.yml
will output
```yaml
- c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
- foobar_c
- *foo

View File

@ -51,7 +51,7 @@ country: Australia
```
then
```bash
yq --security-enable-system-operator '.country = system("/bin/echo"; "test")' sample.yml
yq --security-enable-system-operator '.country = system("/usr/bin/echo"; "test")' sample.yml
```
will output
```yaml
@ -67,7 +67,7 @@ a: hello
```
then
```bash
yq --security-enable-system-operator '.a = system("/bin/echo")' sample.yml
yq --security-enable-system-operator '.a = system("/usr/bin/echo")' sample.yml
```
will output
```yaml

View File

@ -294,13 +294,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@ -325,13 +325,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@ -376,13 +376,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@ -409,13 +409,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@ -442,13 +442,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@ -477,13 +477,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@ -513,13 +513,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@ -546,13 +546,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@ -579,13 +579,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then
@ -614,13 +614,13 @@ bar: &bar
c: bar_c
foobarList:
b: foobarList_b
<<:
!!merge <<:
- *foo
- *bar
c: foobarList_c
foobar:
c: foobar_c
<<: *foo
!!merge <<: *foo
thing: foobar_thing
```
then

View File

@ -125,22 +125,6 @@ will output
{"whatever":"cat"}
```
## Encode json: preserve floats with trailing zero
Whole-number floats keep their decimal point so downstream consumers see a JSON number with a fractional part (matches Go's encoding/json, Python's json, and jq).
Given a sample.yml file of:
```yaml
percentiles: [50.0, 95.0, 99.0, 99.9]
```
then
```bash
yq -o=json -I=0 '.' sample.yml
```
will output
```json
{"percentiles":[50.0,95.0,99.0,99.9]}
```
## Roundtrip JSON Lines / NDJSON
Given a sample.json file of:
```json

View File

@ -1,4 +1,4 @@
# TOML
Encode and decode to and from TOML.
Decode from TOML. Note that `yq` does not yet support outputting in TOML format (and therefore it cannot roundtrip)

View File

@ -1,6 +1,6 @@
# TOML
Encode and decode to and from TOML.
Decode from TOML. Note that `yq` does not yet support outputting in TOML format (and therefore it cannot roundtrip)
## Parse: Simple

View File

@ -132,23 +132,12 @@ func (te *tomlEncoder) encodeRootMapping(w io.Writer, node *CandidateNode) error
}
}
// Preserve existing order by iterating Content
for i := 0; i < len(node.Content); i += 2 {
keyNode := node.Content[i]
valNode := node.Content[i+1]
if isTomlAttribute(valNode) {
if err := te.encodeTopLevelEntry(w, []string{keyNode.Value}, valNode); err != nil {
return err
}
}
}
for i := 0; i < len(node.Content); i += 2 {
keyNode := node.Content[i]
valNode := node.Content[i+1]
if !isTomlAttribute(valNode) {
if err := te.encodeTopLevelEntry(w, []string{keyNode.Value}, valNode); err != nil {
return err
}
if err := te.encodeTopLevelEntry(w, []string{keyNode.Value}, valNode); err != nil {
return err
}
}
return nil
@ -181,13 +170,8 @@ func (te *tomlEncoder) encodeTopLevelEntry(w io.Writer, path []string, node *Can
if allMaps {
key := path[len(path)-1]
quotedKey := tomlKey(key)
if te.wroteRootAttr {
if _, err := w.Write([]byte("\n")); err != nil {
return err
}
te.wroteRootAttr = false
}
for _, it := range node.Content {
// [[key]] then body
if _, err := w.Write([]byte("[[" + quotedKey + "]]\n")); err != nil {
return err
}
@ -200,12 +184,9 @@ func (te *tomlEncoder) encodeTopLevelEntry(w io.Writer, path []string, node *Can
// Regular array attribute
return te.writeArrayAttribute(w, path[len(path)-1], node)
case MappingNode:
// Use inline table syntax only for nodes explicitly marked as TOML inline tables.
// YAML flow-style mappings are not treated as inline tables; the FlowStyle attribute
// is a YAML-specific rendering hint and should not affect TOML output. This ensures
// that auto-detected JSON input (parsed as YAML flow style) produces readable table
// sections, consistent with explicitly parsed JSON input.
if node.EncodeHint == EncodeHintInline {
// Use inline table syntax for nodes explicitly marked as TOML inline tables
// or YAML flow mappings. All other mappings become readable TOML table sections.
if node.EncodeHint == EncodeHintInline || node.Style&FlowStyle != 0 {
return te.writeInlineTableAttribute(w, path[len(path)-1], node)
}
return te.encodeSeparateMapping(w, path, node)
@ -214,30 +195,7 @@ func (te *tomlEncoder) encodeTopLevelEntry(w io.Writer, path []string, node *Can
}
}
func isTomlArrayOfTables(seq *CandidateNode) bool {
if len(seq.Content) == 0 {
return false
}
for _, it := range seq.Content {
if it.Kind != MappingNode || it.EncodeHint == EncodeHintInline {
return false
}
}
return true
}
func isTomlAttribute(node *CandidateNode) bool {
if node.Kind == ScalarNode {
return true
}
return node.Kind == SequenceNode && !isTomlArrayOfTables(node)
}
func (te *tomlEncoder) writeAttribute(w io.Writer, key string, value *CandidateNode) error {
if value.Tag == "!!null" {
return nil
}
te.wroteRootAttr = true // Mark that we wrote a root attribute
// Write head comment before the attribute
@ -433,9 +391,6 @@ func (te *tomlEncoder) mappingToInlineTable(m *CandidateNode) (string, error) {
v := m.Content[i+1]
switch v.Kind {
case ScalarNode:
if v.Tag == "!!null" {
continue
}
parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), te.formatScalar(v)))
case SequenceNode:
// inline array in inline table
@ -498,16 +453,24 @@ func (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *Cand
hasAttrs := false
for i := 0; i < len(m.Content); i += 2 {
v := m.Content[i+1]
if v.Kind == ScalarNode && v.Tag != "!!null" {
if v.Kind == ScalarNode {
hasAttrs = true
break
}
if v.Kind == MappingNode && v.EncodeHint == EncodeHintInline {
if v.Kind == MappingNode && (v.EncodeHint == EncodeHintInline || v.Style&FlowStyle != 0) {
hasAttrs = true
break
}
if v.Kind == SequenceNode {
if !isTomlArrayOfTables(v) {
// Check if it's NOT an array of tables
allMaps := true
for _, it := range v.Content {
if it.Kind != MappingNode {
allMaps = false
break
}
}
if !allMaps {
hasAttrs = true
break
}
@ -537,14 +500,15 @@ func (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *Cand
}
case SequenceNode:
// If sequence of maps, emit [[path.k]] per element
if isTomlArrayOfTables(v) {
key := tomlDottedKey(append(append([]string{}, path...), k))
if te.wroteRootAttr {
if _, err := w.Write([]byte("\n")); err != nil {
return err
}
te.wroteRootAttr = false
allMaps := true
for _, it := range v.Content {
if it.Kind != MappingNode {
allMaps = false
break
}
}
if allMaps {
key := tomlDottedKey(append(append([]string{}, path...), k))
for _, it := range v.Content {
if _, err := w.Write([]byte("[[" + key + "]]\n")); err != nil {
return err
@ -571,7 +535,7 @@ func (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *Cand
// encodeMappingBodyWithPath encodes attributes and nested arrays of tables using full dotted path context
func (te *tomlEncoder) encodeMappingBodyWithPath(w io.Writer, path []string, m *CandidateNode) error {
// First, attributes (scalars, inline mappings, and non-map arrays)
// First, attributes (scalars and non-map arrays)
for i := 0; i < len(m.Content); i += 2 {
k := m.Content[i].Value
v := m.Content[i+1]
@ -580,14 +544,15 @@ func (te *tomlEncoder) encodeMappingBodyWithPath(w io.Writer, path []string, m *
if err := te.writeAttribute(w, k, v); err != nil {
return err
}
case MappingNode:
if v.EncodeHint == EncodeHintInline {
if err := te.writeInlineTableAttribute(w, k, v); err != nil {
return err
case SequenceNode:
allMaps := true
for _, it := range v.Content {
if it.Kind != MappingNode {
allMaps = false
break
}
}
case SequenceNode:
if !isTomlArrayOfTables(v) {
if !allMaps {
if err := te.writeArrayAttribute(w, k, v); err != nil {
return err
}
@ -600,7 +565,14 @@ func (te *tomlEncoder) encodeMappingBodyWithPath(w io.Writer, path []string, m *
k := m.Content[i].Value
v := m.Content[i+1]
if v.Kind == SequenceNode {
if isTomlArrayOfTables(v) {
allMaps := true
for _, it := range v.Content {
if it.Kind != MappingNode {
allMaps = false
break
}
}
if allMaps {
dotted := tomlDottedKey(append(append([]string{}, path...), k))
for _, it := range v.Content {
if _, err := w.Write([]byte("[[" + dotted + "]]\n")); err != nil {
@ -614,15 +586,21 @@ func (te *tomlEncoder) encodeMappingBodyWithPath(w io.Writer, path []string, m *
}
}
// Finally, child mappings: inline-hint ones were emitted above as attributes,
// Finally, child mappings: inline-hint or flow-style ones become inline table attributes,
// while all others are emitted as separate sub-table sections.
for i := 0; i < len(m.Content); i += 2 {
k := m.Content[i].Value
v := m.Content[i+1]
if v.Kind == MappingNode && v.EncodeHint != EncodeHintInline {
subPath := append(append([]string{}, path...), k)
if err := te.encodeSeparateMapping(w, subPath, v); err != nil {
return err
if v.Kind == MappingNode {
if v.EncodeHint == EncodeHintInline || v.Style&FlowStyle != 0 {
if err := te.writeInlineTableAttribute(w, k, v); err != nil {
return err
}
} else {
subPath := append(append([]string{}, path...), k)
if err := te.encodeSeparateMapping(w, subPath, v); err != nil {
return err
}
}
}
}
@ -677,7 +655,7 @@ func (te *tomlEncoder) colorizeToml(input []byte) []byte {
// Table sections - [section] or [[array]]
// Only treat '[' as a table section if it appears at the start of the line
// (possibly after whitespace). This avoids incorrectly colouring inline arrays like
// (possibly after whitespace). This avoids mis-colouring inline arrays like
// "ports = [8000, 8001]" as table sections.
if ch == '[' {
isSectionHeader := true

View File

@ -40,7 +40,7 @@ func tryRemoveTempFile(filename string) {
// thanks https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang
func copyFileContents(src, dst string) (err error) {
// ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory,
// and ensuring that it's not possible to give a path to a file outside that directory.
// and ensuring that it's not possible to give a path to a file outside thar directory.
in, err := os.Open(src) // #nosec
if err != nil {

View File

@ -90,7 +90,7 @@ var LuaFormat = &Format{"lua", []string{"l"},
var INIFormat = &Format{"ini", []string{"i"},
func() Encoder { return NewINIEncoder() },
func() Decoder { return NewINIDecoder(ConfiguredINIPreferences) },
func() Decoder { return NewINIDecoder() },
}
var Formats = []*Format{

View File

@ -230,7 +230,8 @@ var goccyYamlFormatScenarios = []formatScenario{
description: "merge anchor",
skipDoc: true,
input: "a: &remember\n c: mike\nb:\n <<: *remember",
expected: "a: &remember\n c: mike\nb:\n <<: *remember\n",
// fine to have !!merge as that's what the current impl does
expected: "a: &remember\n c: mike\nb:\n !!merge <<: *remember\n",
},
{
description: "custom tag",

View File

@ -1,21 +1,18 @@
package yqlib
type INIPreferences struct {
ColorsEnabled bool
PreserveSurroundedQuote bool
ColorsEnabled bool
}
func NewDefaultINIPreferences() INIPreferences {
return INIPreferences{
ColorsEnabled: false,
PreserveSurroundedQuote: false,
ColorsEnabled: false,
}
}
func (p *INIPreferences) Copy() INIPreferences {
return INIPreferences{
ColorsEnabled: p.ColorsEnabled,
PreserveSurroundedQuote: p.PreserveSurroundedQuote,
ColorsEnabled: p.ColorsEnabled,
}
}

View File

@ -5,7 +5,6 @@ package yqlib
import (
"bufio"
"fmt"
"strings"
"testing"
"github.com/mikefarah/yq/v4/test"
@ -23,16 +22,6 @@ const expectedSimpleINIYaml = `section:
key: value
`
const quotedINIInput = `[section]
color_theme = "Default"
theme_background = "False"
`
const expectedQuotedINIOutput = `[section]
color_theme = "Default"
theme_background = "False"
`
var iniScenarios = []formatScenario{
{
description: "Parse INI: simple",
@ -60,22 +49,6 @@ var iniScenarios = []formatScenario{
},
}
// iniPreserveQuotesPrefs returns INIPreferences with PreserveSurroundedQuote enabled.
func iniPreserveQuotesPrefs() INIPreferences {
prefs := NewDefaultINIPreferences()
prefs.PreserveSurroundedQuote = true
return prefs
}
var iniPreserveQuotesScenarios = []formatScenario{
{
description: "Roundtrip INI: preserve quotes",
input: quotedINIInput,
expected: expectedQuotedINIOutput,
scenarioType: "roundtrip",
},
}
func documentRoundtripINIScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
@ -97,7 +70,7 @@ func documentRoundtripINIScenario(w *bufio.Writer, s formatScenario) {
}
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```ini\n%v```\n\n", mustProcessFormatScenario(s, NewINIDecoder(NewDefaultINIPreferences()), NewINIEncoder())))
writeOrPanic(w, fmt.Sprintf("```ini\n%v```\n\n", mustProcessFormatScenario(s, NewINIDecoder(), NewINIEncoder())))
}
func documentDecodeINIScenario(w *bufio.Writer, s formatScenario) {
@ -121,7 +94,7 @@ func documentDecodeINIScenario(w *bufio.Writer, s formatScenario) {
}
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewINIDecoder(NewDefaultINIPreferences()), NewYamlEncoder(ConfiguredYamlPreferences))))
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewINIDecoder(), NewYamlEncoder(ConfiguredYamlPreferences))))
}
func testINIScenario(t *testing.T, s formatScenario) {
@ -129,11 +102,11 @@ func testINIScenario(t *testing.T, s formatScenario) {
case "encode":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewINIEncoder()), s.description)
case "decode":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewINIDecoder(NewDefaultINIPreferences()), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewINIDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
case "roundtrip":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewINIDecoder(NewDefaultINIPreferences()), NewINIEncoder()), s.description)
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewINIDecoder(), NewINIEncoder()), s.description)
case "decode-error":
result, err := processFormatScenario(s, NewINIDecoder(NewDefaultINIPreferences()), NewINIEncoder())
result, err := processFormatScenario(s, NewINIDecoder(), NewINIEncoder())
if err == nil {
t.Errorf("Expected error '%v' but it worked: %v", s.expectedError, result)
} else {
@ -202,21 +175,6 @@ func documentDecodeErrorINIScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("```\n%v\n```\n\n", s.expectedError))
}
func TestINIDecoderInitResetsFinished(t *testing.T) {
decoder := NewINIDecoder(NewDefaultINIPreferences())
firstDocuments, err := readDocuments(strings.NewReader("[first]\nkey = value\n"), "first.ini", 0, decoder)
if err != nil {
t.Fatal(err)
}
test.AssertResult(t, 1, firstDocuments.Len())
secondDocuments, err := readDocuments(strings.NewReader("[second]\nkey = value\n"), "second.ini", 1, decoder)
if err != nil {
t.Fatal(err)
}
test.AssertResult(t, 1, secondDocuments.Len())
}
func TestINIScenarios(t *testing.T) {
for _, tt := range iniScenarios {
testINIScenario(t, tt)
@ -227,19 +185,3 @@ func TestINIScenarios(t *testing.T) {
}
documentScenarios(t, "usage", "convert", genericScenarios, documentINIScenario)
}
func testINIPreserveQuotesScenario(t *testing.T, s formatScenario) {
prefs := iniPreserveQuotesPrefs()
switch s.scenarioType {
case "roundtrip":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewINIDecoder(prefs), NewINIEncoder()), s.description)
default:
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
}
}
func TestINIPreserveQuotesScenarios(t *testing.T) {
for _, tt := range iniPreserveQuotesScenarios {
testINIPreserveQuotesScenario(t, tt)
}
}

View File

@ -220,54 +220,6 @@ var jsonScenarios = []formatScenario{
expected: "{\"stuff\":\"cool\"}\n{\"whatever\":\"cat\"}\n",
scenarioType: "encode",
},
{
description: "Encode json: preserve floats with trailing zero",
subdescription: "Whole-number floats keep their decimal point so downstream consumers see a JSON number with a fractional part (matches Go's encoding/json, Python's json, and jq).",
input: `percentiles: [50.0, 95.0, 99.0, 99.9]`,
indent: 0,
expected: "{\"percentiles\":[50.0,95.0,99.0,99.9]}\n",
scenarioType: "encode",
},
{
description: "Encode json: ints stay ints",
skipDoc: true,
input: `a: 50`,
indent: 0,
expected: "{\"a\":50}\n",
scenarioType: "encode",
},
{
description: "Encode json: !!float tagged whole number gets .0",
skipDoc: true,
input: `a: !!float 5`,
indent: 0,
expected: "{\"a\":5.0}\n",
scenarioType: "encode",
},
{
description: "Encode json: scientific notation float preserved",
skipDoc: true,
input: `a: 1.5e-3`,
indent: 0,
expected: "{\"a\":1.5e-3}\n",
scenarioType: "encode",
},
{
description: "Encode json: negative float preserved",
skipDoc: true,
input: `a: -7.0`,
indent: 0,
expected: "{\"a\":-7.0}\n",
scenarioType: "encode",
},
{
description: "Encode json: mixed int and float array",
skipDoc: true,
input: `a: [1, 2.0, 3, 4.5]`,
indent: 0,
expected: "{\"a\":[1,2.0,3,4.5]}\n",
scenarioType: "encode",
},
{
description: "Roundtrip JSON Lines / NDJSON",
input: sampleNdJson,

View File

@ -24,7 +24,7 @@ type parseSnippetScenario struct {
var parseSnippetScenarios = []parseSnippetScenario{
{
snippet: ":",
expectedError: "go-yaml load error in parser (while parsing a block mapping) at L1.C1: did not find expected key",
expectedError: "yaml: while parsing a block mapping at <unknown position>: did not find expected key",
},
{
snippet: "",

View File

@ -2,7 +2,7 @@
package yqlib
func NewINIDecoder(prefs INIPreferences) Decoder {
func NewINIDecoder() Decoder {
return nil
}

View File

@ -170,10 +170,6 @@ func fixedReconstructAliasedMap(node *CandidateNode) error {
if mergeNodeSeq.Kind == AliasNode {
mergeNodeSeq = mergeNodeSeq.Alias
}
mergeNodeSeq = mergeNodeSeq.Copy()
if err := explodeNode(mergeNodeSeq, Context{}); err != nil {
return err
}
if mergeNodeSeq.Kind != MappingNode {
return fmt.Errorf("can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing %v", mergeNodeSeq.Tag)
}
@ -183,7 +179,12 @@ func fixedReconstructAliasedMap(node *CandidateNode) error {
})
for _, item := range itemsToAdd {
newContent = append(newContent, item.Copy())
// copy to ensure exploding doesn't modify the original node
itemCopy := item.Copy()
if err := explodeNode(itemCopy, Context{}); err != nil {
return err
}
newContent = append(newContent, itemCopy)
}
}
}

View File

@ -31,7 +31,7 @@ thingOne:
value: false
thingTwo:
name: item_2
<<: *item_value
!!merge <<: *item_value
`
var explodeMergeAnchorsFixedExpected = `D0, P[], (!!map)::foo:
@ -198,15 +198,6 @@ var fixedAnchorOperatorScenarios = []expressionScenario{
"D0, P[], (!!map)::{a: 42}\n",
},
},
{
skipDoc: true,
description: "Nested merge anchor with inline map",
document: `{<<: {<<: {a: 42}}}`,
expression: `explode(.)`,
expected: []string{
"D0, P[], (!!map)::{a: 42}\n",
},
},
{
skipDoc: true,
description: "Merge anchor with sequence with inline map",
@ -288,63 +279,7 @@ var badAnchorOperatorScenarios = []expressionScenario{
},
}
var mixedMergeTagStyleDocument = `
constants:
errorResponse: &errorResponse
status: 200
endpoints:
- condition: true
!!merge <<: *errorResponse
- condition: false
<<: *errorResponse
other:
!!merge <<: *errorResponse
somethingElse:
<<: *errorResponse
`
var mixedMergeTagStyleExplodedDocument = `
constants:
errorResponse:
status: 200
endpoints:
- condition: true
status: 200
- condition: false
status: 200
other:
status: 200
somethingElse:
status: 200
`
var anchorOperatorScenarios = []expressionScenario{
{
// mergeObjects previously skipped all !!merge-tagged nodes. Since !!merge only appears on
// << map keys, this meant applyAssignment was never called for the << key. It was later
// autocreated by createStringScalarNode("<<") with tag !!str, silently dropping !!merge.
// DontFollowAlias:true already prevents aliases being followed, so the skip was redundant.
// Old (buggy) output: "D0, P[], (!!map)::base: &base\n x: 1\ndest:\n <<: *base\n"
skipDoc: true,
description: "direct *+ preserves explicit !!merge tag on << key (regression for issue 2677)",
document: "base: &base\n x: 1\ndest:\n !!merge <<: *base\n",
expression: `. as $d | {} *+ $d`,
expected: []string{"D0, P[], (!!map)::base: &base\n x: 1\ndest:\n !!merge <<: *base\n"},
},
{
skipDoc: true,
description: "explicit !!merge tag on << key is preserved through ireduce merge",
document: mixedMergeTagStyleDocument,
expression: `. as $item ireduce ({}; . *+ $item)`,
expected: []string{"D0, P[], (!!map)::" + mixedMergeTagStyleDocument},
},
{
skipDoc: true,
description: "explode expands << merge keys regardless of explicit tag style (!!merge or plain)",
document: mixedMergeTagStyleDocument,
expression: `explode(.)`,
expected: []string{"D0, P[], (!!map)::" + mixedMergeTagStyleExplodedDocument},
},
{
skipDoc: true,
description: "merge anchor to alias alias",

View File

@ -45,7 +45,7 @@ var divideOperatorScenarios = []expressionScenario{
document: `{a: 1, b: -1}`,
expression: `.a = .a / 0 | .b = .b / 0`,
expected: []string{
"D0, P[], (!!map)::{a: +Inf, b: -Inf}\n",
"D0, P[], (!!map)::{a: !!float +Inf, b: !!float -Inf}\n",
},
},
{

View File

@ -77,7 +77,7 @@ var loadScenarios = []expressionScenario{
document: `{something: {file: "thing.yml"}, over: {here: [{file: "thing.yml"}]}}`,
expression: `(.. | select(has("file"))) |= load("../../examples/" + .file)`,
expected: []string{
"D0, P[], (!!map)::{something: {a: apple is included, b: cool.}, over: {here: [{a: apple is included,\n b: cool.}]}}\n",
"D0, P[], (!!map)::{something: {a: apple is included, b: cool.}, over: {here: [{a: apple is included, b: cool.}]}}\n",
},
},
{

View File

@ -37,7 +37,7 @@ var moduloOperatorScenarios = []expressionScenario{
document: `{a: 12, b: 2.5}`,
expression: `.a = .a % .b`,
expected: []string{
"D0, P[], (!!map)::{a: 2, b: 2.5}\n",
"D0, P[], (!!map)::{a: !!float 2, b: 2.5}\n",
},
},
{
@ -53,7 +53,7 @@ var moduloOperatorScenarios = []expressionScenario{
document: `{a: 1.1, b: 0}`,
expression: `.a = .a % .b`,
expected: []string{
"D0, P[], (!!map)::{a: NaN, b: 0}\n",
"D0, P[], (!!map)::{a: !!float NaN, b: 0}\n",
},
},
{
@ -70,7 +70,7 @@ var moduloOperatorScenarios = []expressionScenario{
document: "a: 2\nb: !goat 2.3",
expression: `.a = .a % .b`,
expected: []string{
"D0, P[], (!!map)::a: 2\nb: !goat 2.3\n",
"D0, P[], (!!map)::a: !!float 2\nb: !goat 2.3\n",
},
},
{

View File

@ -189,6 +189,10 @@ func mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs
log.Debugf("going to applied assignment to LHS: %v with RHS: %v", NodeToString(lhs), NodeToString(candidate))
if candidate.Tag == "!!merge" {
continue
}
err := applyAssignment(d, context, pathIndexToStartFrom, lhs, candidate, preferences)
if err != nil {
return nil, err

View File

@ -2,7 +2,6 @@ package yqlib
import (
"fmt"
"math/bits"
"strings"
"testing"
)
@ -121,7 +120,7 @@ var multiplyOperatorScenarios = []expressionScenario{
document: mergeArrayWithAnchors,
expression: `. * .`,
expected: []string{
"D0, P[], (!!map)::sample:\n - &a\n - <<: *a\n",
"D0, P[], (!!map)::sample:\n - &a\n - !!merge <<: *a\n",
},
},
{
@ -514,7 +513,7 @@ var multiplyOperatorScenarios = []expressionScenario{
environmentVariables: map[string]string{"originalPath": ".myArray", "otherPath": ".newArray", "idPath": ".a"},
expression: mergeExpression,
expected: []string{
"D0, P[], (!!map)::{myArray: [{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC},\n {a: dingo, c: dingoC}], something: else}\n",
"D0, P[], (!!map)::{myArray: [{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC}, {a: dingo, c: dingoC}], something: else}\n",
},
},
{
@ -546,7 +545,7 @@ var multiplyOperatorScenarios = []expressionScenario{
document: mergeDocSample,
expression: `.foobar * .foobarList`,
expected: []string{
"D0, P[foobar], (!!map)::c: foobarList_c\n<<: [*foo, *bar]\nthing: foobar_thing\nb: foobarList_b\n",
"D0, P[foobar], (!!map)::c: foobarList_c\n!!merge <<: [*foo, *bar]\nthing: foobar_thing\nb: foobarList_b\n",
},
},
{
@ -554,7 +553,7 @@ var multiplyOperatorScenarios = []expressionScenario{
document: document,
expression: `.b * .c`,
expected: []string{
"D0, P[b], (!!map)::{name: dog, <<: *cat}\n",
"D0, P[b], (!!map)::{name: dog, \"<<\": *cat}\n",
},
},
{
@ -581,7 +580,7 @@ var multiplyOperatorScenarios = []expressionScenario{
document: "a: 2\nb: !goat 3.5",
expression: ".a = .a * .b",
expected: []string{
"D0, P[], (!!map)::a: 7\nb: !goat 3.5\n",
"D0, P[], (!!map)::a: !!float 7\nb: !goat 3.5\n",
},
},
{
@ -708,13 +707,11 @@ var multiplyOperatorScenarios = []expressionScenario{
expectedError: "result of repeating string (1 bytes) by 99999999 would exceed 10485760 bytes",
},
{
// Pick a count whose product with len("ab") overflows int on
// any architecture: 2^30 on 32-bit, 2^62 on 64-bit. Doubling
// either yields MaxInt+1, which wraps to MinInt and bypasses
// a naive len*count guard.
// The size guard must not overflow: len * count can wrap to
// a negative or small value on 64-bit, bypassing the check.
skipDoc: true,
expression: fmt.Sprintf(`"ab" * %d`, 1<<(bits.UintSize-2)),
expectedError: fmt.Sprintf("result of repeating string (2 bytes) by %d would exceed 10485760 bytes", 1<<(bits.UintSize-2)),
expression: `"ab" * 4611686018427387904`,
expectedError: "result of repeating string (2 bytes) by 4611686018427387904 would exceed 10485760 bytes",
},
}

View File

@ -187,7 +187,7 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
document: mergeDocSample,
expression: `.foobar | [..]`,
expected: []string{
"D0, P[foobar], (!!seq)::- c: foobar_c\n <<: *foo\n thing: foobar_thing\n- foobar_c\n- *foo\n- foobar_thing\n",
"D0, P[foobar], (!!seq)::- c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n- foobar_c\n- *foo\n- foobar_thing\n",
},
},
{
@ -195,7 +195,7 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
document: mergeDocSample,
expression: `.foobar | [...]`,
expected: []string{
"D0, P[foobar], (!!seq)::- c: foobar_c\n <<: *foo\n thing: foobar_thing\n- c\n- foobar_c\n- <<\n- *foo\n- thing\n- foobar_thing\n",
"D0, P[foobar], (!!seq)::- c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n- c\n- foobar_c\n- !!merge <<\n- *foo\n- thing\n- foobar_thing\n",
},
},
{
@ -203,7 +203,7 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
document: mergeDocSample,
expression: `.foobarList | ..`,
expected: []string{
"D0, P[foobarList], (!!map)::b: foobarList_b\n<<: [*foo, *bar]\nc: foobarList_c\n",
"D0, P[foobarList], (!!map)::b: foobarList_b\n!!merge <<: [*foo, *bar]\nc: foobarList_c\n",
"D0, P[foobarList b], (!!str)::foobarList_b\n",
"D0, P[foobarList <<], (!!seq)::[*foo, *bar]\n",
"D0, P[foobarList << 0], (alias)::*foo\n",
@ -216,7 +216,7 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
document: mergeDocSample,
expression: `.foobarList | ...`,
expected: []string{
"D0, P[foobarList], (!!map)::b: foobarList_b\n<<: [*foo, *bar]\nc: foobarList_c\n",
"D0, P[foobarList], (!!map)::b: foobarList_b\n!!merge <<: [*foo, *bar]\nc: foobarList_c\n",
"D0, P[foobarList b], (!!str)::b\n",
"D0, P[foobarList b], (!!str)::foobarList_b\n",
"D0, P[foobarList <<], (!!merge)::<<\n",

View File

@ -98,7 +98,7 @@ var selectOperatorScenarios = []expressionScenario{
document: `[{animal: cat, legs: {cool: true}}, {animal: fish}]`,
expression: `(.[] | select(.legs.cool == true).canWalk) = true | (.[] | .alive.things) = "yes"`,
expected: []string{
"D0, P[], (!!seq)::[{animal: cat, legs: {cool: true}, canWalk: true, alive: {things: yes}}, {animal: fish,\n alive: {things: yes}}]\n",
"D0, P[], (!!seq)::[{animal: cat, legs: {cool: true}, canWalk: true, alive: {things: yes}}, {animal: fish, alive: {things: yes}}]\n",
},
},
{

View File

@ -37,7 +37,7 @@ var sortKeysOperatorScenarios = []expressionScenario{
document: `{bParent: {c: dog, array: [3,1,2]}, aParent: {z: donkey, x: [{c: yum, b: delish}, {b: ew, a: apple}]}}`,
expression: `sort_keys(..)`,
expected: []string{
"D0, P[], (!!map)::{aParent: {x: [{b: delish, c: yum}, {a: apple, b: ew}], z: donkey}, bParent: {array: [\n 3, 1, 2], c: dog}}\n",
"D0, P[], (!!map)::{aParent: {x: [{b: delish, c: yum}, {a: apple, b: ew}], z: donkey}, bParent: {array: [3, 1, 2], c: dog}}\n",
},
},
}

View File

@ -50,7 +50,7 @@ var styleOperatorScenarios = []expressionScenario{
document: "bing: &foo {x: z}\na:\n c: cat\n <<: [*foo]",
expression: `(... | select(tag=="!!str")) style="single"`,
expected: []string{
"D0, P[], (!!map)::'bing': &foo {'x': 'z'}\n'a':\n 'c': 'cat'\n <<: [*foo]\n",
"D0, P[], (!!map)::'bing': &foo {'x': 'z'}\n'a':\n 'c': 'cat'\n !!merge <<: [*foo]\n",
},
},
{

View File

@ -494,7 +494,7 @@ var traversePathOperatorScenarios = []expressionScenario{
document: mergeDocSample,
expression: `.foobar`,
expected: []string{
"D0, P[foobar], (!!map)::c: foobar_c\n<<: *foo\nthing: foobar_thing\n",
"D0, P[foobar], (!!map)::c: foobar_c\n!!merge <<: *foo\nthing: foobar_thing\n",
},
},
{
@ -518,7 +518,7 @@ var traversePathOperatorScenarios = []expressionScenario{
document: mergeDocSample,
expression: `.foobarList`,
expected: []string{
"D0, P[foobarList], (!!map)::b: foobarList_b\n<<: [*foo, *bar]\nc: foobarList_c\n",
"D0, P[foobarList], (!!map)::b: foobarList_b\n!!merge <<: [*foo, *bar]\nc: foobarList_c\n",
},
},
{

View File

@ -202,37 +202,6 @@ var propertyScenarios = []formatScenario{
expected: expectedDecodedYaml,
scenarioType: "decode",
},
{
skipDoc: true,
description: "Decode properties with array brackets",
input: `user.credentials[0].username=user1
user.credentials[0].password=$2b$08$...
user.credentials[1].username=user2
user.credentials[1].password=$2b$08$...
user.credentials[2].username=user3
user.credentials[2].password=$2b$10$...`,
expected: `user:
credentials:
- username: user1
password: $2b$08$...
- username: user2
password: $2b$08$...
- username: user3
password: $2b$10$...
`,
scenarioType: "decode-array-brackets",
},
{
skipDoc: true,
description: "Decode properties with nested array brackets",
input: `user.clowns[0][1] = "cool"`,
expected: `user:
clowns:
- - null
- '"cool"'
`,
scenarioType: "decode-array-brackets",
},
{
skipDoc: true,
@ -473,12 +442,6 @@ func TestPropertyScenarios(t *testing.T) {
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewPropertiesEncoder(ConfiguredPropertiesPreferences)), s.description)
case "decode":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
case "decode-array-brackets":
previousPreferences := ConfiguredPropertiesPreferences.Copy()
ConfiguredPropertiesPreferences.UseArrayBrackets = true
actual := mustProcessFormatScenario(s, NewPropertiesDecoder(), NewYamlEncoder(ConfiguredYamlPreferences))
ConfiguredPropertiesPreferences = previousPreferences
test.AssertResultWithContext(t, s.expected, actual, s.description)
case "encode-wrapped":
prefs := ConfiguredPropertiesPreferences.Copy()
prefs.UnwrapScalar = false

View File

@ -209,50 +209,6 @@ address = "12 cat st"
var rtEmptyArray = `A = []
`
var rtEmptyArrayInTable = `[features]
my-feature = []
`
var rtMixedEmptyArraysInTable = `[features]
my-other-feature = []
my-feature = ["my-other-feature"]
`
var yamlEmptyArrayInTable = `features:
my-feature: []
`
var expectedTomlEmptyArrayInTable = `[features]
my-feature = []
`
var yamlMixedEmptyArraysInTable = `features:
my-other-feature: []
my-feature:
- my-other-feature
`
var expectedTomlMixedEmptyArraysInTable = `[features]
my-other-feature = []
my-feature = ["my-other-feature"]
`
var issue2688SampleToml = `[project]
name = "some-project"
version = "0.5.1"
authors = [{name = "Author", email = "author@example.com"}]
license = { file = "LICENSE" }
readme = "README.md"
`
var issue2688SampleExpected = `[project]
name = "some-project"
version = "0.5.2"
authors = [{ name = "Author", email = "author@example.com" }]
license = { file = "LICENSE" }
readme = "README.md"
`
var rtSampleTable = `var = "x"
[owner.contact]
@ -607,36 +563,6 @@ var tomlScenarios = []formatScenario{
expected: rtEmptyArray,
scenarioType: "roundtrip",
},
{
skipDoc: true,
description: "Issue #2674: roundtrip empty array in table",
input: rtEmptyArrayInTable,
expression: ".",
expected: rtEmptyArrayInTable,
scenarioType: "roundtrip",
},
{
skipDoc: true,
description: "Issue #2674: roundtrip mixed empty and non-empty arrays in table",
input: rtMixedEmptyArraysInTable,
expression: ".",
expected: rtMixedEmptyArraysInTable,
scenarioType: "roundtrip",
},
{
skipDoc: true,
description: "Issue #2674: encode empty array in table",
input: yamlEmptyArrayInTable,
expected: expectedTomlEmptyArrayInTable,
scenarioType: "encode",
},
{
skipDoc: true,
description: "Issue #2674: encode mixed empty and non-empty arrays in table",
input: yamlMixedEmptyArraysInTable,
expected: expectedTomlMixedEmptyArraysInTable,
scenarioType: "encode",
},
{
description: "Roundtrip: sample table",
input: rtSampleTable,
@ -723,33 +649,11 @@ var tomlScenarios = []formatScenario{
},
{
skipDoc: true,
description: "Encode: YAML flow mapping produces table section (same as block mapping)",
description: "Encode: YAML flow mapping stays inline",
input: "arg: {hello: foo}\n",
expected: "[arg]\nhello = \"foo\"\n",
expected: "arg = { hello = \"foo\" }\n",
scenarioType: "encode",
},
{
skipDoc: true,
description: "Issue: JSON auto-detected via YAML decoder produces table sections",
input: `{"arg":{"hello": "foo"}}`,
expected: "[arg]\nhello = \"foo\"\n",
scenarioType: "encode",
},
{
skipDoc: true,
description: "Issue: JSON via JSON decoder produces table sections",
input: `{"arg":{"hello": "foo"}}`,
expected: "[arg]\nhello = \"foo\"\n",
scenarioType: "encode-json",
},
{
skipDoc: true,
description: "Issue 2688: inline table arrays do not change following table scope",
input: issue2688SampleToml,
expression: `.project.version = "0.5.2"`,
expected: issue2688SampleExpected,
scenarioType: "roundtrip",
},
{
skipDoc: true,
description: "Roundtrip: key with special characters in inline table",
@ -791,8 +695,6 @@ func testTomlScenario(t *testing.T, s formatScenario) {
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewTomlDecoder(), NewTomlEncoder()), s.description)
case "encode":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewTomlEncoder()), s.description)
case "encode-json":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewJSONDecoder(), NewTomlEncoder()), s.description)
}
}
@ -873,7 +775,7 @@ func documentTomlScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
documentTomlDecodeScenario(w, s)
case "roundtrip":
documentTomlRoundtripScenario(w, s)
case "encode", "encode-json":
case "encode":
documentTomlEncodeScenario(w, s)
default:
@ -892,60 +794,6 @@ func TestTomlScenarios(t *testing.T) {
documentScenarios(t, "usage", "toml", genericScenarios, documentTomlScenario)
}
func TestTomlEncodeJsonKeepsRootArrayBeforeTables(t *testing.T) {
scenario := formatScenario{
description: "Encode: JSON root array stays outside later tables",
input: `{
"_source": {
"cookie": [
{
"Domain": "",
"Expires": "0001-01-01T00:00:00Z",
"HttpOnly": false,
"MaxAge": 0,
"Name": "name",
"Path": "",
"Raw": "",
"RawExpires": "",
"SameSite": 0,
"Secure": false,
"Unparsed": null,
"Value": "value"
}
]
},
"highlight": {
"did": [
"did"
]
},
"sort": [
1
]
}`,
expected: `sort = [1]
[[_source.cookie]]
Domain = ""
Expires = "0001-01-01T00:00:00Z"
HttpOnly = false
MaxAge = 0
Name = "name"
Path = ""
Raw = ""
RawExpires = ""
SameSite = 0
Secure = false
Value = "value"
[highlight]
did = ["did"]
`,
}
test.AssertResultWithContext(t, scenario.expected, mustProcessFormatScenario(scenario, NewJSONDecoder(), NewTomlEncoder()), scenario.description)
}
// TestTomlColourization tests that colourization correctly distinguishes
// between table section headers and inline arrays
func TestTomlColourization(t *testing.T) {

306
project-words.txt Normal file
View File

@ -0,0 +1,306 @@
abxbbxdbxebxczzx
abxbbxdbxebxczzy
accum
Accum
adithyasunil
AEDT
água
ÁGUA
alecthomas
appleapple
Astuff
autocreating
autoparse
AWST
axbxcxdxe
axbxcxdxexxx
bananabanana
barp
nbaz
bitnami
blarp
blddir
Bobo
BODMAS
bonapite
Brien
Bstuff
BUILDKIT
buildpackage
catmeow
CATYPE
CBVVE
chardata
chillum
choco
chomper
cleanup
cmlu
colorise
colors
Colors
colourize
compinit
coolioo
coverprofile
createmap
csvd
CSVUTF
currentlabel
cygpath
czvf
datestring
datetime
Datetime
datetimes
DEBEMAIL
debhelper
Debugf
debuild
delish
delpaths
DELPATHS
devorbitus
devscripts
dimchansky
Dont
dput
elliotchance
endhint
endofname
Entriesfrom
envsubst
errorlevel
Escandón
Evalall
fakefilename
fakeroot
Farah
fatih
Fifi
filebytes
Fileish
foobar
foobaz
foof
frood
fullpath
gitbook
githubactions
gnupg
goccy
gofmt
gogo
golangci
goreleaser
GORELEASER
GOMODCACHE
GOPATH
gosec
gota
goversion
GOVERSION
haha
hellno
herbygillot
hexdump
Hoang
hostpath
hotdog
howdy
incase
Infof
inlinetables
inplace
ints
ireduce
iwatch
jinzhu
jq's
jsond
keygrip
Keygrip
KEYGRIP
KEYID
keyvalue
kwak
lalilu
ldflags
LDFLAGS
lexer
Lexer
libdistro
lindex
linecomment
LVAs
magiconair
mapvalues
Mier
mikefarah
minideb
minishift
mipsle
mitchellh
mktemp
Mult
multidoc
multimaint
myenv
myenvnonexisting
myfile
myformat
ndjson
NDJSON
NFKD
nixpkgs
nojson
nonascii
nonempty
noninteractive
Nonquoting
nosec
notoml
noxml
nolua
nullinput
onea
Oneshot
opencollect
opstack
orderedmap
osarch
overridign
pacman
Padder
pandoc
parsechangelog
pcsv
pelletier
pflag
prechecking
Prerelease
proc
propsd
qylib
readline
realnames
realpath
repr
rhash
rindex
risentveber
rmescandon
Rosey
roundtrip
roundtrips
Roundtrip
roundtripping
Interp
interp
runningvms
sadface
selfupdate
setpath
sharedfolder
Sharedfolder
shellvariables
shellvars
shortfunc
shortpipe
shunit
snapcraft
somevalue
splt
srcdir
stackoverflow
stiched
Strc
strenv
strload
stylig
subarray
subchild
subdescription
submatch
submatches
SUBSTR
tempfile
tfstate
Tfstate
thar
timezone
Timezone
timezones
Timezones
tojson
Tokenvalue
tsvd
Tuan
tzdata
Uhoh
updateassign
urid
utfbom
Warningf
Wazowski
webi
Webi
wherever
winget
withdots
wizz
woop
workdir
Writable
xmld
xyzzy
yamld
yqlib
yuin
zabbix
tonumber
noyaml
nolint
shortfile
Unmarshalling
noini
nocsv
nobase64
nouri
noprops
nosh
noshell
tinygo
nonexistent
hclsyntax
hclwrite
nohcl
zclconf
cty
go-cty
Colorisation
goimports
errorlint
RDBMS
expeñded
bananabananabananabanana
edwinjhlee
flox
unlabelled
kyaml
KYAML
nokyaml
buildvcs
behaviour
GOFLAGS
gocache
subsubarray
Ffile
Fquery
coverpkg
gsub
ralia
Austr
ustrali
héllo
alia

View File

@ -8,22 +8,15 @@
- git push --tags
- use github actions to publish docker and make github release
- check github updated yq action in marketplace
- update github-action/Dockerfile to pin the newly published docker image digest (must match the mikefarah/yq:4 manifest digest):
docker buildx imagetools inspect docker.io/mikefarah/yq:4 --format '{{printf "%s" .Manifest.Digest}}'
- update github-action/Dockerfile to pin the newly published docker image digest:
skopeo inspect docker://docker.io/mikefarah/yq:4 | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['Digest'])"
then update the FROM line in github-action/Dockerfile with the new digest:
FROM mikefarah/yq:4@<digest-from-above>
- commit the Dockerfile change, then manually run the "Release Docker GitHub Action" workflow
(Actions -> Release Docker GitHub Action -> Run workflow)
- update action.yml to pin the newly published github-action image digest (must match the mikefarah/yq:4-githubaction manifest digest):
docker buildx imagetools inspect docker.io/mikefarah/yq:4-githubaction --format '{{printf "%s" .Manifest.Digest}}'
then update the image line in action.yml with the new digest:
image: 'docker://mikefarah/yq:4-githubaction@<digest-from-above>'
- commit the action.yml change and push
FROM mikefarah/yq:4@sha256:<new-digest>
// release artifacts are signed with cosign keyless signing (Sigstore)
// users can verify with:
// cosign verify-blob --bundle checksums.bundle checksums
// install cosign: brew install cosign OR go install github.com/sigstore/cosign/v2/cmd/cosign@v2.6.1
// install cosign: brew install cosign OR go install github.com/sigstore/cosign/v2/cmd/cosign@latest
- snapcraft

View File

@ -1,34 +1,3 @@
4.53.3:
- Add `--ini-preserve-quotes` flag for INI round-trip quote preservation (#2728) Thanks @toller892!
- Fix: reset INI decoder state on init (#2719) Thanks @xieby1!
- Fix: decode properties array bracket paths (#2693) Thanks @cyphercodes!
- Fix: preserve floats with trailing zero when encoding YAML to JSON (#2701) Thanks @ChrisJr404!
- Fix: JSON to TOML root scope and null handling (#2689) Thanks @LovesAsuna!
- Fix: reset TOML decoder finished flag on Init for multi-doc evaluation (#2704) Thanks @terminalchai!
- Fix: reset TOML decoder between files when evaluating all at once (#2685) Thanks @terminalchai!
- Fix: preserve TOML inline table array scope (#2694) Thanks @cyphercodes!
- Fix: preserve empty TOML arrays in tables (#2686) Thanks @cyphercodes!
- Fix: TOML encoder uses inline tables for YAML FlowStyle mappings (#2687)
- Fix nested inline YAML merge explode (#2699) Thanks @cyphercodes!
- Fix repeatString overflow test on 32-bit platforms (#2680) Thanks @jandubois!
- Bumped dependencies
4.53.2:
- Fixing release process
4.53.1:
- Releases and tags now signed and immutable!
- Add system(command; args) operator (disabled by default) (#2640)
- TOML encoder: prefer readable table sections over inline tables (#2649)
- Fix TOML encoder to quote keys containing special characters (#2648)
- Add string slicing support (#2639)
- Fix findInArray misuse on MappingNodes in equality and contains (#2645) Thanks @jandubois!
- Fix panic on negative slice indices that underflow after adjustment (#2646) Thanks @jandubois!
- Fix stack overflow from circular alias in traverse (#2647) Thanks @jandubois!
- Fix panic and OOM in repeatString for large repeat counts (#2644) Thanks @jandubois!
- Bumped dependencies
4.52.5:
- Fix: reset TOML decoder state between files (#2634) thanks @terminalchai
- Fix: preserve original filename when using --front-matter (#2613) thanks @cobyfrombrooklyn-bot

View File

@ -8,17 +8,14 @@ fi
version=$1
# validate version is in the right format (bash regex — portable; GNU sed's q1 is not on macOS)
if [[ ! $version =~ ^v4\.[0-9][0-9]\.[0-9][0-9]?$ ]]; then
echo "Please specify a valid version (e.g. v4.53.3)"
exit 1
fi
# validate version is in the right format
echo $version | sed -r '/v4\.[0-9][0-9]\.[0-9][0-9]?$/!{q1}'
previousVersion=$(cat cmd/version.go| sed -n 's/.*Version = "\([^"]*\)"/\1/p')
echo "Updating from $previousVersion to $version"
sed "s/\(.*Version =\).*/\1 \"$version\"/" cmd/version.go > cmd/version.go.tmp && mv cmd/version.go.tmp cmd/version.go
sed -i "s/\(.*Version =\).*/\1 \"$version\"/" cmd/version.go
go build .
actualVersion=$(./yq --version)
@ -52,5 +49,5 @@ fi
git add cmd/version.go snap/snapcraft.yaml
git commit -m 'Bumping version'
git tag $version -m "releasing"
git tag -f v4 -m "releasing $version"
git tag $version
git tag -f v4

View File

@ -1,50 +1,5 @@
#!/bin/sh
set -ex
go mod download golang.org/x/tools@v0.44.0
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/6008b81b81c690c046ffc3fd5bce896da715d5fd/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.11.3
curl -sSfL https://raw.githubusercontent.com/securego/gosec/424fc4cd9c82ea0fd6bee9cd49c2db2c3cc0c93f/install.sh | sh -s v2.22.11
TYPOS_VERSION=v1.47.2
TYPOS_OS=$(uname -s | tr '[:upper:]' '[:lower:]')
TYPOS_ARCH=$(uname -m)
case "${TYPOS_ARCH}" in
x86_64) TYPOS_ARCH=x86_64 ;;
aarch64|arm64) TYPOS_ARCH=aarch64 ;;
*)
echo "unsupported architecture for typos: ${TYPOS_ARCH}"
exit 1
;;
esac
case "${TYPOS_OS}" in
linux) TYPOS_TARGET="${TYPOS_ARCH}-unknown-linux-musl" ;;
darwin) TYPOS_TARGET="${TYPOS_ARCH}-apple-darwin" ;;
*)
echo "unsupported OS for typos: ${TYPOS_OS}"
exit 1
;;
esac
TYPOS_ARCHIVE="typos-${TYPOS_VERSION}-${TYPOS_TARGET}.tar.gz"
TYPOS_URL="https://github.com/crate-ci/typos/releases/download/${TYPOS_VERSION}/${TYPOS_ARCHIVE}"
case "${TYPOS_TARGET}" in
aarch64-apple-darwin) TYPOS_SHA256=23ca24a9186b5cb395b5f6c8eea8cdb02911c8980833e016454b56e90c3bd474 ;;
aarch64-unknown-linux-musl) TYPOS_SHA256=596d5c6b9ecf34307f68bea649178c5b45a4398fe3a1fcef9598e85aa2ccb742 ;;
x86_64-apple-darwin) TYPOS_SHA256=469a2d9fc894b0cdcec6e4fa3719b4c4638e195feee6517d4845450f8e8985c6 ;;
x86_64-unknown-linux-musl) TYPOS_SHA256=7aef58932fc123b4cf4b40d86468e89a3297d80169051d7cfd13a235e05fc426 ;;
*)
echo "unsupported typos target: ${TYPOS_TARGET}"
exit 1
;;
esac
TYPOS_TMPDIR=$(mktemp -d)
curl -sSfL "${TYPOS_URL}" -o "${TYPOS_TMPDIR}/${TYPOS_ARCHIVE}"
TYPOS_ACTUAL_SHA256=$(sha256sum "${TYPOS_TMPDIR}/${TYPOS_ARCHIVE}" 2>/dev/null | cut -d' ' -f1)
if [ -z "${TYPOS_ACTUAL_SHA256}" ]; then
TYPOS_ACTUAL_SHA256=$(shasum -a 256 "${TYPOS_TMPDIR}/${TYPOS_ARCHIVE}" | cut -d' ' -f1)
fi
if [ "${TYPOS_ACTUAL_SHA256}" != "${TYPOS_SHA256}" ]; then
echo "typos archive checksum mismatch: expected ${TYPOS_SHA256}, got ${TYPOS_ACTUAL_SHA256}"
exit 1
fi
tar xzf "${TYPOS_TMPDIR}/${TYPOS_ARCHIVE}" -C "${TYPOS_TMPDIR}"
install -m 755 "${TYPOS_TMPDIR}/typos" "$(go env GOPATH)/bin/typos"
rm -rf "${TYPOS_TMPDIR}"
go mod download golang.org/x/tools@latest
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.11.3
curl -sSfL https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.22.11

View File

@ -1,18 +1,3 @@
#!/bin/bash
set -euo pipefail
GOPATH_TYPOS="$(go env GOPATH)/bin/typos"
TYPOS_CMD=""
if [ -f "${GOPATH_TYPOS}" ]; then
TYPOS_CMD="${GOPATH_TYPOS}"
elif command -v typos >/dev/null 2>&1; then
TYPOS_CMD="typos"
else
echo "Error: typos not found in $(go env GOPATH)/bin or PATH."
echo "Please run scripts/devtools.sh or ensure typos is installed correctly."
exit 1
fi
git ls-files '*.go' '*.sh' '*.md' | "${TYPOS_CMD}" --file-list -
npx cspell --no-progress "**/*.{sh,go,md}"

View File

@ -1,7 +1,3 @@
#!/bin/bash
go test $(go list ./... | grep -v -E 'examples' | grep -v -E 'test')
# Run after the main test suite: TestGoInstallCompatibility zips the module tree and
# must not run in parallel with pkg/yqlib tests that rewrite doc/*.md files.
go test -tags goinstall -run TestGoInstallCompatibility .

View File

@ -2,7 +2,7 @@
set -eo pipefail
# You may need to go install github.com/goreleaser/goreleaser/v2@v2.16.0 first
# You may need to go install github.com/goreleaser/goreleaser/v2@latest first
GORELEASER="goreleaser build --clean"
if [ -z "$CI" ] || [[ "${GITHUB_REF_NAME:-}" == draft-* ]]; then
GORELEASER+=" --snapshot"

View File

@ -1,5 +1,5 @@
name: yq
version: 'v4.53.3'
version: 'v4.52.5'
summary: A lightweight and portable command-line data file processor
description: |
`yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml, json, xml, csv, properties and TOML files.
@ -32,6 +32,6 @@ parts:
build-environment:
- CGO_ENABLED: 0
source: https://github.com/mikefarah/yq.git
source-tag: v4.53.3
source-tag: v4.52.5
build-snaps:
- go/latest/stable