mirror of
https://github.com/mikefarah/yq.git
synced 2026-08-24 08:22:13 +08:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Cursor AgentandMike Farah
|
00efd3cc0e | ||
|
Cursor AgentandMike Farah
|
4ee76205b3 |
@@ -40,11 +40,11 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
|
||||
uses: github/codeql-action/init@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
@@ -55,7 +55,7 @@ jobs:
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below)
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
|
||||
uses: github/codeql-action/autobuild@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
||||
@@ -69,4 +69,4 @@ jobs:
|
||||
# make release
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd # v4.37.7
|
||||
uses: github/codeql-action/analyze@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
name: Release Docker GitHub Action
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
publishGithubActionDocker:
|
||||
environment: dockerhub
|
||||
env:
|
||||
IMAGE_NAME: mikefarah/yq
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
|
||||
with:
|
||||
platforms: all
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
|
||||
with:
|
||||
version: latest
|
||||
|
||||
- name: Verify Dockerfile base image digest matches yq:4
|
||||
run: |
|
||||
PINNED_DIGEST=$(grep -oE 'sha256:[a-f0-9]{64}' github-action/Dockerfile | head -1)
|
||||
if [ -z "${PINNED_DIGEST}" ]; then
|
||||
echo "::error::Could not find a sha256 digest in github-action/Dockerfile"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
LATEST_DIGEST=$(docker buildx imagetools inspect "${IMAGE_NAME}:4" --format '{{printf "%s" .Manifest.Digest}}')
|
||||
|
||||
echo "Dockerfile pins: ${PINNED_DIGEST}"
|
||||
echo "mikefarah/yq:4 is: ${LATEST_DIGEST}"
|
||||
|
||||
if [ "${PINNED_DIGEST}" != "${LATEST_DIGEST}" ]; then
|
||||
echo "::error::github-action/Dockerfile digest does not match the current mikefarah/yq:4 image"
|
||||
echo "Update the FROM line in github-action/Dockerfile to:"
|
||||
echo " FROM mikefarah/yq:4@${LATEST_DIGEST}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Resolve version from yq:4
|
||||
run: |
|
||||
IMAGE_VERSION=$(docker run --rm "${IMAGE_NAME}:4" --version | awk '{print $NF}' | sed 's/^v//')
|
||||
if [ -z "${IMAGE_VERSION}" ]; then
|
||||
echo "::error::Could not determine yq version from ${IMAGE_NAME}:4"
|
||||
exit 1
|
||||
fi
|
||||
echo "Resolved yq version: ${IMAGE_VERSION}"
|
||||
echo "IMAGE_VERSION=${IMAGE_VERSION}" >> "${GITHUB_ENV}"
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Build and push github-action image
|
||||
working-directory: github-action
|
||||
run: |
|
||||
PLATFORMS="linux/amd64,linux/ppc64le,linux/arm64,linux/arm/v7,linux/s390x"
|
||||
|
||||
echo "Building and pushing github-action image for version ${IMAGE_VERSION}"
|
||||
docker buildx build \
|
||||
--label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
|
||||
--label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
|
||||
--label "org.opencontainers.image.description=yq is a portable command-line data file processor" \
|
||||
--label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
|
||||
--label "org.opencontainers.image.licenses=MIT" \
|
||||
--label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \
|
||||
--label "org.opencontainers.image.source=https://github.com/mikefarah/yq" \
|
||||
--label "org.opencontainers.image.title=yq" \
|
||||
--label "org.opencontainers.image.url=https://mikefarah.gitbook.io/yq/" \
|
||||
--label "org.opencontainers.image.version=${IMAGE_VERSION}" \
|
||||
--platform "${PLATFORMS}" \
|
||||
--pull \
|
||||
--push \
|
||||
-t "${IMAGE_NAME}:${IMAGE_VERSION}-githubaction" \
|
||||
-t "${IMAGE_NAME}:4-githubaction" \
|
||||
-t "${IMAGE_NAME}:latest-githubaction" \
|
||||
-t "ghcr.io/${IMAGE_NAME}:${IMAGE_VERSION}-githubaction" \
|
||||
-t "ghcr.io/${IMAGE_NAME}:4-githubaction" \
|
||||
-t "ghcr.io/${IMAGE_NAME}:latest-githubaction" \
|
||||
.
|
||||
|
||||
- name: Report action.yml digest to pin
|
||||
run: |
|
||||
GITHUBACTION_DIGEST=$(docker buildx imagetools inspect "${IMAGE_NAME}:4-githubaction" --format '{{printf "%s" .Manifest.Digest}}')
|
||||
echo "Published ${IMAGE_NAME}:4-githubaction at ${GITHUBACTION_DIGEST}"
|
||||
echo "Update action.yml image to:"
|
||||
echo " docker://${IMAGE_NAME}:4-githubaction@${GITHUBACTION_DIGEST}"
|
||||
@@ -19,10 +19,10 @@ jobs:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4.2.0
|
||||
uses: docker/setup-qemu-action@06116385d9baf250c9f4dcb4858b16962ea869c3 # v4.1.0
|
||||
with:
|
||||
platforms: all
|
||||
|
||||
@@ -36,13 +36,13 @@ jobs:
|
||||
run: echo ${{ steps.buildx.outputs.platforms }} && docker version
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0 # v4.4.0
|
||||
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
@@ -80,3 +80,26 @@ jobs:
|
||||
-t "ghcr.io/${IMAGE_NAME}:4" \
|
||||
-t "ghcr.io/${IMAGE_NAME}:latest" \
|
||||
.
|
||||
|
||||
cd github-action
|
||||
docker buildx build \
|
||||
--label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
|
||||
--label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
|
||||
--label "org.opencontainers.image.description=yq is a portable command-line data file processor" \
|
||||
--label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
|
||||
--label "org.opencontainers.image.licenses=MIT" \
|
||||
--label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \
|
||||
--label "org.opencontainers.image.source=https://github.com/mikefarah/yq" \
|
||||
--label "org.opencontainers.image.title=yq" \
|
||||
--label "org.opencontainers.image.url=https://mikefarah.gitbook.io/yq/" \
|
||||
--label "org.opencontainers.image.version=${IMAGE_VERSION}" \
|
||||
--platform "${PLATFORMS}" \
|
||||
--pull \
|
||||
--push \
|
||||
-t "${IMAGE_NAME}:${IMAGE_VERSION}-githubaction" \
|
||||
-t "${IMAGE_NAME}:4-githubaction" \
|
||||
-t "${IMAGE_NAME}:latest-githubaction" \
|
||||
-t "ghcr.io/${IMAGE_NAME}:${IMAGE_VERSION}-githubaction" \
|
||||
-t "ghcr.io/${IMAGE_NAME}:4-githubaction" \
|
||||
-t "ghcr.io/${IMAGE_NAME}:latest-githubaction" \
|
||||
.
|
||||
|
||||
@@ -5,51 +5,25 @@ permissions:
|
||||
|
||||
jobs:
|
||||
|
||||
verify-action-digest:
|
||||
name: Verify action.yml image digest
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
|
||||
- name: Verify action.yml digest matches published image
|
||||
run: |
|
||||
PINNED_DIGEST=$(grep -oE 'sha256:[a-f0-9]{64}' action.yml | head -1)
|
||||
if [ -z "${PINNED_DIGEST}" ]; then
|
||||
echo "::error::action.yml does not pin the runtime image by digest"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
LATEST_DIGEST=$(docker buildx imagetools inspect docker.io/mikefarah/yq:4-githubaction --format '{{printf "%s" .Manifest.Digest}}')
|
||||
|
||||
echo "action.yml pins: ${PINNED_DIGEST}"
|
||||
echo "mikefarah/yq:4-githubaction: ${LATEST_DIGEST}"
|
||||
|
||||
if [ "${PINNED_DIGEST}" != "${LATEST_DIGEST}" ]; then
|
||||
echo "::error::action.yml digest does not match the current mikefarah/yq:4-githubaction image"
|
||||
echo "Update the image line in action.yml to:"
|
||||
echo " docker://mikefarah/yq:4-githubaction@${LATEST_DIGEST}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
build:
|
||||
name: Build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
|
||||
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version: '^1.20'
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
go get -v -t -d ./...
|
||||
if [ -f Gopkg.toml ]; then
|
||||
curl -sSfL https://raw.githubusercontent.com/golang/dep/1f7c19e5f52f49ffb9f956f64c010be14683468b/install.sh | env DEP_RELEASE_TAG=v0.5.4 sh
|
||||
curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
|
||||
dep ensure
|
||||
fi
|
||||
|
||||
|
||||
@@ -14,8 +14,8 @@ jobs:
|
||||
contents: write
|
||||
id-token: write
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
|
||||
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version: '^1.20'
|
||||
check-latest: true
|
||||
@@ -29,7 +29,7 @@ jobs:
|
||||
run: echo "VERSION=${GITHUB_REF##*/}" >> "${GITHUB_OUTPUT}"
|
||||
|
||||
- name: Generate man page
|
||||
uses: docker://pandoc/core:2.14.2@sha256:04e127c6642a2b9d447c26fe0ac6a5932efa8f508eda9f07da51b6e621dd7c19
|
||||
uses: docker://pandoc/core:2.14.2
|
||||
id: gen-man-page
|
||||
with:
|
||||
args: >-
|
||||
@@ -43,12 +43,12 @@ jobs:
|
||||
man.md
|
||||
|
||||
- name: Install cosign
|
||||
uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
|
||||
uses: sigstore/cosign-installer@v3
|
||||
|
||||
- name: Cross compile
|
||||
run: |
|
||||
sudo apt-get install rhash -y
|
||||
go install github.com/goreleaser/goreleaser/v2@v2.16.0
|
||||
go install github.com/goreleaser/goreleaser/v2@latest
|
||||
./scripts/xcompile.sh
|
||||
|
||||
- name: Sign checksums
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
cosign sign-blob --yes --bundle build/checksums-bsd.bundle build/checksums-bsd
|
||||
|
||||
- name: Release
|
||||
uses: softprops/action-gh-release@718ea10b132b3b2eba29c1007bb80653f286566b # v3.0.1
|
||||
uses: softprops/action-gh-release@b4309332981a82ec1c5618f44dd2e27cc8bfbfda # v3.0.0
|
||||
with:
|
||||
files: build/*
|
||||
draft: true
|
||||
|
||||
@@ -34,12 +34,12 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: "Checkout code"
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: "Run analysis"
|
||||
uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4
|
||||
uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3
|
||||
with:
|
||||
results_file: results.sarif
|
||||
results_format: sarif
|
||||
@@ -73,6 +73,6 @@ jobs:
|
||||
# Upload the results to GitHub's code scanning dashboard (optional).
|
||||
# Commenting out will disable upload of results to your repo's Code Scanning dashboard
|
||||
- name: "Upload to code-scanning"
|
||||
uses: github/codeql-action/upload-sarif@d1ba80a13dd99fba24a470575428917156a28b43 # v4.37.5
|
||||
uses: github/codeql-action/upload-sarif@v4
|
||||
with:
|
||||
sarif_file: results.sarif
|
||||
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
- uses: snapcore/action-build@3bdaa03e1ba6bf59a65f84a751d943d549a54e79 # v1.3.0
|
||||
id: build
|
||||
env:
|
||||
|
||||
@@ -13,7 +13,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
|
||||
- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
|
||||
- name: Get test
|
||||
id: get_value
|
||||
uses: mikefarah/yq@master
|
||||
|
||||
@@ -1,26 +1,6 @@
|
||||
# yq — agent instructions
|
||||
|
||||
## ⚠️ MANDATORY: GitHub agent disclosure
|
||||
|
||||
**Always required. No exceptions.**
|
||||
|
||||
Whenever you perform **any** GitHub action on behalf of the user, you **must** disclose that an AI agent (Cursor) wrote the content and is acting on the user's behalf — **not the user personally**. Do this **before** submitting; never post first and add the disclosure later.
|
||||
|
||||
Applies to **all** GitHub interactions, including:
|
||||
|
||||
- Pull requests (titles, descriptions, and reviews)
|
||||
- PR comments and inline review comments
|
||||
- Issues (new issues, comments, and updates)
|
||||
- Any other post or reply on GitHub
|
||||
|
||||
**How to disclose:** Put it prominently at the **top** of every PR description, review body, comment, or issue. Use wording like:
|
||||
|
||||
|
||||
Inline review comments must include a short disclosure too (e.g. `> Generated by Cursor acting on the user's behalf, not the user personally.`).
|
||||
|
||||
**Never** submit a GitHub action without this disclosure.
|
||||
|
||||
---
|
||||
See [agents.md](./agents.md) for contributor workflow (formatting, testing patterns, adding encoders/decoders).
|
||||
|
||||
Always run the spellcheck before raising a PR:
|
||||
|
||||
@@ -39,6 +19,7 @@ This is also included in the full CI pipeline via `make local test`.
|
||||
### Prerequisites
|
||||
|
||||
- **Go ≥ 1.25** (see `go.mod`)
|
||||
- **Node.js** (for `npx cspell` spelling checks in the full `make test` pipeline)
|
||||
- **Bash** (acceptance tests)
|
||||
- **Docker/Podman** is optional; use `make local <target>` to run natively when containers are unavailable
|
||||
|
||||
@@ -50,7 +31,7 @@ After `scripts/devtools.sh`, add Go tool binaries to PATH:
|
||||
export PATH="$HOME/go/bin:$PATH"
|
||||
```
|
||||
|
||||
`golangci-lint` and `typos` install to `$HOME/go/bin`; `gosec` installs to `./bin/gosec` in the repo root.
|
||||
`golangci-lint` installs to `$HOME/go/bin`; `gosec` installs to `./bin/gosec` in the repo root.
|
||||
|
||||
### Common commands (local, no Docker)
|
||||
|
||||
@@ -69,432 +50,6 @@ export PATH="$HOME/go/bin:$PATH"
|
||||
### Caveats
|
||||
|
||||
- **`make` without `local`** tries Docker/Podman (`Dockerfile.dev`). In Cloud Agent VMs without Docker, always prefix with `make local`.
|
||||
- **Spelling step** uses `typos` (installed by `scripts/devtools.sh`).
|
||||
- **Spelling step** uses `npx cspell` and may download cspell on first run (network required).
|
||||
- **`make local test` / `scripts/check.sh`** require `golangci-lint` on PATH (`devtools.sh`).
|
||||
|
||||
---
|
||||
|
||||
# General rules
|
||||
✅ **DO:**
|
||||
- You can use ./yq with the `--debug-node-info` flag to get a deeper understanding of the ast.
|
||||
- run ./scripts/format.sh to format the code; then ./scripts/check.sh lint and finally ./scripts/spelling.sh to check spelling.
|
||||
- Add comprehensive tests to cover the changes
|
||||
- Run test suite to ensure there is no regression
|
||||
- Use UK english spelling
|
||||
- **Follow the mandatory GitHub agent disclosure rule above** on every GitHub action — no exceptions
|
||||
|
||||
❌ **DON'T:**
|
||||
- Git add or commit
|
||||
- Add comments to functions that are self-explanatory
|
||||
- **Post to GitHub without the mandatory agent disclosure** (PRs, reviews, comments, issues, or any other GitHub interaction)
|
||||
|
||||
|
||||
|
||||
# Adding a New Encoder/Decoder
|
||||
|
||||
This guide explains how to add support for a new format (encoder/decoder) to yq without modifying `candidate_node.go`.
|
||||
|
||||
## Overview
|
||||
|
||||
The encoder/decoder architecture in yq is based on two main interfaces:
|
||||
|
||||
- **Encoder**: Converts a `CandidateNode` to output in a specific format
|
||||
- **Decoder**: Reads input in a specific format and creates a `CandidateNode`
|
||||
|
||||
Each format is registered in `pkg/yqlib/format.go` and made available through factory functions.
|
||||
|
||||
## Architecture
|
||||
|
||||
### Key Files
|
||||
|
||||
- `pkg/yqlib/encoder.go` - Defines the `Encoder` interface
|
||||
- `pkg/yqlib/decoder.go` - Defines the `Decoder` interface
|
||||
- `pkg/yqlib/format.go` - Format registry and factory functions
|
||||
- `pkg/yqlib/operator_encoder_decoder.go` - Encode/decode operators
|
||||
- `pkg/yqlib/encoder_*.go` - Encoder implementations
|
||||
- `pkg/yqlib/decoder_*.go` - Decoder implementations
|
||||
|
||||
### Interfaces
|
||||
|
||||
**Encoder Interface:**
|
||||
```go
|
||||
type Encoder interface {
|
||||
Encode(writer io.Writer, node *CandidateNode) error
|
||||
PrintDocumentSeparator(writer io.Writer) error
|
||||
PrintLeadingContent(writer io.Writer, content string) error
|
||||
CanHandleAliases() bool
|
||||
}
|
||||
```
|
||||
|
||||
**Decoder Interface:**
|
||||
```go
|
||||
type Decoder interface {
|
||||
Init(reader io.Reader) error
|
||||
Decode() (*CandidateNode, error)
|
||||
}
|
||||
```
|
||||
|
||||
## Step-by-Step: Adding a New Encoder/Decoder
|
||||
|
||||
### Step 1: Create the Encoder File
|
||||
|
||||
Create `pkg/yqlib/encoder_<format>.go` implementing the `Encoder` interface:
|
||||
- `Encode()` - Convert a `CandidateNode` to your format and write to the output writer
|
||||
- `PrintDocumentSeparator()` - Handle document separators if your format requires them
|
||||
- `PrintLeadingContent()` - Handle leading content/comments if supported
|
||||
- `CanHandleAliases()` - Return whether your format supports YAML aliases
|
||||
|
||||
See `encoder_json.go` or `encoder_base64.go` for examples.
|
||||
|
||||
### Step 2: Create the Decoder File
|
||||
|
||||
Create `pkg/yqlib/decoder_<format>.go` implementing the `Decoder` interface:
|
||||
- `Init()` - Initialize the decoder with the input reader and set up any needed state
|
||||
- `Decode()` - Decode one document from the input and return a `CandidateNode`, or `io.EOF` when finished
|
||||
|
||||
See `decoder_json.go` or `decoder_base64.go` for examples.
|
||||
|
||||
### Step 3: Create Tests (Mandatory)
|
||||
|
||||
Create a test file `pkg/yqlib/<format>_test.go` using the `formatScenario` pattern:
|
||||
- Define test scenarios as `formatScenario` structs with fields: `description`, `input`, `expected`, `scenarioType`
|
||||
- `scenarioType` can be `"decode"` (test decoding to YAML) or `"roundtrip"` (encode/decode preservation)
|
||||
- Create a helper function `test<Format>Scenario()` that switches on `scenarioType`
|
||||
- Create main test function `Test<Format>FormatScenarios()` that iterates over scenarios
|
||||
- The main test function should use `documentScenarios` to ensure testcase documentation is generated.
|
||||
|
||||
Test coverage must include:
|
||||
- Basic data types (scalars, arrays, objects/maps)
|
||||
- Nested structures
|
||||
- Edge cases (empty inputs, special characters, escape sequences)
|
||||
- Format-specific features or syntax
|
||||
- Round-trip tests: decode → encode → decode should preserve data
|
||||
|
||||
See `hcl_test.go` for a complete example.
|
||||
|
||||
### Step 4: Register the Format in format.go
|
||||
|
||||
Edit `pkg/yqlib/format.go`:
|
||||
|
||||
1. Add a new format variable:
|
||||
- `"<format>"` is the formal name (e.g., "json", "yaml")
|
||||
- `[]string{...}` contains short aliases (can be empty)
|
||||
- The first function creates an encoder (can be nil for encode-only formats)
|
||||
- The second function creates a decoder (can be nil for decode-only formats)
|
||||
|
||||
2. Add the format to the `Formats` slice in the same file
|
||||
|
||||
See existing formats in `format.go` for the exact structure.
|
||||
|
||||
### Step 5: Handle Encoder Configuration (if needed)
|
||||
|
||||
If your format has preferences/configuration options:
|
||||
|
||||
1. Create a preferences struct with your configuration fields
|
||||
2. Update the encoder to accept preferences in its factory function
|
||||
3. Update `format.go` to pass the configured preferences
|
||||
4. Update `operator_encoder_decoder.go` if special indent handling is needed (see existing formats like JSON and YAML for the pattern)
|
||||
|
||||
This pattern is optional and only needed if your format has user-configurable options.
|
||||
|
||||
## Build Tags
|
||||
|
||||
Use build tags to allow optional compilation of formats:
|
||||
- Add `//go:build !yq_no<format>` at the top of your encoder and decoder files
|
||||
- Create a no-build version in `pkg/yqlib/no_<format>.go` that returns nil for encoder/decoder factories
|
||||
|
||||
This allows users to compile yq without certain formats using: `go build -tags yq_no<format>`
|
||||
|
||||
## Working with CandidateNode
|
||||
|
||||
The `CandidateNode` struct represents a YAML node with:
|
||||
- `Kind`: The node type (ScalarNode, SequenceNode, MappingNode)
|
||||
- `Tag`: The YAML tag (e.g., "!!str", "!!int", "!!map")
|
||||
- `Value`: The scalar value (for ScalarNode only)
|
||||
- `Content`: Child nodes (for SequenceNode and MappingNode)
|
||||
|
||||
Key methods:
|
||||
- `node.guessTagFromCustomType()` - Infer the tag from Go type
|
||||
- `node.AsList()` - Convert to a list for processing
|
||||
- `node.CreateReplacement()` - Create a new replacement node
|
||||
- `NewCandidate()` - Create a new CandidateNode
|
||||
|
||||
## Key Points
|
||||
|
||||
✅ **DO:**
|
||||
- Implement only the `Encoder` and `Decoder` interfaces
|
||||
- Register your format in `format.go` only
|
||||
- Keep format-specific logic in your encoder/decoder files
|
||||
- Use the candidate_node style attribute to store style information for round-trip. Ask if this needs to be updated with new styles.
|
||||
- Use build tags for optional compilation
|
||||
- Add comprehensive tests
|
||||
- Run the specific encoder/decoder test (e.g. <format>_test.go) whenever you make ay changes to the encoder_<format> or decoder_<format>
|
||||
- Handle errors gracefully
|
||||
- Add the no build directive, like the xml encoder and decoder, that enables a minimal yq builds. e.g. `//go:build !yq_<format>`. Be sure to also update the build_small-yq.sh and build-tinygo-yq.sh to not include the new format.
|
||||
|
||||
❌ **DON'T:**
|
||||
- Modify `candidate_node.go` to add format-specific logic
|
||||
- Add format-specific fields to `CandidateNode`
|
||||
- Create special cases in core navigation or evaluation logic
|
||||
- Bypass the encoder/decoder interfaces
|
||||
- Use candidate_node tag attribute for anything other than indicate the data type
|
||||
|
||||
## Examples
|
||||
|
||||
Refer to existing format implementations for patterns:
|
||||
|
||||
- **Simple encoder/decoder**: `encoder_json.go`, `decoder_json.go`
|
||||
- **Complex with preferences**: `encoder_yaml.go`, `decoder_yaml.go`
|
||||
- **Encoder-only**: `encoder_sh.go` (ShFormat has nil decoder)
|
||||
- **String-only operations**: `encoder_base64.go`, `decoder_base64.go`
|
||||
|
||||
## Testing Your Implementation (Mandatory)
|
||||
|
||||
Tests must be implemented in `<format>_test.go` following the `formatScenario` pattern:
|
||||
|
||||
1. **Create test scenarios** using the `formatScenario` struct with fields:
|
||||
- `description`: Brief description of what's being tested
|
||||
- `input`: Sample input in your format
|
||||
- `expected`: Expected output (typically in YAML for decode tests)
|
||||
- `scenarioType`: Either `"decode"` or `"roundtrip"`
|
||||
|
||||
2. **Test coverage must include:**
|
||||
- Basic data types (scalars, arrays, objects/maps)
|
||||
- Nested structures
|
||||
- Edge cases (empty inputs, special characters, escape sequences)
|
||||
- Format-specific features or syntax
|
||||
- Round-trip tests: decode → encode → decode should preserve data
|
||||
|
||||
3. **Test function pattern:**
|
||||
- `test<Format>Scenario()`: Helper function that switches on `scenarioType`
|
||||
- `Test<Format>FormatScenarios()`: Main test function that iterates over scenarios
|
||||
|
||||
4. **Example from existing formats:**
|
||||
- See `hcl_test.go` for a complete example
|
||||
- See `yaml_test.go` for YAML-specific patterns
|
||||
- See `json_test.go` for more complex scenarios
|
||||
|
||||
## Common Patterns
|
||||
|
||||
### Format with Indentation
|
||||
Use preferences to control output formatting:
|
||||
```go
|
||||
type <format>Preferences struct {
|
||||
Indent int
|
||||
}
|
||||
|
||||
func (prefs *<format>Preferences) Copy() <format>Preferences {
|
||||
return *prefs
|
||||
}
|
||||
```
|
||||
|
||||
### Multiple Documents
|
||||
Decoders should support reading multiple documents:
|
||||
```go
|
||||
func (dec *<format>Decoder) Decode() (*CandidateNode, error) {
|
||||
if dec.finished {
|
||||
return nil, io.EOF
|
||||
}
|
||||
// ... decode next document ...
|
||||
if noMoreDocuments {
|
||||
dec.finished = true
|
||||
}
|
||||
return candidate, nil
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
# Adding a New Operator
|
||||
|
||||
This guide explains how to add a new operator to yq. Operators are the core of yq's expression language and process `CandidateNode` objects without requiring modifications to `candidate_node.go` itself.
|
||||
|
||||
## Overview
|
||||
|
||||
Operators transform data by implementing a handler function that processes a `Context` containing `CandidateNode` objects. Each operator is:
|
||||
|
||||
1. Defined as an `operationType` in `operation.go`
|
||||
2. Registered in the lexer in `lexer_participle.go`
|
||||
3. Implemented in its own `operator_<type>.go` file
|
||||
4. Tested in `operator_<type>_test.go`
|
||||
5. Documented in `pkg/yqlib/doc/operators/headers/<type>.md`
|
||||
|
||||
## Architecture
|
||||
|
||||
### Key Files
|
||||
|
||||
- `pkg/yqlib/operation.go` - Defines `operationType` and operator registry
|
||||
- `pkg/yqlib/lexer_participle.go` - Registers operators with their syntax patterns
|
||||
- `pkg/yqlib/operator_<type>.go` - Operator implementation
|
||||
- `pkg/yqlib/operator_<type>_test.go` - Operator tests using `expressionScenario`
|
||||
- `pkg/yqlib/doc/operators/headers/<type>.md` - Documentation header
|
||||
|
||||
### Core Types
|
||||
|
||||
**operationType:**
|
||||
```go
|
||||
type operationType struct {
|
||||
Type string // Unique operator name (e.g., "REVERSE")
|
||||
NumArgs uint // Number of arguments (0 for no args)
|
||||
Precedence uint // Operator precedence (higher = higher precedence)
|
||||
Handler operatorHandler // The function that executes the operator
|
||||
CheckForPostTraverse bool // Whether to apply post-traversal logic
|
||||
ToString func(*Operation) string // Custom string representation
|
||||
}
|
||||
```
|
||||
|
||||
**operatorHandler signature:**
|
||||
```go
|
||||
type operatorHandler func(*dataTreeNavigator, Context, *ExpressionNode) (Context, error)
|
||||
```
|
||||
|
||||
**expressionScenario for tests:**
|
||||
```go
|
||||
type expressionScenario struct {
|
||||
description string
|
||||
subdescription string
|
||||
document string
|
||||
expression string
|
||||
expected []string
|
||||
skipDoc bool
|
||||
expectedError string
|
||||
}
|
||||
```
|
||||
|
||||
## Step-by-Step: Adding a New Operator
|
||||
|
||||
### Step 1: Create the Operator Implementation File
|
||||
|
||||
Create `pkg/yqlib/operator_<type>.go` implementing the operator handler function:
|
||||
- Implement the `operatorHandler` function signature
|
||||
- Process nodes from `context.MatchingNodes`
|
||||
- Return a new `Context` with results using `context.ChildContext()`
|
||||
- Use `candidate.CreateReplacement()` or `candidate.CreateReplacementWithComments()` to create new nodes
|
||||
- Handle errors gracefully with meaningful error messages
|
||||
|
||||
See `operator_reverse.go` or `operator_keys.go` for examples.
|
||||
|
||||
### Step 2: Register the Operator in operation.go
|
||||
|
||||
Add the operator type definition to `pkg/yqlib/operation.go`:
|
||||
|
||||
```go
|
||||
var <type>OpType = &operationType{
|
||||
Type: "<TYPE>", // All caps, matches pattern in lexer
|
||||
NumArgs: 0, // 0 for no args, 1+ for args
|
||||
Precedence: 50, // Typical range: 40-55
|
||||
Handler: <type>Operator, // Reference to handler function
|
||||
}
|
||||
```
|
||||
|
||||
**Precedence guidelines:**
|
||||
- 10-20: Logical operators (OR, AND, UNION)
|
||||
- 30: Pipe operator
|
||||
- 40: Assignment and comparison operators
|
||||
- 42: Arithmetic operators (ADD, SUBTRACT, MULTIPLY, DIVIDE)
|
||||
- 50-52: Most other operators
|
||||
- 55: High precedence (e.g., GET_VARIABLE)
|
||||
|
||||
**Optional fields:**
|
||||
- `CheckForPostTraverse: true` - If your operator can have another directly after it without the pipe character. Most of the time this is false.
|
||||
- `ToString: customToString` - Custom string representation (rarely needed)
|
||||
|
||||
### Step 3: Register the Operator in lexer_participle.go
|
||||
|
||||
Edit `pkg/yqlib/lexer_participle.go` to add the operator to the lexer rules:
|
||||
- Use `simpleOp()` for simple keyword patterns
|
||||
- Use object syntax for regex patterns or complex syntax
|
||||
- Support optional characters with `_?` and aliases with `|`
|
||||
|
||||
See existing operators in `lexer_participle.go` for pattern examples.
|
||||
|
||||
### Step 4: Create Tests (Mandatory)
|
||||
|
||||
Create `pkg/yqlib/operator_<type>_test.go` using the `expressionScenario` pattern:
|
||||
- Define test scenarios with `description`, `document`, `expression`, and `expected` fields
|
||||
- `expected` is a slice of strings showing output format: `"D<doc>, P[<path>], (<tag>)::<value>\n"`
|
||||
- Set `skipDoc: true` for edge cases you don't want in generated documentation
|
||||
- Include `subdescription` for longer test names
|
||||
- Set `expectedError` if testing error cases
|
||||
- Create main test function that iterates over scenarios
|
||||
- The main test function should use `documentScenarios` to ensure testcase documentation is generated.
|
||||
|
||||
Test coverage must include:
|
||||
- Basic data types and nested structures
|
||||
- Edge cases (empty inputs, special characters, type errors)
|
||||
- Multiple outputs if applicable
|
||||
- Format-specific features
|
||||
|
||||
See `operator_reverse_test.go` for a simple example and `operator_keys_test.go` for complex cases.
|
||||
|
||||
### Step 5: Create Documentation Header
|
||||
|
||||
Create `pkg/yqlib/doc/operators/headers/<type>.md`:
|
||||
- Use the exact operator name as the title
|
||||
- Include a concise 1-2 sentence summary
|
||||
- Add additional context or examples if the operator is complex
|
||||
|
||||
See existing headers in `doc/operators/headers/` for examples.
|
||||
|
||||
## Working with Context and CandidateNode
|
||||
|
||||
### Context Management
|
||||
- `context.ChildContext(results)` - Create child context with results
|
||||
- `context.GetVariable("varName")` - Get variables stored in context
|
||||
- `context.SetVariable("varName", value)` - Set variables in context
|
||||
|
||||
### CandidateNode Operations
|
||||
- `candidate.CreateReplacement(ScalarNode, "!!str", stringValue)` - Create a replacement node
|
||||
- `candidate.CreateReplacementWithComments(SequenceNode, "!!seq", candidate.Style)` - With style preserved
|
||||
- `candidate.Kind` - The node type (ScalarNode, SequenceNode, MappingNode)
|
||||
- `candidate.Tag` - The YAML tag (!!str, !!int, etc.)
|
||||
- `candidate.Value` - The scalar value (for ScalarNode only)
|
||||
- `candidate.Content` - Child nodes (for SequenceNode and MappingNode)
|
||||
- `candidate.guessTagFromCustomType()` - Infer the tag from Go type
|
||||
- `candidate.AsList()` - Convert to a list representation
|
||||
|
||||
## Key Points
|
||||
|
||||
✅ **DO:**
|
||||
- Implement the operator handler with the correct signature
|
||||
- Register in `operation.go` with appropriate precedence
|
||||
- Add the lexer pattern in `lexer_participle.go`
|
||||
- Write comprehensive tests covering normal and edge cases
|
||||
- Create a documentation header in `doc/operators/headers/`
|
||||
- Use `Context.ChildContext()` for proper context threading
|
||||
- Handle all node types gracefully
|
||||
- Return meaningful error messages
|
||||
|
||||
❌ **DON'T:**
|
||||
- Modify `candidate_node.go` (operators shouldn't need this)
|
||||
- Modify core navigation or evaluation logic
|
||||
- Bypass the handler function pattern
|
||||
- Add format-specific or operator-specific fields to `CandidateNode`
|
||||
- Skip tests or documentation
|
||||
|
||||
## Examples
|
||||
|
||||
Refer to existing operator implementations for patterns:
|
||||
|
||||
- **No-argument operator**: `operator_reverse.go` - Processes arrays/sequences
|
||||
- **Single-argument operator**: `operator_map.go` - Takes an expression argument
|
||||
- **Complex multi-output**: `operator_keys.go` - Produces multiple results
|
||||
- **With preferences**: `operator_to_number.go` - Configuration options
|
||||
- **Error handling**: `operator_error.go` - Control flow with errors
|
||||
- **String operations**: `operator_strings.go` - Multiple related operators
|
||||
|
||||
## Testing Patterns
|
||||
|
||||
Refer to existing test files for specific patterns:
|
||||
- Basic expression tests in `operator_reverse_test.go`
|
||||
- Multi-output tests in `operator_keys_test.go`
|
||||
- Error handling tests in `operator_error_test.go`
|
||||
- Tests with `skipDoc` flag to exclude from generated documentation
|
||||
|
||||
## Common Patterns
|
||||
|
||||
Refer to existing operator implementations for these patterns:
|
||||
- Simple transformation: see `operator_reverse.go`
|
||||
- Type checking: see `operator_error.go`
|
||||
- Working with arguments: see `operator_map.go`
|
||||
- Post-traversal operators: see `operator_with.go`
|
||||
- As of the current tree, `pkg/yqlib/ini_test.go` calls `NewINIDecoder()` without required `INIPreferences`, which breaks unit-test compilation in `make local check` / `make local test`. The **binary still builds** (`go build -o yq .`) and **all acceptance tests pass** (`bash scripts/acceptance.sh`).
|
||||
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.26.5@sha256:079e59808d2d252516e27e3f3a9c003740dee7f75e55aa71528766d52bcfc16a AS builder
|
||||
FROM golang:1.26.4@sha256:68cb6d68bed024785b69195b89af7ac7a444f27791435f98647edff595aa0479 AS builder
|
||||
|
||||
WORKDIR /go/src/mikefarah/yq
|
||||
|
||||
@@ -10,7 +10,7 @@ RUN ./scripts/acceptance.sh
|
||||
|
||||
# Choose alpine as a base image to make this useful for CI, as many
|
||||
# CI tools expect an interactive shell inside the container
|
||||
FROM alpine:3@sha256:28bd5fe8b56d1bd048e5babf5b10710ebe0bae67db86916198a6eec434943f8b AS production
|
||||
FROM alpine:3@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 AS production
|
||||
LABEL maintainer="Mike Farah <mikefarah@users.noreply.github.com>"
|
||||
|
||||
COPY --from=builder /go/src/mikefarah/yq/yq /usr/bin/yq
|
||||
|
||||
+5
-1
@@ -1,4 +1,8 @@
|
||||
FROM golang:1.26.5@sha256:079e59808d2d252516e27e3f3a9c003740dee7f75e55aa71528766d52bcfc16a
|
||||
FROM golang:1.26.4@sha256:68cb6d68bed024785b69195b89af7ac7a444f27791435f98647edff595aa0479
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y npm && \
|
||||
npm install -g npx cspell@latest
|
||||
|
||||
COPY scripts/devtools.sh /opt/devtools.sh
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ quiet: # this is silly but shuts up 'Nothing to be done for `local`'
|
||||
@:
|
||||
|
||||
prepare: tmp/dev_image_id
|
||||
tmp/dev_image_id: Dockerfile.dev scripts/devtools.sh _typos.toml
|
||||
tmp/dev_image_id: Dockerfile.dev scripts/devtools.sh
|
||||
@mkdir -p tmp
|
||||
@${ENGINE} rmi -f ${DEV_IMAGE} > /dev/null 2>&1 || true
|
||||
@${ENGINE} build -t ${DEV_IMAGE} -f Dockerfile.dev .
|
||||
|
||||
-20
@@ -1,20 +0,0 @@
|
||||
[files]
|
||||
extend-exclude = ["vendor", "bin"]
|
||||
|
||||
[default]
|
||||
locale = "en"
|
||||
extend-ignore-identifiers-re = [
|
||||
"NdJson",
|
||||
]
|
||||
|
||||
[default.extend-identifiers]
|
||||
AttributeIDSupressMenu = "AttributeIDSupressMenu"
|
||||
|
||||
[default.extend-words]
|
||||
Teh = "Teh"
|
||||
teh = "teh"
|
||||
Supress = "Supress"
|
||||
HashiCorp = "HashiCorp"
|
||||
Hashi = "Hashi"
|
||||
fot = "fot"
|
||||
nd = "nd"
|
||||
+1
-1
@@ -12,6 +12,6 @@ outputs:
|
||||
description: "The complete result from the yq command being run"
|
||||
runs:
|
||||
using: 'docker'
|
||||
image: 'docker://mikefarah/yq:4-githubaction@sha256:e1b8c865f299ea6b02910a7ddf147d5d431244d4cc116f89c2148c9f53822906'
|
||||
image: 'docker://mikefarah/yq:4-githubaction'
|
||||
args:
|
||||
- ${{ inputs.cmd }}
|
||||
|
||||
@@ -0,0 +1,422 @@
|
||||
# General rules
|
||||
✅ **DO:**
|
||||
- You can use ./yq with the `--debug-node-info` flag to get a deeper understanding of the ast.
|
||||
- run ./scripts/format.sh to format the code; then ./scripts/check.sh lint and finally ./scripts/spelling.sh to check spelling.
|
||||
- Add comprehensive tests to cover the changes
|
||||
- Run test suite to ensure there is no regression
|
||||
- Use UK english spelling
|
||||
|
||||
❌ **DON'T:**
|
||||
- Git add or commit
|
||||
- Add comments to functions that are self-explanatory
|
||||
|
||||
|
||||
|
||||
# Adding a New Encoder/Decoder
|
||||
|
||||
This guide explains how to add support for a new format (encoder/decoder) to yq without modifying `candidate_node.go`.
|
||||
|
||||
## Overview
|
||||
|
||||
The encoder/decoder architecture in yq is based on two main interfaces:
|
||||
|
||||
- **Encoder**: Converts a `CandidateNode` to output in a specific format
|
||||
- **Decoder**: Reads input in a specific format and creates a `CandidateNode`
|
||||
|
||||
Each format is registered in `pkg/yqlib/format.go` and made available through factory functions.
|
||||
|
||||
## Architecture
|
||||
|
||||
### Key Files
|
||||
|
||||
- `pkg/yqlib/encoder.go` - Defines the `Encoder` interface
|
||||
- `pkg/yqlib/decoder.go` - Defines the `Decoder` interface
|
||||
- `pkg/yqlib/format.go` - Format registry and factory functions
|
||||
- `pkg/yqlib/operator_encoder_decoder.go` - Encode/decode operators
|
||||
- `pkg/yqlib/encoder_*.go` - Encoder implementations
|
||||
- `pkg/yqlib/decoder_*.go` - Decoder implementations
|
||||
|
||||
### Interfaces
|
||||
|
||||
**Encoder Interface:**
|
||||
```go
|
||||
type Encoder interface {
|
||||
Encode(writer io.Writer, node *CandidateNode) error
|
||||
PrintDocumentSeparator(writer io.Writer) error
|
||||
PrintLeadingContent(writer io.Writer, content string) error
|
||||
CanHandleAliases() bool
|
||||
}
|
||||
```
|
||||
|
||||
**Decoder Interface:**
|
||||
```go
|
||||
type Decoder interface {
|
||||
Init(reader io.Reader) error
|
||||
Decode() (*CandidateNode, error)
|
||||
}
|
||||
```
|
||||
|
||||
## Step-by-Step: Adding a New Encoder/Decoder
|
||||
|
||||
### Step 1: Create the Encoder File
|
||||
|
||||
Create `pkg/yqlib/encoder_<format>.go` implementing the `Encoder` interface:
|
||||
- `Encode()` - Convert a `CandidateNode` to your format and write to the output writer
|
||||
- `PrintDocumentSeparator()` - Handle document separators if your format requires them
|
||||
- `PrintLeadingContent()` - Handle leading content/comments if supported
|
||||
- `CanHandleAliases()` - Return whether your format supports YAML aliases
|
||||
|
||||
See `encoder_json.go` or `encoder_base64.go` for examples.
|
||||
|
||||
### Step 2: Create the Decoder File
|
||||
|
||||
Create `pkg/yqlib/decoder_<format>.go` implementing the `Decoder` interface:
|
||||
- `Init()` - Initialize the decoder with the input reader and set up any needed state
|
||||
- `Decode()` - Decode one document from the input and return a `CandidateNode`, or `io.EOF` when finished
|
||||
|
||||
See `decoder_json.go` or `decoder_base64.go` for examples.
|
||||
|
||||
### Step 3: Create Tests (Mandatory)
|
||||
|
||||
Create a test file `pkg/yqlib/<format>_test.go` using the `formatScenario` pattern:
|
||||
- Define test scenarios as `formatScenario` structs with fields: `description`, `input`, `expected`, `scenarioType`
|
||||
- `scenarioType` can be `"decode"` (test decoding to YAML) or `"roundtrip"` (encode/decode preservation)
|
||||
- Create a helper function `test<Format>Scenario()` that switches on `scenarioType`
|
||||
- Create main test function `Test<Format>FormatScenarios()` that iterates over scenarios
|
||||
- The main test function should use `documentScenarios` to ensure testcase documentation is generated.
|
||||
|
||||
Test coverage must include:
|
||||
- Basic data types (scalars, arrays, objects/maps)
|
||||
- Nested structures
|
||||
- Edge cases (empty inputs, special characters, escape sequences)
|
||||
- Format-specific features or syntax
|
||||
- Round-trip tests: decode → encode → decode should preserve data
|
||||
|
||||
See `hcl_test.go` for a complete example.
|
||||
|
||||
### Step 4: Register the Format in format.go
|
||||
|
||||
Edit `pkg/yqlib/format.go`:
|
||||
|
||||
1. Add a new format variable:
|
||||
- `"<format>"` is the formal name (e.g., "json", "yaml")
|
||||
- `[]string{...}` contains short aliases (can be empty)
|
||||
- The first function creates an encoder (can be nil for encode-only formats)
|
||||
- The second function creates a decoder (can be nil for decode-only formats)
|
||||
|
||||
2. Add the format to the `Formats` slice in the same file
|
||||
|
||||
See existing formats in `format.go` for the exact structure.
|
||||
|
||||
### Step 5: Handle Encoder Configuration (if needed)
|
||||
|
||||
If your format has preferences/configuration options:
|
||||
|
||||
1. Create a preferences struct with your configuration fields
|
||||
2. Update the encoder to accept preferences in its factory function
|
||||
3. Update `format.go` to pass the configured preferences
|
||||
4. Update `operator_encoder_decoder.go` if special indent handling is needed (see existing formats like JSON and YAML for the pattern)
|
||||
|
||||
This pattern is optional and only needed if your format has user-configurable options.
|
||||
|
||||
## Build Tags
|
||||
|
||||
Use build tags to allow optional compilation of formats:
|
||||
- Add `//go:build !yq_no<format>` at the top of your encoder and decoder files
|
||||
- Create a no-build version in `pkg/yqlib/no_<format>.go` that returns nil for encoder/decoder factories
|
||||
|
||||
This allows users to compile yq without certain formats using: `go build -tags yq_no<format>`
|
||||
|
||||
## Working with CandidateNode
|
||||
|
||||
The `CandidateNode` struct represents a YAML node with:
|
||||
- `Kind`: The node type (ScalarNode, SequenceNode, MappingNode)
|
||||
- `Tag`: The YAML tag (e.g., "!!str", "!!int", "!!map")
|
||||
- `Value`: The scalar value (for ScalarNode only)
|
||||
- `Content`: Child nodes (for SequenceNode and MappingNode)
|
||||
|
||||
Key methods:
|
||||
- `node.guessTagFromCustomType()` - Infer the tag from Go type
|
||||
- `node.AsList()` - Convert to a list for processing
|
||||
- `node.CreateReplacement()` - Create a new replacement node
|
||||
- `NewCandidate()` - Create a new CandidateNode
|
||||
|
||||
## Key Points
|
||||
|
||||
✅ **DO:**
|
||||
- Implement only the `Encoder` and `Decoder` interfaces
|
||||
- Register your format in `format.go` only
|
||||
- Keep format-specific logic in your encoder/decoder files
|
||||
- Use the candidate_node style attribute to store style information for round-trip. Ask if this needs to be updated with new styles.
|
||||
- Use build tags for optional compilation
|
||||
- Add comprehensive tests
|
||||
- Run the specific encoder/decoder test (e.g. <format>_test.go) whenever you make ay changes to the encoder_<format> or decoder_<format>
|
||||
- Handle errors gracefully
|
||||
- Add the no build directive, like the xml encoder and decoder, that enables a minimal yq builds. e.g. `//go:build !yq_<format>`. Be sure to also update the build_small-yq.sh and build-tinygo-yq.sh to not include the new format.
|
||||
|
||||
❌ **DON'T:**
|
||||
- Modify `candidate_node.go` to add format-specific logic
|
||||
- Add format-specific fields to `CandidateNode`
|
||||
- Create special cases in core navigation or evaluation logic
|
||||
- Bypass the encoder/decoder interfaces
|
||||
- Use candidate_node tag attribute for anything other than indicate the data type
|
||||
|
||||
## Examples
|
||||
|
||||
Refer to existing format implementations for patterns:
|
||||
|
||||
- **Simple encoder/decoder**: `encoder_json.go`, `decoder_json.go`
|
||||
- **Complex with preferences**: `encoder_yaml.go`, `decoder_yaml.go`
|
||||
- **Encoder-only**: `encoder_sh.go` (ShFormat has nil decoder)
|
||||
- **String-only operations**: `encoder_base64.go`, `decoder_base64.go`
|
||||
|
||||
## Testing Your Implementation (Mandatory)
|
||||
|
||||
Tests must be implemented in `<format>_test.go` following the `formatScenario` pattern:
|
||||
|
||||
1. **Create test scenarios** using the `formatScenario` struct with fields:
|
||||
- `description`: Brief description of what's being tested
|
||||
- `input`: Sample input in your format
|
||||
- `expected`: Expected output (typically in YAML for decode tests)
|
||||
- `scenarioType`: Either `"decode"` or `"roundtrip"`
|
||||
|
||||
2. **Test coverage must include:**
|
||||
- Basic data types (scalars, arrays, objects/maps)
|
||||
- Nested structures
|
||||
- Edge cases (empty inputs, special characters, escape sequences)
|
||||
- Format-specific features or syntax
|
||||
- Round-trip tests: decode → encode → decode should preserve data
|
||||
|
||||
3. **Test function pattern:**
|
||||
- `test<Format>Scenario()`: Helper function that switches on `scenarioType`
|
||||
- `Test<Format>FormatScenarios()`: Main test function that iterates over scenarios
|
||||
|
||||
4. **Example from existing formats:**
|
||||
- See `hcl_test.go` for a complete example
|
||||
- See `yaml_test.go` for YAML-specific patterns
|
||||
- See `json_test.go` for more complex scenarios
|
||||
|
||||
## Common Patterns
|
||||
|
||||
### Format with Indentation
|
||||
Use preferences to control output formatting:
|
||||
```go
|
||||
type <format>Preferences struct {
|
||||
Indent int
|
||||
}
|
||||
|
||||
func (prefs *<format>Preferences) Copy() <format>Preferences {
|
||||
return *prefs
|
||||
}
|
||||
```
|
||||
|
||||
### Multiple Documents
|
||||
Decoders should support reading multiple documents:
|
||||
```go
|
||||
func (dec *<format>Decoder) Decode() (*CandidateNode, error) {
|
||||
if dec.finished {
|
||||
return nil, io.EOF
|
||||
}
|
||||
// ... decode next document ...
|
||||
if noMoreDocuments {
|
||||
dec.finished = true
|
||||
}
|
||||
return candidate, nil
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
# Adding a New Operator
|
||||
|
||||
This guide explains how to add a new operator to yq. Operators are the core of yq's expression language and process `CandidateNode` objects without requiring modifications to `candidate_node.go` itself.
|
||||
|
||||
## Overview
|
||||
|
||||
Operators transform data by implementing a handler function that processes a `Context` containing `CandidateNode` objects. Each operator is:
|
||||
|
||||
1. Defined as an `operationType` in `operation.go`
|
||||
2. Registered in the lexer in `lexer_participle.go`
|
||||
3. Implemented in its own `operator_<type>.go` file
|
||||
4. Tested in `operator_<type>_test.go`
|
||||
5. Documented in `pkg/yqlib/doc/operators/headers/<type>.md`
|
||||
|
||||
## Architecture
|
||||
|
||||
### Key Files
|
||||
|
||||
- `pkg/yqlib/operation.go` - Defines `operationType` and operator registry
|
||||
- `pkg/yqlib/lexer_participle.go` - Registers operators with their syntax patterns
|
||||
- `pkg/yqlib/operator_<type>.go` - Operator implementation
|
||||
- `pkg/yqlib/operator_<type>_test.go` - Operator tests using `expressionScenario`
|
||||
- `pkg/yqlib/doc/operators/headers/<type>.md` - Documentation header
|
||||
|
||||
### Core Types
|
||||
|
||||
**operationType:**
|
||||
```go
|
||||
type operationType struct {
|
||||
Type string // Unique operator name (e.g., "REVERSE")
|
||||
NumArgs uint // Number of arguments (0 for no args)
|
||||
Precedence uint // Operator precedence (higher = higher precedence)
|
||||
Handler operatorHandler // The function that executes the operator
|
||||
CheckForPostTraverse bool // Whether to apply post-traversal logic
|
||||
ToString func(*Operation) string // Custom string representation
|
||||
}
|
||||
```
|
||||
|
||||
**operatorHandler signature:**
|
||||
```go
|
||||
type operatorHandler func(*dataTreeNavigator, Context, *ExpressionNode) (Context, error)
|
||||
```
|
||||
|
||||
**expressionScenario for tests:**
|
||||
```go
|
||||
type expressionScenario struct {
|
||||
description string
|
||||
subdescription string
|
||||
document string
|
||||
expression string
|
||||
expected []string
|
||||
skipDoc bool
|
||||
expectedError string
|
||||
}
|
||||
```
|
||||
|
||||
## Step-by-Step: Adding a New Operator
|
||||
|
||||
### Step 1: Create the Operator Implementation File
|
||||
|
||||
Create `pkg/yqlib/operator_<type>.go` implementing the operator handler function:
|
||||
- Implement the `operatorHandler` function signature
|
||||
- Process nodes from `context.MatchingNodes`
|
||||
- Return a new `Context` with results using `context.ChildContext()`
|
||||
- Use `candidate.CreateReplacement()` or `candidate.CreateReplacementWithComments()` to create new nodes
|
||||
- Handle errors gracefully with meaningful error messages
|
||||
|
||||
See `operator_reverse.go` or `operator_keys.go` for examples.
|
||||
|
||||
### Step 2: Register the Operator in operation.go
|
||||
|
||||
Add the operator type definition to `pkg/yqlib/operation.go`:
|
||||
|
||||
```go
|
||||
var <type>OpType = &operationType{
|
||||
Type: "<TYPE>", // All caps, matches pattern in lexer
|
||||
NumArgs: 0, // 0 for no args, 1+ for args
|
||||
Precedence: 50, // Typical range: 40-55
|
||||
Handler: <type>Operator, // Reference to handler function
|
||||
}
|
||||
```
|
||||
|
||||
**Precedence guidelines:**
|
||||
- 10-20: Logical operators (OR, AND, UNION)
|
||||
- 30: Pipe operator
|
||||
- 40: Assignment and comparison operators
|
||||
- 42: Arithmetic operators (ADD, SUBTRACT, MULTIPLY, DIVIDE)
|
||||
- 50-52: Most other operators
|
||||
- 55: High precedence (e.g., GET_VARIABLE)
|
||||
|
||||
**Optional fields:**
|
||||
- `CheckForPostTraverse: true` - If your operator can have another directly after it without the pipe character. Most of the time this is false.
|
||||
- `ToString: customToString` - Custom string representation (rarely needed)
|
||||
|
||||
### Step 3: Register the Operator in lexer_participle.go
|
||||
|
||||
Edit `pkg/yqlib/lexer_participle.go` to add the operator to the lexer rules:
|
||||
- Use `simpleOp()` for simple keyword patterns
|
||||
- Use object syntax for regex patterns or complex syntax
|
||||
- Support optional characters with `_?` and aliases with `|`
|
||||
|
||||
See existing operators in `lexer_participle.go` for pattern examples.
|
||||
|
||||
### Step 4: Create Tests (Mandatory)
|
||||
|
||||
Create `pkg/yqlib/operator_<type>_test.go` using the `expressionScenario` pattern:
|
||||
- Define test scenarios with `description`, `document`, `expression`, and `expected` fields
|
||||
- `expected` is a slice of strings showing output format: `"D<doc>, P[<path>], (<tag>)::<value>\n"`
|
||||
- Set `skipDoc: true` for edge cases you don't want in generated documentation
|
||||
- Include `subdescription` for longer test names
|
||||
- Set `expectedError` if testing error cases
|
||||
- Create main test function that iterates over scenarios
|
||||
- The main test function should use `documentScenarios` to ensure testcase documentation is generated.
|
||||
|
||||
Test coverage must include:
|
||||
- Basic data types and nested structures
|
||||
- Edge cases (empty inputs, special characters, type errors)
|
||||
- Multiple outputs if applicable
|
||||
- Format-specific features
|
||||
|
||||
See `operator_reverse_test.go` for a simple example and `operator_keys_test.go` for complex cases.
|
||||
|
||||
### Step 5: Create Documentation Header
|
||||
|
||||
Create `pkg/yqlib/doc/operators/headers/<type>.md`:
|
||||
- Use the exact operator name as the title
|
||||
- Include a concise 1-2 sentence summary
|
||||
- Add additional context or examples if the operator is complex
|
||||
|
||||
See existing headers in `doc/operators/headers/` for examples.
|
||||
|
||||
## Working with Context and CandidateNode
|
||||
|
||||
### Context Management
|
||||
- `context.ChildContext(results)` - Create child context with results
|
||||
- `context.GetVariable("varName")` - Get variables stored in context
|
||||
- `context.SetVariable("varName", value)` - Set variables in context
|
||||
|
||||
### CandidateNode Operations
|
||||
- `candidate.CreateReplacement(ScalarNode, "!!str", stringValue)` - Create a replacement node
|
||||
- `candidate.CreateReplacementWithComments(SequenceNode, "!!seq", candidate.Style)` - With style preserved
|
||||
- `candidate.Kind` - The node type (ScalarNode, SequenceNode, MappingNode)
|
||||
- `candidate.Tag` - The YAML tag (!!str, !!int, etc.)
|
||||
- `candidate.Value` - The scalar value (for ScalarNode only)
|
||||
- `candidate.Content` - Child nodes (for SequenceNode and MappingNode)
|
||||
- `candidate.guessTagFromCustomType()` - Infer the tag from Go type
|
||||
- `candidate.AsList()` - Convert to a list representation
|
||||
|
||||
## Key Points
|
||||
|
||||
✅ **DO:**
|
||||
- Implement the operator handler with the correct signature
|
||||
- Register in `operation.go` with appropriate precedence
|
||||
- Add the lexer pattern in `lexer_participle.go`
|
||||
- Write comprehensive tests covering normal and edge cases
|
||||
- Create a documentation header in `doc/operators/headers/`
|
||||
- Use `Context.ChildContext()` for proper context threading
|
||||
- Handle all node types gracefully
|
||||
- Return meaningful error messages
|
||||
|
||||
❌ **DON'T:**
|
||||
- Modify `candidate_node.go` (operators shouldn't need this)
|
||||
- Modify core navigation or evaluation logic
|
||||
- Bypass the handler function pattern
|
||||
- Add format-specific or operator-specific fields to `CandidateNode`
|
||||
- Skip tests or documentation
|
||||
|
||||
## Examples
|
||||
|
||||
Refer to existing operator implementations for patterns:
|
||||
|
||||
- **No-argument operator**: `operator_reverse.go` - Processes arrays/sequences
|
||||
- **Single-argument operator**: `operator_map.go` - Takes an expression argument
|
||||
- **Complex multi-output**: `operator_keys.go` - Produces multiple results
|
||||
- **With preferences**: `operator_to_number.go` - Configuration options
|
||||
- **Error handling**: `operator_error.go` - Control flow with errors
|
||||
- **String operations**: `operator_strings.go` - Multiple related operators
|
||||
|
||||
## Testing Patterns
|
||||
|
||||
Refer to existing test files for specific patterns:
|
||||
- Basic expression tests in `operator_reverse_test.go`
|
||||
- Multi-output tests in `operator_keys_test.go`
|
||||
- Error handling tests in `operator_error_test.go`
|
||||
- Tests with `skipDoc` flag to exclude from generated documentation
|
||||
|
||||
## Common Patterns
|
||||
|
||||
Refer to existing operator implementations for these patterns:
|
||||
- Simple transformation: see `operator_reverse.go`
|
||||
- Type checking: see `operator_error.go`
|
||||
- Working with arguments: see `operator_map.go`
|
||||
- Post-traversal operators: see `operator_with.go`
|
||||
@@ -87,10 +87,6 @@ func validateCommandFlags(args []string) error {
|
||||
return fmt.Errorf("cannot pass files in when using null-input flag")
|
||||
}
|
||||
|
||||
if indent < 0 {
|
||||
return fmt.Errorf("indent must not be negative")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -1086,7 +1086,6 @@ func TestValidateCommandFlags(t *testing.T) {
|
||||
frontMatter string
|
||||
splitFileExp string
|
||||
nullInput bool
|
||||
indent int
|
||||
expectError bool
|
||||
errorContains string
|
||||
}{
|
||||
@@ -1149,27 +1148,6 @@ func TestValidateCommandFlags(t *testing.T) {
|
||||
expectError: true,
|
||||
errorContains: "cannot pass files in when using null-input flag",
|
||||
},
|
||||
{
|
||||
name: "negative indent",
|
||||
args: []string{"file.yaml"},
|
||||
writeInplace: false,
|
||||
frontMatter: "",
|
||||
splitFileExp: "",
|
||||
nullInput: false,
|
||||
indent: -1,
|
||||
expectError: true,
|
||||
errorContains: "indent must not be negative",
|
||||
},
|
||||
{
|
||||
name: "zero indent is valid",
|
||||
args: []string{"file.yaml"},
|
||||
writeInplace: false,
|
||||
frontMatter: "",
|
||||
splitFileExp: "",
|
||||
nullInput: false,
|
||||
indent: 0,
|
||||
expectError: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
@@ -1179,20 +1157,17 @@ func TestValidateCommandFlags(t *testing.T) {
|
||||
originalFrontMatter := frontMatter
|
||||
originalSplitFileExp := splitFileExp
|
||||
originalNullInput := nullInput
|
||||
originalIndent := indent
|
||||
defer func() {
|
||||
writeInplace = originalWriteInplace
|
||||
frontMatter = originalFrontMatter
|
||||
splitFileExp = originalSplitFileExp
|
||||
nullInput = originalNullInput
|
||||
indent = originalIndent
|
||||
}()
|
||||
|
||||
writeInplace = tt.writeInplace
|
||||
frontMatter = tt.frontMatter
|
||||
splitFileExp = tt.splitFileExp
|
||||
nullInput = tt.nullInput
|
||||
indent = tt.indent
|
||||
|
||||
err := validateCommandFlags(tt.args)
|
||||
if tt.expectError {
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@ var (
|
||||
GitDescribe string
|
||||
|
||||
// Version is main version number that is being run at the moment.
|
||||
Version = "v4.53.4"
|
||||
Version = "v4.53.2"
|
||||
|
||||
// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
|
||||
// then it means that it is a final release. Otherwise, this is a pre-release
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
---
|
||||
$schema: https://raw.githubusercontent.com/streetsidesoftware/cspell/main/cspell.schema.json
|
||||
version: '0.2'
|
||||
language: en-GB
|
||||
dictionaryDefinitions:
|
||||
- name: project-words
|
||||
path: './project-words.txt'
|
||||
addWords: true
|
||||
dictionaries:
|
||||
- project-words
|
||||
ignorePaths:
|
||||
- 'vendor'
|
||||
- 'bin'
|
||||
- '/project-words.txt'
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM mikefarah/yq:4@sha256:11a1f0b604b13dbbdc662260d8db6f644b22d8553122a25c1b5b2e8713ca6977
|
||||
FROM mikefarah/yq:4@sha256:603ebff15eb308a05f1c5b8b7613179cad859aed3ec9fdd04f2ef5d32345950e
|
||||
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ module github.com/mikefarah/yq/v4
|
||||
require (
|
||||
github.com/a8m/envsubst v1.4.3
|
||||
github.com/alecthomas/participle/v2 v2.1.4
|
||||
github.com/alecthomas/repr v0.5.4
|
||||
github.com/alecthomas/repr v0.5.2
|
||||
github.com/dimchansky/utfbom v1.1.1
|
||||
github.com/elliotchance/orderedmap v1.8.0
|
||||
github.com/fatih/color v1.19.0
|
||||
@@ -12,31 +12,30 @@ require (
|
||||
github.com/goccy/go-yaml v1.19.2
|
||||
github.com/hashicorp/hcl/v2 v2.24.0
|
||||
github.com/jinzhu/copier v0.4.0
|
||||
github.com/magiconair/properties v1.18.11
|
||||
github.com/pelletier/go-toml/v2 v2.4.3
|
||||
github.com/magiconair/properties v1.8.10
|
||||
github.com/pelletier/go-toml/v2 v2.3.1
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/spf13/pflag v1.0.10
|
||||
github.com/yuin/gopher-lua v1.1.2
|
||||
github.com/zclconf/go-cty v1.19.0
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6
|
||||
golang.org/x/mod v0.38.0
|
||||
golang.org/x/net v0.57.0
|
||||
golang.org/x/text v0.40.0
|
||||
github.com/zclconf/go-cty v1.18.1
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.4
|
||||
golang.org/x/mod v0.36.0
|
||||
golang.org/x/net v0.55.0
|
||||
golang.org/x/text v0.37.0
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/agext/levenshtein v1.2.1 // indirect
|
||||
github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect
|
||||
github.com/apparentlymart/go-textseg/v17 v17.0.1 // indirect
|
||||
github.com/google/go-cmp v0.6.0 // indirect
|
||||
github.com/inconshreveable/mousetrap v1.1.0 // indirect
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
|
||||
golang.org/x/sync v0.22.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/tools v0.47.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/sys v0.45.0 // indirect
|
||||
golang.org/x/tools v0.44.0 // indirect
|
||||
)
|
||||
|
||||
go 1.25.0
|
||||
|
||||
@@ -6,12 +6,10 @@ github.com/alecthomas/assert/v2 v2.11.0 h1:2Q9r3ki8+JYXvGsDyBXwH3LcJ+WK5D0gc5E8v
|
||||
github.com/alecthomas/assert/v2 v2.11.0/go.mod h1:Bze95FyfUr7x34QZrjL+XP+0qgp/zg8yS+TtBj1WA3k=
|
||||
github.com/alecthomas/participle/v2 v2.1.4 h1:W/H79S8Sat/krZ3el6sQMvMaahJ+XcM9WSI2naI7w2U=
|
||||
github.com/alecthomas/participle/v2 v2.1.4/go.mod h1:8tqVbpTX20Ru4NfYQgZf4mP18eXPTBViyMWiArNEgGI=
|
||||
github.com/alecthomas/repr v0.5.4 h1:OVP7JEcuzU9CCDsT6STCr3rg17oQfWILtPWd2EG0uN4=
|
||||
github.com/alecthomas/repr v0.5.4/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
|
||||
github.com/alecthomas/repr v0.5.2 h1:SU73FTI9D1P5UNtvseffFSGmdNci/O6RsqzeXJtP0Qs=
|
||||
github.com/alecthomas/repr v0.5.2/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
|
||||
github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
|
||||
github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
|
||||
github.com/apparentlymart/go-textseg/v17 v17.0.1 h1:bpMXRgQ5cEoRNuQke1a80/Nl6w3G5eoIbWo9f3gXkAs=
|
||||
github.com/apparentlymart/go-textseg/v17 v17.0.1/go.mod h1:fa8X4jgGeevslICIY6LcdjkSecWnXmYd9Lk34z/VxZs=
|
||||
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
@@ -40,16 +38,16 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
|
||||
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
|
||||
github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8=
|
||||
github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
|
||||
github.com/magiconair/properties v1.18.11 h1:j5ozYZl0zCjG7ahMDH0GWIobOvvUzT0BdAguG0ViKy0=
|
||||
github.com/magiconair/properties v1.18.11/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||
github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
|
||||
github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||
github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
|
||||
github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
|
||||
github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
|
||||
github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
|
||||
github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pelletier/go-toml/v2 v2.3.1 h1:MYEvvGnQjeNkRF1qUuGolNtNExTDwct51yp7olPtrEc=
|
||||
github.com/pelletier/go-toml/v2 v2.3.1/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
@@ -65,26 +63,26 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/yuin/gopher-lua v1.1.2 h1:yF/FjE3hD65tBbt0VXLE13HWS9h34fdzJmrWRXwobGA=
|
||||
github.com/yuin/gopher-lua v1.1.2/go.mod h1:7aRmXIWl37SqRf0koeyylBEzJ+aPt8A+mmkQ4f1ntR8=
|
||||
github.com/zclconf/go-cty v1.19.0 h1:IV8WdqYZc2c5rLX9bEoLNXKojBAp0MZPBHMIrCoa/s4=
|
||||
github.com/zclconf/go-cty v1.19.0/go.mod h1:12W89jGn3JCOIQi7infWr9m80rOkb5RNYJqXMZcN4c8=
|
||||
github.com/zclconf/go-cty v1.18.1 h1:yEGE8M4iIZlyKQURZNb2SnEyZlZHUcBCnx6KF81KuwM=
|
||||
github.com/zclconf/go-cty v1.18.1/go.mod h1:qpnV6EDNgC1sns/AleL1fvatHw72j+S+nS+MJ+T2CSg=
|
||||
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo=
|
||||
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||
golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk=
|
||||
golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40=
|
||||
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
|
||||
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.4 h1:UP4+v6fFrBIb1l934bDl//mmnoIZEDK0idg1+AIvX5U=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.4/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||
golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4=
|
||||
golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ=
|
||||
golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
|
||||
golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
||||
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
||||
golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q=
|
||||
golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA=
|
||||
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
|
||||
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
|
||||
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
|
||||
golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c=
|
||||
golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -1,5 +1,3 @@
|
||||
//go:build goinstall
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -13,10 +11,6 @@ import (
|
||||
// TestGoInstallCompatibility ensures the module can be zipped for go install.
|
||||
// This is an integration test that uses the same zip.CreateFromDir function
|
||||
// that go install uses internally. If this test fails, go install will fail.
|
||||
//
|
||||
// Built with the goinstall tag and run after the main test suite (see scripts/test.sh)
|
||||
// so it does not race with pkg/yqlib tests that rewrite doc/*.md during execution.
|
||||
//
|
||||
// See: https://github.com/mikefarah/yq/issues/2587
|
||||
func TestGoInstallCompatibility(t *testing.T) {
|
||||
mod := module.Version{
|
||||
|
||||
@@ -49,13 +49,12 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
|
||||
|
||||
var allDocuments = list.New()
|
||||
for _, filename := range filenames {
|
||||
reader, cleanup, err := readStream(filename)
|
||||
reader, err := readStream(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fileDocuments, err := readDocuments(reader, filename, fileIndex, decoder)
|
||||
cleanup()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -46,11 +46,7 @@ const (
|
||||
func createStringScalarNode(stringValue string) *CandidateNode {
|
||||
var node = &CandidateNode{Kind: ScalarNode}
|
||||
node.Value = stringValue
|
||||
if stringValue == "<<" {
|
||||
node.Tag = "!!merge"
|
||||
} else {
|
||||
node.Tag = "!!str"
|
||||
}
|
||||
node.Tag = "!!str"
|
||||
return node
|
||||
}
|
||||
|
||||
|
||||
@@ -106,31 +106,6 @@ func TestCreateScalarNodeScenarios(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
type createStringScalarNodeScenario struct {
|
||||
stringValue string
|
||||
expectedTag string
|
||||
}
|
||||
|
||||
var createStringScalarNodeScenarios = []createStringScalarNodeScenario{
|
||||
{
|
||||
stringValue: "yourKey",
|
||||
expectedTag: "!!str",
|
||||
},
|
||||
{
|
||||
stringValue: "<<",
|
||||
expectedTag: "!!merge",
|
||||
},
|
||||
}
|
||||
|
||||
func TestCreateStringScalarNodeScenarios(t *testing.T) {
|
||||
for _, tt := range createStringScalarNodeScenarios {
|
||||
actual := createStringScalarNode(tt.stringValue)
|
||||
test.AssertResultWithContext(t, tt.stringValue, actual.Value, fmt.Sprintf("Value for: %v", tt.stringValue))
|
||||
test.AssertResultWithContext(t, tt.expectedTag, actual.Tag, fmt.Sprintf("Tag for: %v", tt.stringValue))
|
||||
test.AssertResultWithContext(t, ScalarNode, actual.Kind, fmt.Sprintf("Kind for: %v", tt.stringValue))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetKeyForMapValue(t *testing.T) {
|
||||
key := createStringScalarNode("yourKey")
|
||||
n := CandidateNode{Key: key, Value: "meow", document: 3}
|
||||
|
||||
@@ -300,7 +300,8 @@ func convertHclExprToNode(expr hclsyntax.Expression, src []byte) *CandidateNode
|
||||
it := v.ElementIterator()
|
||||
for it.Next() {
|
||||
key, val := it.Element()
|
||||
keyNode := convertCtyValueToNode(key)
|
||||
keyStr := key.AsString()
|
||||
keyNode := createStringScalarNode(keyStr)
|
||||
valNode := convertCtyValueToNode(val)
|
||||
m.AddKeyValueChild(keyNode, valNode)
|
||||
}
|
||||
@@ -337,7 +338,8 @@ func convertHclExprToNode(expr hclsyntax.Expression, src []byte) *CandidateNode
|
||||
}
|
||||
continue
|
||||
}
|
||||
keyNode := convertCtyValueToNode(keyVal)
|
||||
keyStr := keyVal.AsString()
|
||||
keyNode := createStringScalarNode(keyStr)
|
||||
valNode := convertHclExprToNode(item.ValueExpr, src)
|
||||
m.AddKeyValueChild(keyNode, valNode)
|
||||
}
|
||||
@@ -458,7 +460,7 @@ func convertCtyValueToNode(v cty.Value) *CandidateNode {
|
||||
it := v.ElementIterator()
|
||||
for it.Next() {
|
||||
key, val := it.Element()
|
||||
keyNode := convertCtyValueToNode(key)
|
||||
keyNode := createStringScalarNode(key.AsString())
|
||||
valNode := convertCtyValueToNode(val)
|
||||
m.AddKeyValueChild(keyNode, valNode)
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names).
|
||||
|
||||
`yq` supports merge keys (like `<<: *blah`) from YAML 1.1. These are no longer part of the YAML 1.2 standard, but remain common in practice. Plain `<<:` keys are recognised as merge keys and round-trip as `<<:` without an explicit `!!merge` tag. When the source uses an explicit `!!merge` tag, that is preserved on output. Internally, when `yq` synthesises a `<<` map key (for example during merge operations), it tags the key as `!!merge` rather than `!!str`.
|
||||
`yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.
|
||||
|
||||
|
||||
## NOTE --yaml-fix-merge-anchor-to-spec flag
|
||||
@@ -32,7 +32,7 @@ Given a sample.yml file of:
|
||||
r: 10
|
||||
- &SMALL
|
||||
r: 1
|
||||
- <<: *CENTRE
|
||||
- !!merge <<: *CENTRE
|
||||
r: 10
|
||||
```
|
||||
then
|
||||
@@ -213,10 +213,10 @@ item_value: &item_value
|
||||
value: true
|
||||
thingOne:
|
||||
name: item_1
|
||||
<<: *item_value
|
||||
!!merge <<: *item_value
|
||||
thingTwo:
|
||||
name: item_2
|
||||
<<: *item_value
|
||||
!!merge <<: *item_value
|
||||
```
|
||||
then
|
||||
```bash
|
||||
@@ -231,7 +231,7 @@ thingOne:
|
||||
value: false
|
||||
thingTwo:
|
||||
name: item_2
|
||||
<<: *item_value
|
||||
!!merge <<: *item_value
|
||||
```
|
||||
|
||||
## LEGACY: Explode with merge anchors
|
||||
@@ -249,13 +249,13 @@ bar: &bar
|
||||
c: bar_c
|
||||
foobarList:
|
||||
b: foobarList_b
|
||||
<<:
|
||||
!!merge <<:
|
||||
- *foo
|
||||
- *bar
|
||||
c: foobarList_c
|
||||
foobar:
|
||||
c: foobar_c
|
||||
<<: *foo
|
||||
!!merge <<: *foo
|
||||
thing: foobar_thing
|
||||
```
|
||||
then
|
||||
@@ -298,7 +298,7 @@ Given a sample.yml file of:
|
||||
r: 10
|
||||
- &SMALL
|
||||
r: 1
|
||||
- <<:
|
||||
- !!merge <<:
|
||||
- *CENTRE
|
||||
- *BIG
|
||||
```
|
||||
@@ -328,7 +328,7 @@ Given a sample.yml file of:
|
||||
r: 10
|
||||
- &SMALL
|
||||
r: 1
|
||||
- <<:
|
||||
- !!merge <<:
|
||||
- *BIG
|
||||
- *LEFT
|
||||
- *SMALL
|
||||
@@ -361,13 +361,13 @@ bar: &bar
|
||||
c: bar_c
|
||||
foobarList:
|
||||
b: foobarList_b
|
||||
<<:
|
||||
!!merge <<:
|
||||
- *foo
|
||||
- *bar
|
||||
c: foobarList_c
|
||||
foobar:
|
||||
c: foobar_c
|
||||
<<: *foo
|
||||
!!merge <<: *foo
|
||||
thing: foobar_thing
|
||||
```
|
||||
then
|
||||
@@ -411,7 +411,7 @@ Given a sample.yml file of:
|
||||
r: 10
|
||||
- &SMALL
|
||||
r: 1
|
||||
- <<:
|
||||
- !!merge <<:
|
||||
- *CENTRE
|
||||
- *BIG
|
||||
```
|
||||
@@ -442,7 +442,7 @@ Given a sample.yml file of:
|
||||
r: 10
|
||||
- &SMALL
|
||||
r: 1
|
||||
- <<:
|
||||
- !!merge <<:
|
||||
- *BIG
|
||||
- *LEFT
|
||||
- *SMALL
|
||||
@@ -467,7 +467,7 @@ Given a sample.yml file of:
|
||||
```yaml
|
||||
a:
|
||||
b: &b 42
|
||||
<<:
|
||||
!!merge <<:
|
||||
c: *b
|
||||
```
|
||||
then
|
||||
|
||||
@@ -55,7 +55,7 @@ yq '.a = .a / 0 | .b = .b / 0' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
a: +Inf
|
||||
b: -Inf
|
||||
a: !!float +Inf
|
||||
b: !!float -Inf
|
||||
```
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
Use the `alias` and `anchor` operators to read and write yaml aliases and anchors. The `explode` operator normalises a yaml file (dereference (or expands) aliases and remove anchor names).
|
||||
|
||||
`yq` supports merge keys (like `<<: *blah`) from YAML 1.1. These are no longer part of the YAML 1.2 standard, but remain common in practice. Plain `<<:` keys are recognised as merge keys and round-trip as `<<:` without an explicit `!!merge` tag. When the source uses an explicit `!!merge` tag, that is preserved on output. Internally, when `yq` synthesises a `<<` map key (for example during merge operations), it tags the key as `!!merge` rather than `!!str`.
|
||||
`yq` supports merge aliases (like `<<: *blah`) however this is no longer in the standard yaml spec (1.2) and so `yq` will automatically add the `!!merge` tag to these nodes as it is effectively a custom tag.
|
||||
|
||||
|
||||
## NOTE --yaml-fix-merge-anchor-to-spec flag
|
||||
|
||||
@@ -34,7 +34,7 @@ yq '.a = .a % .b' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
a: 2
|
||||
a: !!float 2
|
||||
b: 2.5
|
||||
```
|
||||
|
||||
@@ -69,7 +69,7 @@ yq '.a = .a % .b' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
a: NaN
|
||||
a: !!float NaN
|
||||
b: 0
|
||||
```
|
||||
|
||||
|
||||
@@ -471,13 +471,13 @@ bar: &bar
|
||||
c: bar_c
|
||||
foobarList:
|
||||
b: foobarList_b
|
||||
<<:
|
||||
!!merge <<:
|
||||
- *foo
|
||||
- *bar
|
||||
c: foobarList_c
|
||||
foobar:
|
||||
c: foobar_c
|
||||
<<: *foo
|
||||
!!merge <<: *foo
|
||||
thing: foobar_thing
|
||||
```
|
||||
then
|
||||
@@ -487,7 +487,7 @@ yq '.foobar * .foobarList' sample.yml
|
||||
will output
|
||||
```yaml
|
||||
c: foobarList_c
|
||||
<<:
|
||||
!!merge <<:
|
||||
- *foo
|
||||
- *bar
|
||||
thing: foobar_thing
|
||||
|
||||
@@ -131,13 +131,13 @@ bar: &bar
|
||||
c: bar_c
|
||||
foobarList:
|
||||
b: foobarList_b
|
||||
<<:
|
||||
!!merge <<:
|
||||
- *foo
|
||||
- *bar
|
||||
c: foobarList_c
|
||||
foobar:
|
||||
c: foobar_c
|
||||
<<: *foo
|
||||
!!merge <<: *foo
|
||||
thing: foobar_thing
|
||||
```
|
||||
then
|
||||
@@ -147,7 +147,7 @@ yq '.foobar | [..]' sample.yml
|
||||
will output
|
||||
```yaml
|
||||
- c: foobar_c
|
||||
<<: *foo
|
||||
!!merge <<: *foo
|
||||
thing: foobar_thing
|
||||
- foobar_c
|
||||
- *foo
|
||||
|
||||
@@ -294,13 +294,13 @@ bar: &bar
|
||||
c: bar_c
|
||||
foobarList:
|
||||
b: foobarList_b
|
||||
<<:
|
||||
!!merge <<:
|
||||
- *foo
|
||||
- *bar
|
||||
c: foobarList_c
|
||||
foobar:
|
||||
c: foobar_c
|
||||
<<: *foo
|
||||
!!merge <<: *foo
|
||||
thing: foobar_thing
|
||||
```
|
||||
then
|
||||
@@ -325,13 +325,13 @@ bar: &bar
|
||||
c: bar_c
|
||||
foobarList:
|
||||
b: foobarList_b
|
||||
<<:
|
||||
!!merge <<:
|
||||
- *foo
|
||||
- *bar
|
||||
c: foobarList_c
|
||||
foobar:
|
||||
c: foobar_c
|
||||
<<: *foo
|
||||
!!merge <<: *foo
|
||||
thing: foobar_thing
|
||||
```
|
||||
then
|
||||
@@ -376,13 +376,13 @@ bar: &bar
|
||||
c: bar_c
|
||||
foobarList:
|
||||
b: foobarList_b
|
||||
<<:
|
||||
!!merge <<:
|
||||
- *foo
|
||||
- *bar
|
||||
c: foobarList_c
|
||||
foobar:
|
||||
c: foobar_c
|
||||
<<: *foo
|
||||
!!merge <<: *foo
|
||||
thing: foobar_thing
|
||||
```
|
||||
then
|
||||
@@ -409,13 +409,13 @@ bar: &bar
|
||||
c: bar_c
|
||||
foobarList:
|
||||
b: foobarList_b
|
||||
<<:
|
||||
!!merge <<:
|
||||
- *foo
|
||||
- *bar
|
||||
c: foobarList_c
|
||||
foobar:
|
||||
c: foobar_c
|
||||
<<: *foo
|
||||
!!merge <<: *foo
|
||||
thing: foobar_thing
|
||||
```
|
||||
then
|
||||
@@ -442,13 +442,13 @@ bar: &bar
|
||||
c: bar_c
|
||||
foobarList:
|
||||
b: foobarList_b
|
||||
<<:
|
||||
!!merge <<:
|
||||
- *foo
|
||||
- *bar
|
||||
c: foobarList_c
|
||||
foobar:
|
||||
c: foobar_c
|
||||
<<: *foo
|
||||
!!merge <<: *foo
|
||||
thing: foobar_thing
|
||||
```
|
||||
then
|
||||
@@ -477,13 +477,13 @@ bar: &bar
|
||||
c: bar_c
|
||||
foobarList:
|
||||
b: foobarList_b
|
||||
<<:
|
||||
!!merge <<:
|
||||
- *foo
|
||||
- *bar
|
||||
c: foobarList_c
|
||||
foobar:
|
||||
c: foobar_c
|
||||
<<: *foo
|
||||
!!merge <<: *foo
|
||||
thing: foobar_thing
|
||||
```
|
||||
then
|
||||
@@ -513,13 +513,13 @@ bar: &bar
|
||||
c: bar_c
|
||||
foobarList:
|
||||
b: foobarList_b
|
||||
<<:
|
||||
!!merge <<:
|
||||
- *foo
|
||||
- *bar
|
||||
c: foobarList_c
|
||||
foobar:
|
||||
c: foobar_c
|
||||
<<: *foo
|
||||
!!merge <<: *foo
|
||||
thing: foobar_thing
|
||||
```
|
||||
then
|
||||
@@ -546,13 +546,13 @@ bar: &bar
|
||||
c: bar_c
|
||||
foobarList:
|
||||
b: foobarList_b
|
||||
<<:
|
||||
!!merge <<:
|
||||
- *foo
|
||||
- *bar
|
||||
c: foobarList_c
|
||||
foobar:
|
||||
c: foobar_c
|
||||
<<: *foo
|
||||
!!merge <<: *foo
|
||||
thing: foobar_thing
|
||||
```
|
||||
then
|
||||
@@ -579,13 +579,13 @@ bar: &bar
|
||||
c: bar_c
|
||||
foobarList:
|
||||
b: foobarList_b
|
||||
<<:
|
||||
!!merge <<:
|
||||
- *foo
|
||||
- *bar
|
||||
c: foobarList_c
|
||||
foobar:
|
||||
c: foobar_c
|
||||
<<: *foo
|
||||
!!merge <<: *foo
|
||||
thing: foobar_thing
|
||||
```
|
||||
then
|
||||
@@ -614,13 +614,13 @@ bar: &bar
|
||||
c: bar_c
|
||||
foobarList:
|
||||
b: foobarList_b
|
||||
<<:
|
||||
!!merge <<:
|
||||
- *foo
|
||||
- *bar
|
||||
c: foobarList_c
|
||||
foobar:
|
||||
c: foobar_c
|
||||
<<: *foo
|
||||
!!merge <<: *foo
|
||||
thing: foobar_thing
|
||||
```
|
||||
then
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# TOML
|
||||
|
||||
Encode and decode to and from TOML.
|
||||
Decode from TOML. Note that `yq` does not yet support outputting in TOML format (and therefore it cannot roundtrip)
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# TOML
|
||||
|
||||
Encode and decode to and from TOML.
|
||||
Decode from TOML. Note that `yq` does not yet support outputting in TOML format (and therefore it cannot roundtrip)
|
||||
|
||||
|
||||
## Parse: Simple
|
||||
|
||||
@@ -677,7 +677,7 @@ func (te *tomlEncoder) colorizeToml(input []byte) []byte {
|
||||
|
||||
// Table sections - [section] or [[array]]
|
||||
// Only treat '[' as a table section if it appears at the start of the line
|
||||
// (possibly after whitespace). This avoids incorrectly colouring inline arrays like
|
||||
// (possibly after whitespace). This avoids mis-colouring inline arrays like
|
||||
// "ports = [8000, 8001]" as table sections.
|
||||
if ch == '[' {
|
||||
isSectionHeader := true
|
||||
|
||||
@@ -40,7 +40,7 @@ func tryRemoveTempFile(filename string) {
|
||||
// thanks https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang
|
||||
func copyFileContents(src, dst string) (err error) {
|
||||
// ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory,
|
||||
// and ensuring that it's not possible to give a path to a file outside that directory.
|
||||
// and ensuring that it's not possible to give a path to a file outside thar directory.
|
||||
|
||||
in, err := os.Open(src) // #nosec
|
||||
if err != nil {
|
||||
|
||||
+2
-5
@@ -128,11 +128,8 @@ func FormatStringFromFilename(filename string) string {
|
||||
ext := filepath.Ext(filename)
|
||||
if len(ext) >= 2 && ext[0] == '.' {
|
||||
format := strings.ToLower(ext[1:])
|
||||
if _, err := FormatFromString(format); err == nil {
|
||||
GetLogger().Debugf("detected format '%s'", format)
|
||||
return format
|
||||
}
|
||||
GetLogger().Debugf("extension '%s' is not a recognised format, defaulting to yaml", format)
|
||||
GetLogger().Debugf("detected format '%s'", format)
|
||||
return format
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -59,7 +59,4 @@ func TestFormatStringFromFilename(t *testing.T) {
|
||||
test.AssertResult(t, "json", FormatStringFromFilename("TEST.JSON"))
|
||||
test.AssertResult(t, "yaml", FormatStringFromFilename("test.json/foo"))
|
||||
test.AssertResult(t, "yaml", FormatStringFromFilename(""))
|
||||
// unrecognised extensions should default to yaml instead of being passed through verbatim
|
||||
// (see https://github.com/mikefarah/yq/issues/1608)
|
||||
test.AssertResult(t, "yaml", FormatStringFromFilename("test.tfstate"))
|
||||
}
|
||||
|
||||
@@ -2,26 +2,11 @@ package yqlib
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
var utf8BOM = []byte{0xEF, 0xBB, 0xBF}
|
||||
|
||||
// stripUTF8BOM returns a reader that skips a leading UTF-8 BOM, if present.
|
||||
func stripUTF8BOM(r io.Reader) io.Reader {
|
||||
br := bufio.NewReader(r)
|
||||
|
||||
peek, err := br.Peek(3)
|
||||
if err == nil && bytes.Equal(peek, utf8BOM) {
|
||||
_, _ = br.Discard(3)
|
||||
}
|
||||
|
||||
return br
|
||||
}
|
||||
|
||||
type frontMatterHandler interface {
|
||||
Split() error
|
||||
GetYamlFrontMatterFilename() string
|
||||
@@ -58,13 +43,13 @@ func (f *frontMatterHandlerImpl) Split() error {
|
||||
var reader *bufio.Reader
|
||||
var err error
|
||||
if f.originalFilename == "-" {
|
||||
reader = bufio.NewReader(stripUTF8BOM(os.Stdin))
|
||||
reader = bufio.NewReader(os.Stdin)
|
||||
} else {
|
||||
file, err := os.Open(f.originalFilename) // #nosec
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reader = bufio.NewReader(stripUTF8BOM(file))
|
||||
reader = bufio.NewReader(file)
|
||||
}
|
||||
f.contentReader = reader
|
||||
|
||||
|
||||
@@ -136,11 +136,10 @@ Some content
|
||||
test.AssertResult(t, originalFilename, resolved)
|
||||
|
||||
// Read documents using the temp file, verify they get the original filename
|
||||
reader, cleanup, err := readStream(tempFilename)
|
||||
reader, err := readStream(tempFilename)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer cleanup()
|
||||
decoder := NewYamlDecoder(ConfiguredYamlPreferences)
|
||||
docs, err := readDocuments(reader, tempFilename, 0, decoder)
|
||||
if err != nil {
|
||||
@@ -158,92 +157,6 @@ Some content
|
||||
fmHandler.CleanUp()
|
||||
}
|
||||
|
||||
func TestFrontMatterSplitWithBOM(t *testing.T) {
|
||||
// Regression test for https://github.com/mikefarah/yq/issues/2496
|
||||
// A UTF-8 BOM before the opening --- must be skipped, otherwise the
|
||||
// separator isn't recognised and the opening --- is lost.
|
||||
file := createTestFile("\ufeff---\na: apple\nb: banana\n---\nnot a\nyaml: doc\n")
|
||||
|
||||
expectedYamlFm := `---
|
||||
a: apple
|
||||
b: banana
|
||||
`
|
||||
|
||||
expectedContent := `---
|
||||
not a
|
||||
yaml: doc
|
||||
`
|
||||
|
||||
fmHandler := NewFrontMatterHandler(file)
|
||||
err := fmHandler.Split()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename())
|
||||
|
||||
test.AssertResult(t, expectedYamlFm, yamlFm)
|
||||
|
||||
contentBytes, err := io.ReadAll(fmHandler.GetContentReader())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
test.AssertResult(t, expectedContent, string(contentBytes))
|
||||
|
||||
tryRemoveTempFile(file)
|
||||
fmHandler.CleanUp()
|
||||
}
|
||||
|
||||
func TestFrontMatterSplitWithBOMFromStdin(t *testing.T) {
|
||||
// Regression test for https://github.com/mikefarah/yq/issues/2496
|
||||
// A UTF-8 BOM must also be skipped when reading front matter from stdin.
|
||||
originalStdin := os.Stdin
|
||||
defer func() { os.Stdin = originalStdin }()
|
||||
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
os.Stdin = r
|
||||
defer safelyCloseFile(r)
|
||||
|
||||
go func() {
|
||||
_, writeErr := w.WriteString("\ufeff---\na: apple\nb: banana\n---\nnot a\nyaml: doc\n")
|
||||
if writeErr != nil {
|
||||
t.Errorf("failed to write front matter to the stdin pipe: %v", writeErr)
|
||||
}
|
||||
safelyCloseFile(w)
|
||||
}()
|
||||
|
||||
expectedYamlFm := `---
|
||||
a: apple
|
||||
b: banana
|
||||
`
|
||||
|
||||
expectedContent := `---
|
||||
not a
|
||||
yaml: doc
|
||||
`
|
||||
|
||||
fmHandler := NewFrontMatterHandler("-")
|
||||
err = fmHandler.Split()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
yamlFm := readFile(fmHandler.GetYamlFrontMatterFilename())
|
||||
|
||||
test.AssertResult(t, expectedYamlFm, yamlFm)
|
||||
|
||||
contentBytes, err := io.ReadAll(fmHandler.GetContentReader())
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
test.AssertResult(t, expectedContent, string(contentBytes))
|
||||
|
||||
fmHandler.CleanUp()
|
||||
}
|
||||
|
||||
func TestFrontMatterSplitWithArray(t *testing.T) {
|
||||
file := createTestFile(`[1,2,3]
|
||||
---
|
||||
|
||||
@@ -230,7 +230,8 @@ var goccyYamlFormatScenarios = []formatScenario{
|
||||
description: "merge anchor",
|
||||
skipDoc: true,
|
||||
input: "a: &remember\n c: mike\nb:\n <<: *remember",
|
||||
expected: "a: &remember\n c: mike\nb:\n <<: *remember\n",
|
||||
// fine to have !!merge as that's what the current impl does
|
||||
expected: "a: &remember\n c: mike\nb:\n !!merge <<: *remember\n",
|
||||
},
|
||||
{
|
||||
description: "custom tag",
|
||||
|
||||
@@ -176,41 +176,6 @@ var hclFormatScenarios = []formatScenario{
|
||||
expected: "obj: {a: 1, b: \"two\"}\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "object with integer keys",
|
||||
skipDoc: true,
|
||||
input: `obj = { 1 = "one", 2 = "two" }`,
|
||||
expected: "obj: {1: \"one\", 2: \"two\"}\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "object with boolean keys",
|
||||
skipDoc: true,
|
||||
input: `obj = { (true) = "yes", (false) = "no" }`,
|
||||
expected: "obj: {true: \"yes\", false: \"no\"}\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "object with float keys",
|
||||
skipDoc: true,
|
||||
input: `obj = { (3.14) = "pi" }`,
|
||||
expected: "obj: {3.14: \"pi\"}\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "object with mixed scalar keys",
|
||||
skipDoc: true,
|
||||
input: `obj = { a = 1, 1 = "one", (true) = "yes" }`,
|
||||
expected: "obj: {a: 1, 1: \"one\", true: \"yes\"}\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "nested object with integer keys",
|
||||
skipDoc: true,
|
||||
input: `config = { levels = { 1 = "debug", 2 = "info" } }`,
|
||||
expected: "config: {levels: {1: \"debug\", 2: \"info\"}}\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
{
|
||||
description: "nested block",
|
||||
skipDoc: true,
|
||||
@@ -507,13 +472,6 @@ var hclFormatScenarios = []formatScenario{
|
||||
expected: "service {\n optional_field = null\n}\n",
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
description: "object with integer key and empty value",
|
||||
skipDoc: true,
|
||||
input: `intdict = { 1 = {} }`,
|
||||
expected: "intdict: {1: {}}\n",
|
||||
scenarioType: "decode",
|
||||
},
|
||||
}
|
||||
|
||||
func testHclScenario(t *testing.T, s formatScenario) {
|
||||
|
||||
@@ -203,7 +203,7 @@ func documentDecodeErrorINIScenario(w *bufio.Writer, s formatScenario) {
|
||||
}
|
||||
|
||||
func TestINIDecoderInitResetsFinished(t *testing.T) {
|
||||
decoder := NewINIDecoder(NewDefaultINIPreferences())
|
||||
decoder := NewINIDecoder()
|
||||
firstDocuments, err := readDocuments(strings.NewReader("[first]\nkey = value\n"), "first.ini", 0, decoder)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -189,38 +189,6 @@ var jsonScenarios = []formatScenario{
|
||||
expected: "{\n \"cat\": \"meow\"\n}\n",
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
description: "Encode json: signed hex int",
|
||||
skipDoc: true,
|
||||
input: `+0x12`,
|
||||
indent: 0,
|
||||
expected: "18\n",
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
description: "Encode json: negative hex int",
|
||||
skipDoc: true,
|
||||
input: `-0x12`,
|
||||
indent: 0,
|
||||
expected: "-18\n",
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
description: "Encode json: signed octal int",
|
||||
skipDoc: true,
|
||||
input: `+0o22`,
|
||||
indent: 0,
|
||||
expected: "18\n",
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
description: "Encode json: negative octal int",
|
||||
skipDoc: true,
|
||||
input: `-0o22`,
|
||||
indent: 0,
|
||||
expected: "-18\n",
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
description: "Encode json: simple - in one line",
|
||||
input: `cat: meow # this is a comment, and it will be dropped.`,
|
||||
|
||||
+8
-27
@@ -8,24 +8,14 @@ import (
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
var ExpressionParser ExpressionParserInterface
|
||||
|
||||
var expressionParserOnce sync.Once
|
||||
|
||||
// InitExpressionParser initialises the package level ExpressionParser, and is
|
||||
// safe to call concurrently. The evaluators call it on every Evaluate, so
|
||||
// without the guard two goroutines could construct a parser at the same time,
|
||||
// and newParticipleLexer populates the shared participleYqRules entries as it
|
||||
// goes, which a third goroutine could be reading through getYqDefinition.
|
||||
func InitExpressionParser() {
|
||||
expressionParserOnce.Do(func() {
|
||||
if ExpressionParser == nil {
|
||||
ExpressionParser = newExpressionParser()
|
||||
}
|
||||
})
|
||||
if ExpressionParser == nil {
|
||||
ExpressionParser = newExpressionParser()
|
||||
}
|
||||
}
|
||||
|
||||
var log = newLogger()
|
||||
@@ -171,21 +161,12 @@ func parseInt64(numberString string) (string, int64, error) {
|
||||
numberString = strings.ReplaceAll(numberString, "_", "")
|
||||
}
|
||||
|
||||
// A leading +/- sign would hide the 0x/0o prefix below, so peel it off and
|
||||
// hand it back to ParseInt with the digits.
|
||||
sign := ""
|
||||
digits := numberString
|
||||
if len(digits) > 0 && (digits[0] == '+' || digits[0] == '-') {
|
||||
sign = digits[:1]
|
||||
digits = digits[1:]
|
||||
}
|
||||
|
||||
if strings.HasPrefix(digits, "0x") ||
|
||||
strings.HasPrefix(digits, "0X") {
|
||||
num, err := strconv.ParseInt(sign+digits[2:], 16, 64)
|
||||
if strings.HasPrefix(numberString, "0x") ||
|
||||
strings.HasPrefix(numberString, "0X") {
|
||||
num, err := strconv.ParseInt(numberString[2:], 16, 64)
|
||||
return "0x%X", num, err
|
||||
} else if strings.HasPrefix(digits, "0o") {
|
||||
num, err := strconv.ParseInt(sign+digits[2:], 8, 64)
|
||||
} else if strings.HasPrefix(numberString, "0o") {
|
||||
num, err := strconv.ParseInt(numberString[2:], 8, 64)
|
||||
return "0o%o", num, err
|
||||
}
|
||||
num, err := strconv.ParseInt(numberString, 10, 64)
|
||||
|
||||
+1
-48
@@ -3,7 +3,6 @@ package yqlib
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/mikefarah/yq/v4/test"
|
||||
@@ -25,7 +24,7 @@ type parseSnippetScenario struct {
|
||||
var parseSnippetScenarios = []parseSnippetScenario{
|
||||
{
|
||||
snippet: ":",
|
||||
expectedError: "go-yaml load error in parser (while parsing a block mapping) at L1.C1: did not find expected key",
|
||||
expectedError: "yaml: while parsing a block mapping at <unknown position>: did not find expected key",
|
||||
},
|
||||
{
|
||||
snippet: "",
|
||||
@@ -144,16 +143,6 @@ var parseInt64Scenarios = []parseInt64Scenario{
|
||||
numberString: "0o10",
|
||||
expectedParsedNumber: 8,
|
||||
},
|
||||
{
|
||||
numberString: "+0x12",
|
||||
expectedParsedNumber: 18,
|
||||
expectedFormatString: "0x12",
|
||||
},
|
||||
{
|
||||
numberString: "+0o22",
|
||||
expectedParsedNumber: 18,
|
||||
expectedFormatString: "0o22",
|
||||
},
|
||||
}
|
||||
|
||||
func TestParseInt64(t *testing.T) {
|
||||
@@ -556,39 +545,3 @@ func TestProcessEscapeCharacters(t *testing.T) {
|
||||
test.AssertResultComplexWithContext(t, tt.expected, actual, fmt.Sprintf("Input: %q", tt.input))
|
||||
}
|
||||
}
|
||||
|
||||
// TestInitExpressionParserConcurrent covers the data race described in #2788.
|
||||
// The evaluators call InitExpressionParser on every Evaluate, so before it was
|
||||
// guarded two goroutines could build a parser at the same time while a third
|
||||
// read the shared participleYqRules entries that newParticipleLexer fills in.
|
||||
//
|
||||
// To reproduce the original race the process must not have initialised the
|
||||
// parser yet, so run this test on its own with the detector enabled:
|
||||
//
|
||||
// go test -race -run TestInitExpressionParserConcurrent ./pkg/yqlib/
|
||||
func TestInitExpressionParserConcurrent(t *testing.T) {
|
||||
const goroutines = 32
|
||||
|
||||
var wg sync.WaitGroup
|
||||
errs := make(chan error, goroutines)
|
||||
|
||||
for range goroutines {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
InitExpressionParser()
|
||||
if _, err := ExpressionParser.ParseExpression(".a.b"); err != nil {
|
||||
errs <- err
|
||||
}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
|
||||
for err := range errs {
|
||||
t.Fatalf("concurrent ParseExpression failed: %v", err)
|
||||
}
|
||||
if ExpressionParser == nil {
|
||||
t.Fatal("ExpressionParser should be initialised after concurrent InitExpressionParser calls")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -183,9 +183,7 @@ func fixedReconstructAliasedMap(node *CandidateNode) error {
|
||||
})
|
||||
|
||||
for _, item := range itemsToAdd {
|
||||
copied := item.Copy()
|
||||
copied.Parent = node
|
||||
newContent = append(newContent, copied)
|
||||
newContent = append(newContent, item.Copy())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -228,12 +226,8 @@ func reconstructAliasedMap(node *CandidateNode, context Context) error {
|
||||
}
|
||||
}
|
||||
node.Content = make([]*CandidateNode, 0)
|
||||
entries := make([]*CandidateNode, 0, newContent.Len())
|
||||
for newEl := newContent.Front(); newEl != nil; newEl = newEl.Next() {
|
||||
entries = append(entries, newEl.Value.(*CandidateNode))
|
||||
}
|
||||
for i := 0; i < len(entries); i += 2 {
|
||||
node.AddKeyValueChild(entries[i], entries[i+1])
|
||||
node.AddChild(newEl.Value.(*CandidateNode))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ thingOne:
|
||||
value: false
|
||||
thingTwo:
|
||||
name: item_2
|
||||
<<: *item_value
|
||||
!!merge <<: *item_value
|
||||
`
|
||||
|
||||
var explodeMergeAnchorsFixedExpected = `D0, P[], (!!map)::foo:
|
||||
@@ -234,29 +234,6 @@ var fixedAnchorOperatorScenarios = []expressionScenario{
|
||||
"D0, P[], (!!map)::a:\n b: 42\nb: 42\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "Merge after explode preserves correct parent references",
|
||||
document: `opensearch: &opensearch-cluster
|
||||
ip2geo:
|
||||
enabled: false
|
||||
|
||||
opensearch-client:
|
||||
<<: *opensearch-cluster
|
||||
nodeGroup: client
|
||||
opensearchJavaOpts: "-Xmx1024m -Xms1024m"`,
|
||||
document2: `opensearch: &opensearch-cluster
|
||||
ip2geo:
|
||||
enabled: true
|
||||
|
||||
opensearch-client:
|
||||
<<: *opensearch-cluster
|
||||
opensearchJavaOpts: "-Xmx1536m -Xms1536m"`,
|
||||
expression: `(select(fi == 0) | explode(.)) * (select(fi == 1) | explode(.))`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::opensearch:\n ip2geo:\n enabled: true\nopensearch-client:\n ip2geo:\n enabled: true\n nodeGroup: client\n opensearchJavaOpts: \"-Xmx1536m -Xms1536m\"\n",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
var badAnchorOperatorScenarios = []expressionScenario{
|
||||
@@ -311,63 +288,7 @@ var badAnchorOperatorScenarios = []expressionScenario{
|
||||
},
|
||||
}
|
||||
|
||||
var mixedMergeTagStyleDocument = `
|
||||
constants:
|
||||
errorResponse: &errorResponse
|
||||
status: 200
|
||||
endpoints:
|
||||
- condition: true
|
||||
!!merge <<: *errorResponse
|
||||
- condition: false
|
||||
<<: *errorResponse
|
||||
other:
|
||||
!!merge <<: *errorResponse
|
||||
somethingElse:
|
||||
<<: *errorResponse
|
||||
`
|
||||
|
||||
var mixedMergeTagStyleExplodedDocument = `
|
||||
constants:
|
||||
errorResponse:
|
||||
status: 200
|
||||
endpoints:
|
||||
- condition: true
|
||||
status: 200
|
||||
- condition: false
|
||||
status: 200
|
||||
other:
|
||||
status: 200
|
||||
somethingElse:
|
||||
status: 200
|
||||
`
|
||||
|
||||
var anchorOperatorScenarios = []expressionScenario{
|
||||
{
|
||||
// mergeObjects previously skipped all !!merge-tagged nodes. Since !!merge only appears on
|
||||
// << map keys, this meant applyAssignment was never called for the << key. It was later
|
||||
// autocreated by createStringScalarNode("<<") with tag !!str, silently dropping !!merge.
|
||||
// DontFollowAlias:true already prevents aliases being followed, so the skip was redundant.
|
||||
// Old (buggy) output: "D0, P[], (!!map)::base: &base\n x: 1\ndest:\n <<: *base\n"
|
||||
skipDoc: true,
|
||||
description: "direct *+ preserves explicit !!merge tag on << key (regression for issue 2677)",
|
||||
document: "base: &base\n x: 1\ndest:\n !!merge <<: *base\n",
|
||||
expression: `. as $d | {} *+ $d`,
|
||||
expected: []string{"D0, P[], (!!map)::base: &base\n x: 1\ndest:\n !!merge <<: *base\n"},
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "explicit !!merge tag on << key is preserved through ireduce merge",
|
||||
document: mixedMergeTagStyleDocument,
|
||||
expression: `. as $item ireduce ({}; . *+ $item)`,
|
||||
expected: []string{"D0, P[], (!!map)::" + mixedMergeTagStyleDocument},
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "explode expands << merge keys regardless of explicit tag style (!!merge or plain)",
|
||||
document: mixedMergeTagStyleDocument,
|
||||
expression: `explode(.)`,
|
||||
expected: []string{"D0, P[], (!!map)::" + mixedMergeTagStyleExplodedDocument},
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "merge anchor to alias alias",
|
||||
|
||||
@@ -69,30 +69,6 @@ func deleteFromMap(node *CandidateNode, childPath interface{}) {
|
||||
}
|
||||
}
|
||||
node.Content = newContents
|
||||
normaliseEmptyCollectionMapKeyComment(node)
|
||||
}
|
||||
|
||||
func normaliseEmptyCollectionMapKeyComment(node *CandidateNode) {
|
||||
if (node.Kind != SequenceNode && node.Kind != MappingNode) || len(node.Content) != 0 || node.LineComment != "" {
|
||||
return
|
||||
}
|
||||
|
||||
key := node.Key
|
||||
if node.Parent != nil && node.Parent.Kind == MappingNode {
|
||||
for index := 0; index < len(node.Parent.Content)-1; index += 2 {
|
||||
if node.Parent.Content[index+1] == node {
|
||||
key = node.Parent.Content[index]
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if key == nil || key.LineComment == "" {
|
||||
return
|
||||
}
|
||||
|
||||
node.LineComment = key.LineComment
|
||||
key.LineComment = ""
|
||||
node.Style = FlowStyle
|
||||
}
|
||||
|
||||
func deleteFromArray(node *CandidateNode, childPath interface{}) {
|
||||
@@ -111,5 +87,4 @@ func deleteFromArray(node *CandidateNode, childPath interface{}) {
|
||||
}
|
||||
}
|
||||
node.Content = newContents
|
||||
normaliseEmptyCollectionMapKeyComment(node)
|
||||
}
|
||||
|
||||
@@ -119,30 +119,6 @@ var deleteOperatorScenarios = []expressionScenario{
|
||||
"D0, P[], (!!map)::a: []\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "Delete all entries from list with inline key comment",
|
||||
document: `testList: # A comment
|
||||
- name: test1
|
||||
value: 123
|
||||
- name: test2
|
||||
value: 456`,
|
||||
expression: `del(.testList[])`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::testList: [] # A comment\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "Delete all entries from map with inline key comment",
|
||||
document: `testMap: # A comment
|
||||
name: test1
|
||||
value: 123`,
|
||||
expression: `del(.testMap[])`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::testMap: {} # A comment\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "Delete entry appended to an array",
|
||||
|
||||
@@ -45,7 +45,7 @@ var divideOperatorScenarios = []expressionScenario{
|
||||
document: `{a: 1, b: -1}`,
|
||||
expression: `.a = .a / 0 | .b = .b / 0`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::{a: +Inf, b: -Inf}\n",
|
||||
"D0, P[], (!!map)::{a: !!float +Inf, b: !!float -Inf}\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
@@ -44,7 +44,7 @@ func hasOperator(d *dataTreeNavigator, context Context, expressionNode *Expressi
|
||||
if errParsingInt != nil {
|
||||
return Context{}, errParsingInt
|
||||
}
|
||||
candidateHasKey = number >= 0 && int64(len(contents)) > number
|
||||
candidateHasKey = int64(len(contents)) > number
|
||||
}
|
||||
results.PushBack(createBooleanCandidate(candidate, candidateHasKey))
|
||||
default:
|
||||
|
||||
@@ -71,16 +71,6 @@ var hasOperatorScenarios = []expressionScenario{
|
||||
"D0, P[4], (!!bool)::true\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "Negative array index is never present",
|
||||
document: "[[1, 2, 3], []]",
|
||||
expression: `.[] | has(-1)`,
|
||||
expected: []string{
|
||||
"D0, P[0], (!!bool)::false\n",
|
||||
"D0, P[1], (!!bool)::false\n",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func TestHasOperatorScenarios(t *testing.T) {
|
||||
|
||||
@@ -77,7 +77,7 @@ var loadScenarios = []expressionScenario{
|
||||
document: `{something: {file: "thing.yml"}, over: {here: [{file: "thing.yml"}]}}`,
|
||||
expression: `(.. | select(has("file"))) |= load("../../examples/" + .file)`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::{something: {a: apple is included, b: cool.}, over: {here: [{a: apple is included,\n b: cool.}]}}\n",
|
||||
"D0, P[], (!!map)::{something: {a: apple is included, b: cool.}, over: {here: [{a: apple is included, b: cool.}]}}\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
@@ -37,7 +37,7 @@ var moduloOperatorScenarios = []expressionScenario{
|
||||
document: `{a: 12, b: 2.5}`,
|
||||
expression: `.a = .a % .b`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::{a: 2, b: 2.5}\n",
|
||||
"D0, P[], (!!map)::{a: !!float 2, b: 2.5}\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -53,7 +53,7 @@ var moduloOperatorScenarios = []expressionScenario{
|
||||
document: `{a: 1.1, b: 0}`,
|
||||
expression: `.a = .a % .b`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::{a: NaN, b: 0}\n",
|
||||
"D0, P[], (!!map)::{a: !!float NaN, b: 0}\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -70,7 +70,7 @@ var moduloOperatorScenarios = []expressionScenario{
|
||||
document: "a: 2\nb: !goat 2.3",
|
||||
expression: `.a = .a % .b`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::a: 2\nb: !goat 2.3\n",
|
||||
"D0, P[], (!!map)::a: !!float 2\nb: !goat 2.3\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
@@ -189,6 +189,10 @@ func mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs
|
||||
|
||||
log.Debugf("going to applied assignment to LHS: %v with RHS: %v", NodeToString(lhs), NodeToString(candidate))
|
||||
|
||||
if candidate.Tag == "!!merge" {
|
||||
continue
|
||||
}
|
||||
|
||||
err := applyAssignment(d, context, pathIndexToStartFrom, lhs, candidate, preferences)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -121,7 +121,7 @@ var multiplyOperatorScenarios = []expressionScenario{
|
||||
document: mergeArrayWithAnchors,
|
||||
expression: `. * .`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::sample:\n - &a\n - <<: *a\n",
|
||||
"D0, P[], (!!map)::sample:\n - &a\n - !!merge <<: *a\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -514,7 +514,7 @@ var multiplyOperatorScenarios = []expressionScenario{
|
||||
environmentVariables: map[string]string{"originalPath": ".myArray", "otherPath": ".newArray", "idPath": ".a"},
|
||||
expression: mergeExpression,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::{myArray: [{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC},\n {a: dingo, c: dingoC}], something: else}\n",
|
||||
"D0, P[], (!!map)::{myArray: [{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC}, {a: dingo, c: dingoC}], something: else}\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -546,7 +546,7 @@ var multiplyOperatorScenarios = []expressionScenario{
|
||||
document: mergeDocSample,
|
||||
expression: `.foobar * .foobarList`,
|
||||
expected: []string{
|
||||
"D0, P[foobar], (!!map)::c: foobarList_c\n<<: [*foo, *bar]\nthing: foobar_thing\nb: foobarList_b\n",
|
||||
"D0, P[foobar], (!!map)::c: foobarList_c\n!!merge <<: [*foo, *bar]\nthing: foobar_thing\nb: foobarList_b\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -554,7 +554,7 @@ var multiplyOperatorScenarios = []expressionScenario{
|
||||
document: document,
|
||||
expression: `.b * .c`,
|
||||
expected: []string{
|
||||
"D0, P[b], (!!map)::{name: dog, <<: *cat}\n",
|
||||
"D0, P[b], (!!map)::{name: dog, \"<<\": *cat}\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -581,7 +581,7 @@ var multiplyOperatorScenarios = []expressionScenario{
|
||||
document: "a: 2\nb: !goat 3.5",
|
||||
expression: ".a = .a * .b",
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::a: 7\nb: !goat 3.5\n",
|
||||
"D0, P[], (!!map)::a: !!float 7\nb: !goat 3.5\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
@@ -187,7 +187,7 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
|
||||
document: mergeDocSample,
|
||||
expression: `.foobar | [..]`,
|
||||
expected: []string{
|
||||
"D0, P[foobar], (!!seq)::- c: foobar_c\n <<: *foo\n thing: foobar_thing\n- foobar_c\n- *foo\n- foobar_thing\n",
|
||||
"D0, P[foobar], (!!seq)::- c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n- foobar_c\n- *foo\n- foobar_thing\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -195,7 +195,7 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
|
||||
document: mergeDocSample,
|
||||
expression: `.foobar | [...]`,
|
||||
expected: []string{
|
||||
"D0, P[foobar], (!!seq)::- c: foobar_c\n <<: *foo\n thing: foobar_thing\n- c\n- foobar_c\n- <<\n- *foo\n- thing\n- foobar_thing\n",
|
||||
"D0, P[foobar], (!!seq)::- c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n- c\n- foobar_c\n- !!merge <<\n- *foo\n- thing\n- foobar_thing\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -203,7 +203,7 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
|
||||
document: mergeDocSample,
|
||||
expression: `.foobarList | ..`,
|
||||
expected: []string{
|
||||
"D0, P[foobarList], (!!map)::b: foobarList_b\n<<: [*foo, *bar]\nc: foobarList_c\n",
|
||||
"D0, P[foobarList], (!!map)::b: foobarList_b\n!!merge <<: [*foo, *bar]\nc: foobarList_c\n",
|
||||
"D0, P[foobarList b], (!!str)::foobarList_b\n",
|
||||
"D0, P[foobarList <<], (!!seq)::[*foo, *bar]\n",
|
||||
"D0, P[foobarList << 0], (alias)::*foo\n",
|
||||
@@ -216,7 +216,7 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
|
||||
document: mergeDocSample,
|
||||
expression: `.foobarList | ...`,
|
||||
expected: []string{
|
||||
"D0, P[foobarList], (!!map)::b: foobarList_b\n<<: [*foo, *bar]\nc: foobarList_c\n",
|
||||
"D0, P[foobarList], (!!map)::b: foobarList_b\n!!merge <<: [*foo, *bar]\nc: foobarList_c\n",
|
||||
"D0, P[foobarList b], (!!str)::b\n",
|
||||
"D0, P[foobarList b], (!!str)::foobarList_b\n",
|
||||
"D0, P[foobarList <<], (!!merge)::<<\n",
|
||||
|
||||
@@ -98,7 +98,7 @@ var selectOperatorScenarios = []expressionScenario{
|
||||
document: `[{animal: cat, legs: {cool: true}}, {animal: fish}]`,
|
||||
expression: `(.[] | select(.legs.cool == true).canWalk) = true | (.[] | .alive.things) = "yes"`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!seq)::[{animal: cat, legs: {cool: true}, canWalk: true, alive: {things: yes}}, {animal: fish,\n alive: {things: yes}}]\n",
|
||||
"D0, P[], (!!seq)::[{animal: cat, legs: {cool: true}, canWalk: true, alive: {things: yes}}, {animal: fish, alive: {things: yes}}]\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
@@ -166,12 +166,7 @@ func (a sortableNodeArray) compare(lhs *CandidateNode, rhs *CandidateNode, dateT
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if lhsNum < rhsNum {
|
||||
return -1
|
||||
} else if lhsNum > rhsNum {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
return int(lhsNum - rhsNum)
|
||||
} else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") {
|
||||
lhsNum, err := strconv.ParseFloat(lhs.Value, 64)
|
||||
if err != nil {
|
||||
|
||||
@@ -37,7 +37,7 @@ var sortKeysOperatorScenarios = []expressionScenario{
|
||||
document: `{bParent: {c: dog, array: [3,1,2]}, aParent: {z: donkey, x: [{c: yum, b: delish}, {b: ew, a: apple}]}}`,
|
||||
expression: `sort_keys(..)`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::{aParent: {x: [{b: delish, c: yum}, {a: apple, b: ew}], z: donkey}, bParent: {array: [\n 3, 1, 2], c: dog}}\n",
|
||||
"D0, P[], (!!map)::{aParent: {x: [{b: delish, c: yum}, {a: apple, b: ew}], z: donkey}, bParent: {array: [3, 1, 2], c: dog}}\n",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
@@ -171,15 +171,6 @@ var sortByOperatorScenarios = []expressionScenario{
|
||||
"D0, P[], (!!seq)::# abc\n- def\n# ghi\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "Sort large integers (no int64 subtraction overflow)",
|
||||
document: "[5000000000000000000, -5000000000000000000]",
|
||||
expression: `sort`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!seq)::[-5000000000000000000, 5000000000000000000]\n",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func TestSortByOperatorScenarios(t *testing.T) {
|
||||
|
||||
@@ -50,7 +50,7 @@ var styleOperatorScenarios = []expressionScenario{
|
||||
document: "bing: &foo {x: z}\na:\n c: cat\n <<: [*foo]",
|
||||
expression: `(... | select(tag=="!!str")) style="single"`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::'bing': &foo {'x': 'z'}\n'a':\n 'c': 'cat'\n <<: [*foo]\n",
|
||||
"D0, P[], (!!map)::'bing': &foo {'x': 'z'}\n'a':\n 'c': 'cat'\n !!merge <<: [*foo]\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
@@ -494,7 +494,7 @@ var traversePathOperatorScenarios = []expressionScenario{
|
||||
document: mergeDocSample,
|
||||
expression: `.foobar`,
|
||||
expected: []string{
|
||||
"D0, P[foobar], (!!map)::c: foobar_c\n<<: *foo\nthing: foobar_thing\n",
|
||||
"D0, P[foobar], (!!map)::c: foobar_c\n!!merge <<: *foo\nthing: foobar_thing\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -518,7 +518,7 @@ var traversePathOperatorScenarios = []expressionScenario{
|
||||
document: mergeDocSample,
|
||||
expression: `.foobarList`,
|
||||
expected: []string{
|
||||
"D0, P[foobarList], (!!map)::b: foobarList_b\n<<: [*foo, *bar]\nc: foobarList_c\n",
|
||||
"D0, P[foobarList], (!!map)::b: foobarList_b\n!!merge <<: [*foo, *bar]\nc: foobarList_c\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
)
|
||||
|
||||
// A yaml expression evaluator that runs the expression multiple times for each given yaml document.
|
||||
@@ -49,17 +50,21 @@ func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, p
|
||||
}
|
||||
|
||||
for _, filename := range filenames {
|
||||
reader, cleanup, err := readStream(filename)
|
||||
reader, err := readStream(filename)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
processedDocs, err := s.Evaluate(filename, reader, node, printer, decoder)
|
||||
cleanup()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
totalProcessDocs = totalProcessDocs + processedDocs
|
||||
|
||||
switch reader := reader.(type) {
|
||||
case *os.File:
|
||||
safelyCloseFile(reader)
|
||||
}
|
||||
}
|
||||
|
||||
if totalProcessDocs == 0 {
|
||||
|
||||
@@ -369,7 +369,7 @@ var tomlScenarios = []formatScenario{
|
||||
skipDoc: true,
|
||||
description: "blank",
|
||||
input: `A = "hello`,
|
||||
expectedError: `bad file 'sample.yml': unterminated basic string`,
|
||||
expectedError: `bad file 'sample.yml': basic string not terminated by "`,
|
||||
scenarioType: "decode-error",
|
||||
},
|
||||
{
|
||||
|
||||
+14
-13
@@ -32,24 +32,25 @@ func resolveFilename(filename string) string {
|
||||
return filename
|
||||
}
|
||||
|
||||
// readStream returns a reader for the given file, along with a cleanup function
|
||||
// that must be called once the reader is no longer needed. The cleanup is a no-op
|
||||
// for stdin.
|
||||
func readStream(filename string) (io.Reader, func(), error) {
|
||||
func readStream(filename string) (io.Reader, error) {
|
||||
var reader *bufio.Reader
|
||||
if filename == "-" {
|
||||
return bufio.NewReader(os.Stdin), func() {}, nil
|
||||
reader = bufio.NewReader(os.Stdin)
|
||||
} else {
|
||||
// ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory,
|
||||
// and ensuring that it's not possible to give a path to a file outside that directory.
|
||||
file, err := os.Open(filename) // #nosec
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reader = bufio.NewReader(file)
|
||||
}
|
||||
// ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory,
|
||||
// and ensuring that it's not possible to give a path to a file outside that directory.
|
||||
file, err := os.Open(filename) // #nosec
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return bufio.NewReader(file), func() { safelyCloseFile(file) }, nil
|
||||
return reader, nil
|
||||
|
||||
}
|
||||
|
||||
func writeString(writer io.Writer, txt string) error {
|
||||
_, errorWriting := io.WriteString(writer, txt)
|
||||
_, errorWriting := writer.Write([]byte(txt))
|
||||
return errorWriting
|
||||
}
|
||||
|
||||
|
||||
@@ -1,119 +0,0 @@
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime/debug"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/mikefarah/yq/v4/test"
|
||||
)
|
||||
|
||||
// countOpenFileDescriptors returns the number of file descriptors this process
|
||||
// currently holds open, or -1 if the platform does not expose them.
|
||||
func countOpenFileDescriptors() int {
|
||||
for _, dir := range []string{"/proc/self/fd", "/dev/fd"} {
|
||||
// Readdirnames avoids stat-ing each entry, which races with descriptors
|
||||
// (including this directory handle) being closed underneath us.
|
||||
handle, err := os.Open(dir)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
names, err := handle.Readdirnames(-1)
|
||||
safelyCloseFile(handle)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
// discount the directory handle itself
|
||||
return len(names) - 1
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
func writeSampleFiles(t *testing.T, count int) []string {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
filenames := make([]string, count)
|
||||
for i := 0; i < count; i++ {
|
||||
filename := filepath.Join(dir, "sample-"+strconv.Itoa(i)+".yml")
|
||||
if err := os.WriteFile(filename, []byte("a: apple\n"), 0600); err != nil {
|
||||
t.Fatalf("failed to write sample file: %v", err)
|
||||
}
|
||||
filenames[i] = filename
|
||||
}
|
||||
return filenames
|
||||
}
|
||||
|
||||
func discardingPrinter() Printer {
|
||||
return NewPrinter(NewYamlEncoder(ConfiguredYamlPreferences), NewSinglePrinterWriter(bufio.NewWriter(io.Discard)))
|
||||
}
|
||||
|
||||
func assertNoLeakedFileDescriptors(t *testing.T, evaluate func(filenames []string) error) {
|
||||
t.Helper()
|
||||
InitExpressionParser()
|
||||
|
||||
// os.File finalisers close leaked descriptors on collection, which would
|
||||
// let a genuine leak pass unnoticed.
|
||||
defer debug.SetGCPercent(debug.SetGCPercent(-1))
|
||||
|
||||
before := countOpenFileDescriptors()
|
||||
if before < 0 {
|
||||
t.Skip("file descriptors are not observable on this platform")
|
||||
}
|
||||
|
||||
if err := evaluate(writeSampleFiles(t, 50)); err != nil {
|
||||
t.Fatalf("failed to evaluate files: %v", err)
|
||||
}
|
||||
|
||||
after := countOpenFileDescriptors()
|
||||
if after > before {
|
||||
t.Errorf("expected no additional open file descriptors, had %d before and %d after", before, after)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamEvaluatorClosesInputFiles(t *testing.T) {
|
||||
assertNoLeakedFileDescriptors(t, func(filenames []string) error {
|
||||
return NewStreamEvaluator().EvaluateFiles(".a", filenames, discardingPrinter(), NewYamlDecoder(ConfiguredYamlPreferences))
|
||||
})
|
||||
}
|
||||
|
||||
func TestAllAtOnceEvaluatorClosesInputFiles(t *testing.T) {
|
||||
assertNoLeakedFileDescriptors(t, func(filenames []string) error {
|
||||
return NewAllAtOnceEvaluator().EvaluateFiles(".a", filenames, discardingPrinter(), NewYamlDecoder(ConfiguredYamlPreferences))
|
||||
})
|
||||
}
|
||||
|
||||
// plainWriter only implements io.Writer, so io.WriteString must fall back to Write.
|
||||
type plainWriter struct {
|
||||
buf bytes.Buffer
|
||||
}
|
||||
|
||||
func (w *plainWriter) Write(p []byte) (int, error) {
|
||||
return w.buf.Write(p)
|
||||
}
|
||||
|
||||
func TestWriteStringToStringWriter(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
writer := bufio.NewWriter(&buf)
|
||||
test.AssertResult(t, nil, writeString(writer, "hello world"))
|
||||
test.AssertResult(t, nil, writer.Flush())
|
||||
test.AssertResult(t, "hello world", buf.String())
|
||||
}
|
||||
|
||||
func TestWriteStringToPlainWriter(t *testing.T) {
|
||||
writer := &plainWriter{}
|
||||
test.AssertResult(t, nil, writeString(writer, "hello world"))
|
||||
test.AssertResult(t, "hello world", writer.buf.String())
|
||||
}
|
||||
|
||||
func TestWriteStringDoesNotAllocate(t *testing.T) {
|
||||
writer := bufio.NewWriter(io.Discard)
|
||||
allocations := testing.AllocsPerRun(100, func() {
|
||||
_ = writeString(writer, "hello world")
|
||||
})
|
||||
test.AssertResult(t, 0.0, allocations)
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
abxbbxdbxebxczzx
|
||||
abxbbxdbxebxczzy
|
||||
accum
|
||||
Accum
|
||||
adithyasunil
|
||||
AEDT
|
||||
água
|
||||
ÁGUA
|
||||
alecthomas
|
||||
appleapple
|
||||
Astuff
|
||||
autocreating
|
||||
autoparse
|
||||
AWST
|
||||
axbxcxdxe
|
||||
axbxcxdxexxx
|
||||
bananabanana
|
||||
barp
|
||||
nbaz
|
||||
bitnami
|
||||
blarp
|
||||
blddir
|
||||
Bobo
|
||||
BODMAS
|
||||
bonapite
|
||||
Brien
|
||||
Bstuff
|
||||
BUILDKIT
|
||||
buildpackage
|
||||
catmeow
|
||||
CATYPE
|
||||
CBVVE
|
||||
chardata
|
||||
chillum
|
||||
choco
|
||||
chomper
|
||||
cleanup
|
||||
cmlu
|
||||
colorise
|
||||
colors
|
||||
Colors
|
||||
colourize
|
||||
compinit
|
||||
coolioo
|
||||
coverprofile
|
||||
createmap
|
||||
csvd
|
||||
CSVUTF
|
||||
currentlabel
|
||||
cygpath
|
||||
czvf
|
||||
datestring
|
||||
datetime
|
||||
Datetime
|
||||
datetimes
|
||||
DEBEMAIL
|
||||
debhelper
|
||||
Debugf
|
||||
debuild
|
||||
delish
|
||||
delpaths
|
||||
DELPATHS
|
||||
devorbitus
|
||||
devscripts
|
||||
dimchansky
|
||||
Dont
|
||||
dput
|
||||
elliotchance
|
||||
endhint
|
||||
endofname
|
||||
Entriesfrom
|
||||
envsubst
|
||||
errorlevel
|
||||
Escandón
|
||||
Evalall
|
||||
fakefilename
|
||||
fakeroot
|
||||
Farah
|
||||
fatih
|
||||
Fifi
|
||||
filebytes
|
||||
Fileish
|
||||
foobar
|
||||
foobaz
|
||||
foof
|
||||
frood
|
||||
fullpath
|
||||
gitbook
|
||||
githubactions
|
||||
gnupg
|
||||
goccy
|
||||
gofmt
|
||||
gogo
|
||||
golangci
|
||||
goreleaser
|
||||
GORELEASER
|
||||
GOMODCACHE
|
||||
GOPATH
|
||||
gosec
|
||||
gota
|
||||
goversion
|
||||
GOVERSION
|
||||
haha
|
||||
hellno
|
||||
herbygillot
|
||||
hexdump
|
||||
Hoang
|
||||
hostpath
|
||||
hotdog
|
||||
howdy
|
||||
incase
|
||||
Infof
|
||||
inlinetables
|
||||
inplace
|
||||
ints
|
||||
ireduce
|
||||
iwatch
|
||||
jinzhu
|
||||
jq's
|
||||
jsond
|
||||
keygrip
|
||||
Keygrip
|
||||
KEYGRIP
|
||||
KEYID
|
||||
keyvalue
|
||||
kwak
|
||||
lalilu
|
||||
ldflags
|
||||
LDFLAGS
|
||||
lexer
|
||||
Lexer
|
||||
libdistro
|
||||
lindex
|
||||
linecomment
|
||||
LVAs
|
||||
magiconair
|
||||
mapvalues
|
||||
Mier
|
||||
mikefarah
|
||||
minideb
|
||||
minishift
|
||||
mipsle
|
||||
mitchellh
|
||||
mktemp
|
||||
Mult
|
||||
multidoc
|
||||
multimaint
|
||||
myenv
|
||||
myenvnonexisting
|
||||
myfile
|
||||
myformat
|
||||
ndjson
|
||||
NDJSON
|
||||
NFKD
|
||||
nixpkgs
|
||||
nojson
|
||||
nonascii
|
||||
nonempty
|
||||
noninteractive
|
||||
Nonquoting
|
||||
nosec
|
||||
notoml
|
||||
noxml
|
||||
nolua
|
||||
nullinput
|
||||
onea
|
||||
Oneshot
|
||||
opencollect
|
||||
opstack
|
||||
orderedmap
|
||||
osarch
|
||||
overridign
|
||||
pacman
|
||||
Padder
|
||||
pandoc
|
||||
parsechangelog
|
||||
pcsv
|
||||
pelletier
|
||||
pflag
|
||||
prechecking
|
||||
Prerelease
|
||||
proc
|
||||
propsd
|
||||
qylib
|
||||
readline
|
||||
realnames
|
||||
realpath
|
||||
repr
|
||||
rhash
|
||||
rindex
|
||||
risentveber
|
||||
rmescandon
|
||||
Rosey
|
||||
roundtrip
|
||||
roundtrips
|
||||
Roundtrip
|
||||
roundtripping
|
||||
Interp
|
||||
interp
|
||||
runningvms
|
||||
sadface
|
||||
selfupdate
|
||||
setpath
|
||||
sharedfolder
|
||||
Sharedfolder
|
||||
shellvariables
|
||||
shellvars
|
||||
shortfunc
|
||||
shortpipe
|
||||
shunit
|
||||
snapcraft
|
||||
somevalue
|
||||
splt
|
||||
srcdir
|
||||
stackoverflow
|
||||
stiched
|
||||
Strc
|
||||
strenv
|
||||
strload
|
||||
stylig
|
||||
subarray
|
||||
subchild
|
||||
subdescription
|
||||
submatch
|
||||
submatches
|
||||
SUBSTR
|
||||
tempfile
|
||||
tfstate
|
||||
Tfstate
|
||||
thar
|
||||
timezone
|
||||
Timezone
|
||||
timezones
|
||||
Timezones
|
||||
tojson
|
||||
Tokenvalue
|
||||
tsvd
|
||||
Tuan
|
||||
tzdata
|
||||
Uhoh
|
||||
updateassign
|
||||
urid
|
||||
utfbom
|
||||
Warningf
|
||||
Wazowski
|
||||
webi
|
||||
Webi
|
||||
wherever
|
||||
winget
|
||||
withdots
|
||||
wizz
|
||||
woop
|
||||
workdir
|
||||
Writable
|
||||
xmld
|
||||
xyzzy
|
||||
yamld
|
||||
yqlib
|
||||
yuin
|
||||
zabbix
|
||||
tonumber
|
||||
noyaml
|
||||
nolint
|
||||
shortfile
|
||||
Unmarshalling
|
||||
noini
|
||||
nocsv
|
||||
nobase64
|
||||
nouri
|
||||
noprops
|
||||
nosh
|
||||
noshell
|
||||
tinygo
|
||||
nonexistent
|
||||
hclsyntax
|
||||
hclwrite
|
||||
nohcl
|
||||
zclconf
|
||||
cty
|
||||
go-cty
|
||||
Colorisation
|
||||
goimports
|
||||
errorlint
|
||||
RDBMS
|
||||
expeñded
|
||||
bananabananabananabanana
|
||||
edwinjhlee
|
||||
flox
|
||||
unlabelled
|
||||
kyaml
|
||||
KYAML
|
||||
nokyaml
|
||||
buildvcs
|
||||
behaviour
|
||||
GOFLAGS
|
||||
gocache
|
||||
subsubarray
|
||||
Ffile
|
||||
Fquery
|
||||
coverpkg
|
||||
gsub
|
||||
ralia
|
||||
Austr
|
||||
ustrali
|
||||
héllo
|
||||
alia
|
||||
@@ -8,22 +8,15 @@
|
||||
- git push --tags
|
||||
- use github actions to publish docker and make github release
|
||||
- check github updated yq action in marketplace
|
||||
- update github-action/Dockerfile to pin the newly published docker image digest (must match the mikefarah/yq:4 manifest digest):
|
||||
docker buildx imagetools inspect docker.io/mikefarah/yq:4 --format '{{printf "%s" .Manifest.Digest}}'
|
||||
- update github-action/Dockerfile to pin the newly published docker image digest:
|
||||
skopeo inspect docker://docker.io/mikefarah/yq:4 | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['Digest'])"
|
||||
then update the FROM line in github-action/Dockerfile with the new digest:
|
||||
FROM mikefarah/yq:4@<digest-from-above>
|
||||
- commit the Dockerfile change, then manually run the "Release Docker GitHub Action" workflow
|
||||
(Actions -> Release Docker GitHub Action -> Run workflow)
|
||||
- update action.yml to pin the newly published github-action image digest (must match the mikefarah/yq:4-githubaction manifest digest):
|
||||
docker buildx imagetools inspect docker.io/mikefarah/yq:4-githubaction --format '{{printf "%s" .Manifest.Digest}}'
|
||||
then update the image line in action.yml with the new digest:
|
||||
image: 'docker://mikefarah/yq:4-githubaction@<digest-from-above>'
|
||||
- commit the action.yml change and push
|
||||
FROM mikefarah/yq:4@sha256:<new-digest>
|
||||
|
||||
// release artifacts are signed with cosign keyless signing (Sigstore)
|
||||
// users can verify with:
|
||||
// cosign verify-blob --bundle checksums.bundle checksums
|
||||
// install cosign: brew install cosign OR go install github.com/sigstore/cosign/v2/cmd/cosign@v2.6.1
|
||||
// install cosign: brew install cosign OR go install github.com/sigstore/cosign/v2/cmd/cosign@latest
|
||||
|
||||
|
||||
- snapcraft
|
||||
|
||||
@@ -1,34 +1,3 @@
|
||||
4.53.4:
|
||||
- Close input files after processing each one (#2796) (#2808) Thanks @MsfPablo
|
||||
- Fixed non string HCL string keys (#2795)
|
||||
- Fixed heap allocation bug (#2809) Thanks @MsfPablo
|
||||
- Fix deleting commented empty list YAML output (#2765) Thanks @dpersek
|
||||
- Fix (has) return false for negative array indices (#2769) Thanks @maximilize
|
||||
- Fix (sort): avoid int64 overflow comparing large integers (#2771) Thanks @maximilize
|
||||
- Fix: preserve correct parent references in explode merge anchor reconstruction (#2730) Thanks @vomba
|
||||
- Improve Guard ExpressionParser initialization with sync.Once (#2789) Thanks @arcaven
|
||||
- Fix: reject negative indent instead of panicking (#2746) Thanks @StressTestor
|
||||
- Fix: parse signed hex and octal integers (#2749) Thanks @StressTestor
|
||||
- Fix: skip UTF-8 BOM when processing front matter (#2751) Thanks @StressTestor
|
||||
- Fix !!merge tag regression for yq (#2705) Thanks @W-Floyd
|
||||
- Default to yaml when a file's extension is not a recognised format (#2785) Thanks @devthedevil
|
||||
- Bumped dependencies
|
||||
|
||||
4.53.3:
|
||||
- Add `--ini-preserve-quotes` flag for INI round-trip quote preservation (#2728) Thanks @toller892!
|
||||
- Fix: reset INI decoder state on init (#2719) Thanks @xieby1!
|
||||
- Fix: decode properties array bracket paths (#2693) Thanks @cyphercodes!
|
||||
- Fix: preserve floats with trailing zero when encoding YAML to JSON (#2701) Thanks @ChrisJr404!
|
||||
- Fix: JSON to TOML root scope and null handling (#2689) Thanks @LovesAsuna!
|
||||
- Fix: reset TOML decoder finished flag on Init for multi-doc evaluation (#2704) Thanks @terminalchai!
|
||||
- Fix: reset TOML decoder between files when evaluating all at once (#2685) Thanks @terminalchai!
|
||||
- Fix: preserve TOML inline table array scope (#2694) Thanks @cyphercodes!
|
||||
- Fix: preserve empty TOML arrays in tables (#2686) Thanks @cyphercodes!
|
||||
- Fix: TOML encoder uses inline tables for YAML FlowStyle mappings (#2687)
|
||||
- Fix nested inline YAML merge explode (#2699) Thanks @cyphercodes!
|
||||
- Fix repeatString overflow test on 32-bit platforms (#2680) Thanks @jandubois!
|
||||
- Bumped dependencies
|
||||
|
||||
4.53.2:
|
||||
- Fixing release process
|
||||
|
||||
|
||||
@@ -8,17 +8,14 @@ fi
|
||||
|
||||
version=$1
|
||||
|
||||
# validate version is in the right format (bash regex — portable; GNU sed's q1 is not on macOS)
|
||||
if [[ ! $version =~ ^v4\.[0-9][0-9]\.[0-9][0-9]?$ ]]; then
|
||||
echo "Please specify a valid version (e.g. v4.53.3)"
|
||||
exit 1
|
||||
fi
|
||||
# validate version is in the right format
|
||||
echo $version | sed -r '/v4\.[0-9][0-9]\.[0-9][0-9]?$/!{q1}'
|
||||
|
||||
previousVersion=$(cat cmd/version.go| sed -n 's/.*Version = "\([^"]*\)"/\1/p')
|
||||
|
||||
echo "Updating from $previousVersion to $version"
|
||||
|
||||
sed "s/\(.*Version =\).*/\1 \"$version\"/" cmd/version.go > cmd/version.go.tmp && mv cmd/version.go.tmp cmd/version.go
|
||||
sed -i "s/\(.*Version =\).*/\1 \"$version\"/" cmd/version.go
|
||||
|
||||
go build .
|
||||
actualVersion=$(./yq --version)
|
||||
|
||||
+3
-48
@@ -1,50 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -ex
|
||||
go mod download golang.org/x/tools@v0.44.0
|
||||
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/6008b81b81c690c046ffc3fd5bce896da715d5fd/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.11.3
|
||||
curl -sSfL https://raw.githubusercontent.com/securego/gosec/424fc4cd9c82ea0fd6bee9cd49c2db2c3cc0c93f/install.sh | sh -s v2.22.11
|
||||
|
||||
TYPOS_VERSION=v1.47.2
|
||||
TYPOS_OS=$(uname -s | tr '[:upper:]' '[:lower:]')
|
||||
TYPOS_ARCH=$(uname -m)
|
||||
case "${TYPOS_ARCH}" in
|
||||
x86_64) TYPOS_ARCH=x86_64 ;;
|
||||
aarch64|arm64) TYPOS_ARCH=aarch64 ;;
|
||||
*)
|
||||
echo "unsupported architecture for typos: ${TYPOS_ARCH}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
case "${TYPOS_OS}" in
|
||||
linux) TYPOS_TARGET="${TYPOS_ARCH}-unknown-linux-musl" ;;
|
||||
darwin) TYPOS_TARGET="${TYPOS_ARCH}-apple-darwin" ;;
|
||||
*)
|
||||
echo "unsupported OS for typos: ${TYPOS_OS}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
TYPOS_ARCHIVE="typos-${TYPOS_VERSION}-${TYPOS_TARGET}.tar.gz"
|
||||
TYPOS_URL="https://github.com/crate-ci/typos/releases/download/${TYPOS_VERSION}/${TYPOS_ARCHIVE}"
|
||||
case "${TYPOS_TARGET}" in
|
||||
aarch64-apple-darwin) TYPOS_SHA256=23ca24a9186b5cb395b5f6c8eea8cdb02911c8980833e016454b56e90c3bd474 ;;
|
||||
aarch64-unknown-linux-musl) TYPOS_SHA256=596d5c6b9ecf34307f68bea649178c5b45a4398fe3a1fcef9598e85aa2ccb742 ;;
|
||||
x86_64-apple-darwin) TYPOS_SHA256=469a2d9fc894b0cdcec6e4fa3719b4c4638e195feee6517d4845450f8e8985c6 ;;
|
||||
x86_64-unknown-linux-musl) TYPOS_SHA256=7aef58932fc123b4cf4b40d86468e89a3297d80169051d7cfd13a235e05fc426 ;;
|
||||
*)
|
||||
echo "unsupported typos target: ${TYPOS_TARGET}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
TYPOS_TMPDIR=$(mktemp -d)
|
||||
curl -sSfL "${TYPOS_URL}" -o "${TYPOS_TMPDIR}/${TYPOS_ARCHIVE}"
|
||||
TYPOS_ACTUAL_SHA256=$(sha256sum "${TYPOS_TMPDIR}/${TYPOS_ARCHIVE}" 2>/dev/null | cut -d' ' -f1)
|
||||
if [ -z "${TYPOS_ACTUAL_SHA256}" ]; then
|
||||
TYPOS_ACTUAL_SHA256=$(shasum -a 256 "${TYPOS_TMPDIR}/${TYPOS_ARCHIVE}" | cut -d' ' -f1)
|
||||
fi
|
||||
if [ "${TYPOS_ACTUAL_SHA256}" != "${TYPOS_SHA256}" ]; then
|
||||
echo "typos archive checksum mismatch: expected ${TYPOS_SHA256}, got ${TYPOS_ACTUAL_SHA256}"
|
||||
exit 1
|
||||
fi
|
||||
tar xzf "${TYPOS_TMPDIR}/${TYPOS_ARCHIVE}" -C "${TYPOS_TMPDIR}"
|
||||
install -m 755 "${TYPOS_TMPDIR}/typos" "$(go env GOPATH)/bin/typos"
|
||||
rm -rf "${TYPOS_TMPDIR}"
|
||||
go mod download golang.org/x/tools@latest
|
||||
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.11.3
|
||||
curl -sSfL https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.22.11
|
||||
+1
-16
@@ -1,18 +1,3 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
GOPATH_TYPOS="$(go env GOPATH)/bin/typos"
|
||||
TYPOS_CMD=""
|
||||
|
||||
if [ -f "${GOPATH_TYPOS}" ]; then
|
||||
TYPOS_CMD="${GOPATH_TYPOS}"
|
||||
elif command -v typos >/dev/null 2>&1; then
|
||||
TYPOS_CMD="typos"
|
||||
else
|
||||
echo "Error: typos not found in $(go env GOPATH)/bin or PATH."
|
||||
echo "Please run scripts/devtools.sh or ensure typos is installed correctly."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
git ls-files '*.go' '*.sh' '*.md' | "${TYPOS_CMD}" --file-list -
|
||||
npx cspell --no-progress "**/*.{sh,go,md}"
|
||||
@@ -1,7 +1,3 @@
|
||||
#!/bin/bash
|
||||
|
||||
go test $(go list ./... | grep -v -E 'examples' | grep -v -E 'test')
|
||||
|
||||
# Run after the main test suite: TestGoInstallCompatibility zips the module tree and
|
||||
# must not run in parallel with pkg/yqlib tests that rewrite doc/*.md files.
|
||||
go test -tags goinstall -run TestGoInstallCompatibility .
|
||||
|
||||
+1
-1
@@ -2,7 +2,7 @@
|
||||
|
||||
set -eo pipefail
|
||||
|
||||
# You may need to go install github.com/goreleaser/goreleaser/v2@v2.16.0 first
|
||||
# You may need to go install github.com/goreleaser/goreleaser/v2@latest first
|
||||
GORELEASER="goreleaser build --clean"
|
||||
if [ -z "$CI" ] || [[ "${GITHUB_REF_NAME:-}" == draft-* ]]; then
|
||||
GORELEASER+=" --snapshot"
|
||||
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
name: yq
|
||||
version: 'v4.53.4'
|
||||
version: 'v4.53.2'
|
||||
summary: A lightweight and portable command-line data file processor
|
||||
description: |
|
||||
`yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml, json, xml, csv, properties and TOML files.
|
||||
@@ -32,6 +32,6 @@ parts:
|
||||
build-environment:
|
||||
- CGO_ENABLED: 0
|
||||
source: https://github.com/mikefarah/yq.git
|
||||
source-tag: v4.53.4
|
||||
source-tag: v4.53.2
|
||||
build-snaps:
|
||||
- go/latest/stable
|
||||
|
||||
Reference in New Issue
Block a user