Compare commits

..
Author SHA1 Message Date
Mike Farah 8372683ef3 Fixing comments in TOML arrays #2592 2026-02-03 19:18:39 +11:00
100 changed files with 362 additions and 1474 deletions
+4 -4
View File
@@ -38,11 +38,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
uses: actions/checkout@v6
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1
uses: github/codeql-action/init@v4
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
@@ -53,7 +53,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1
uses: github/codeql-action/autobuild@v4
# ️ Command-line programs to run using the OS shell.
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -67,4 +67,4 @@ jobs:
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1
uses: github/codeql-action/analyze@v4
+5 -5
View File
@@ -14,16 +14,16 @@ jobs:
IMAGE_NAME: mikefarah/yq
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: actions/checkout@v6
- name: Set up QEMU
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0
uses: docker/setup-qemu-action@v3
with:
platforms: all
- name: Set up Docker Buildx
id: buildx
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
uses: docker/setup-buildx-action@v3
with:
version: latest
@@ -31,13 +31,13 @@ jobs:
run: echo ${{ steps.buildx.outputs.platforms }} && docker version
- name: Login to Docker Hub
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
+2 -2
View File
@@ -11,13 +11,13 @@ jobs:
steps:
- name: Set up Go
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
uses: actions/setup-go@v6
with:
go-version: '^1.20'
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
uses: actions/checkout@v6
- name: Get dependencies
run: |
+3 -14
View File
@@ -8,12 +8,9 @@ on:
jobs:
publishGitRelease:
runs-on: ubuntu-latest
permissions:
contents: write
id-token: write
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
- uses: actions/checkout@v6
- uses: actions/setup-go@v6
with:
go-version: '^1.20'
check-latest: true
@@ -40,22 +37,14 @@ jobs:
--output=yq.1
man.md
- name: Install cosign
uses: sigstore/cosign-installer@v3
- name: Cross compile
run: |
sudo apt-get install rhash -y
go install github.com/goreleaser/goreleaser/v2@latest
./scripts/xcompile.sh
- name: Sign checksums
run: |
cosign sign-blob --yes --output-bundle build/checksums.bundle build/checksums
cosign sign-blob --yes --output-bundle build/checksums-bsd.bundle build/checksums-bsd
- name: Release
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2.6.2
uses: softprops/action-gh-release@v1
with:
files: build/*
draft: true
+3 -3
View File
@@ -12,14 +12,14 @@ jobs:
environment: snap
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: snapcore/action-build@3bdaa03e1ba6bf59a65f84a751d943d549a54e79 # v1.3.0
- uses: actions/checkout@v6
- uses: snapcore/action-build@v1
id: build
env:
SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.STORE_LOGIN }}
with:
snapcraft-args: "remote-build --launchpad-accept-public-upload"
- uses: snapcore/action-publish@214b86e5ca036ead1668c79afb81e550e6c54d40 # v1.2.0
- uses: snapcore/action-publish@v1
env:
SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.STORE_LOGIN }}
with:
+1 -1
View File
@@ -13,7 +13,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- uses: actions/checkout@v6
- name: Get test
id: get_value
uses: mikefarah/yq@master
+1
View File
@@ -39,6 +39,7 @@ builds:
- openbsd_amd64
- windows_386
- windows_amd64
- windows_arm
- windows_arm64
no_unique_dist_dir: true
+2 -2
View File
@@ -1,4 +1,4 @@
FROM golang:1.26.2@sha256:2a2b4b5791cea8ae09caecba7bad0bd9631def96e5fe362e4a5e67009fe4ae61 AS builder
FROM golang:1.25.6 AS builder
WORKDIR /go/src/mikefarah/yq
@@ -10,7 +10,7 @@ RUN ./scripts/acceptance.sh
# Choose alpine as a base image to make this useful for CI, as many
# CI tools expect an interactive shell inside the container
FROM alpine:3@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659 AS production
FROM alpine:3 AS production
LABEL maintainer="Mike Farah <mikefarah@users.noreply.github.com>"
COPY --from=builder /go/src/mikefarah/yq/yq /usr/bin/yq
+1 -1
View File
@@ -1,4 +1,4 @@
FROM golang:1.26.2@sha256:2a2b4b5791cea8ae09caecba7bad0bd9631def96e5fe362e4a5e67009fe4ae61
FROM golang:1.25.6
RUN apt-get update && \
apt-get install -y npm && \
-26
View File
@@ -1,26 +0,0 @@
# Security Policy
## Reporting a Vulnerability
Please **do not** report security vulnerabilities through public GitHub issues.
Instead, use GitHub's private vulnerability reporting feature:
👉 https://github.com/mikefarah/yq/security
This allows vulnerabilities to be triaged and addressed confidentially before any public disclosure.
## Scope
### HTTP / TLS / Network vulnerabilities
yq is a command-line YAML/JSON/TOML processor that reads from files or standard input and writes to standard output. **yq does not include any HTTP or network libraries** and makes no network connections at runtime. CVEs related to HTTP, TLS, or networking are therefore **not applicable** to yq.
### Dependency version bumps
yq uses [Dependabot](https://docs.github.com/en/code-security/dependabot) to automatically raise pull requests for:
- Go module dependencies
- Go toolchain version
- Docker base images
Please **do not** raise pull requests or issues solely to bump dependency or Go versions — Dependabot handles this automatically and the maintainers merge those PRs regularly.
-3
View File
@@ -101,15 +101,12 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
}
if frontMatter != "" {
originalFilename := args[0]
frontMatterHandler := yqlib.NewFrontMatterHandler(args[0])
err = frontMatterHandler.Split()
if err != nil {
return err
}
args[0] = frontMatterHandler.GetYamlFrontMatterFilename()
yqlib.SetFilenameAlias(args[0], originalFilename)
defer yqlib.ClearFilenameAliases()
if frontMatter == "process" {
reader := frontMatterHandler.GetContentReader()
-1
View File
@@ -13,7 +13,6 @@ func TestCreateEvaluateAllCommand(t *testing.T) {
if cmd == nil {
t.Fatal("createEvaluateAllCommand returned nil")
return
}
// Test basic command properties
-3
View File
@@ -122,15 +122,12 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
if frontMatter != "" {
yqlib.GetLogger().Debug("using front matter handler")
originalFilename := args[0]
frontMatterHandler := yqlib.NewFrontMatterHandler(args[0])
err = frontMatterHandler.Split()
if err != nil {
return err
}
args[0] = frontMatterHandler.GetYamlFrontMatterFilename()
yqlib.SetFilenameAlias(args[0], originalFilename)
defer yqlib.ClearFilenameAliases()
if frontMatter == "process" {
reader := frontMatterHandler.GetContentReader()
-1
View File
@@ -13,7 +13,6 @@ func TestCreateEvaluateSequenceCommand(t *testing.T) {
if cmd == nil {
t.Fatal("createEvaluateSequenceCommand returned nil")
return
}
// Test basic command properties
+17 -9
View File
@@ -2,12 +2,12 @@ package cmd
import (
"fmt"
"log/slog"
"os"
"strings"
"github.com/mikefarah/yq/v4/pkg/yqlib"
"github.com/spf13/cobra"
logging "gopkg.in/op/go-logging.v1"
)
type runeValue rune
@@ -68,21 +68,30 @@ yq -P -oy sample.json
},
PersistentPreRunE: func(cmd *cobra.Command, _ []string) error {
cmd.SetOut(cmd.OutOrStdout())
level := logging.WARNING
stringFormat := `[%{level}] %{color}%{time:15:04:05}%{color:reset} %{message}`
// when NO_COLOR environment variable presents and not an empty string the coloured output should be disabled;
// refer to no-color.org
forceNoColor = forceNoColor || os.Getenv("NO_COLOR") != ""
level := slog.LevelWarn
if verbose {
level = slog.LevelDebug
if verbose && forceNoColor {
level = logging.DEBUG
stringFormat = `[%{level:5.5s}] %{time:15:04:05} %{shortfile:-33s} %{shortfunc:-25s} %{message}`
} else if verbose {
level = logging.DEBUG
stringFormat = `[%{level:5.5s}] %{color}%{time:15:04:05}%{color:bold} %{shortfile:-33s} %{shortfunc:-25s}%{color:reset} %{message}`
} else if forceNoColor {
stringFormat = `[%{level}] %{time:15:04:05} %{message}`
}
yqlib.GetLogger().SetLevel(level)
opts := &slog.HandlerOptions{Level: level, AddSource: verbose}
handler := slog.NewTextHandler(os.Stderr, opts)
yqlib.GetLogger().SetSlogger(slog.New(handler))
var format = logging.MustStringFormatter(stringFormat)
var backend = logging.AddModuleLevel(
logging.NewBackendFormatter(logging.NewLogBackend(os.Stderr, "", 0), format))
backend.SetLevel(level, "")
logging.SetBackend(backend)
yqlib.InitExpressionParser()
return nil
@@ -212,7 +221,6 @@ yq -P -oy sample.json
rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableEnvOps, "security-disable-env-ops", "", false, "Disable env related operations.")
rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableFileOps, "security-disable-file-ops", "", false, "Disable file related operations (e.g. load)")
rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.EnableSystemOps, "security-enable-system-operator", "", false, "Enable system operator to allow execution of external commands.")
rootCmd.AddCommand(
createEvaluateSequenceCommand(),
-1
View File
@@ -195,7 +195,6 @@ func TestNew(t *testing.T) {
if rootCmd == nil {
t.Fatal("New() returned nil")
return
}
// Test basic command properties
+8 -8
View File
@@ -3,12 +3,12 @@ package cmd
import (
"fmt"
"io"
"log/slog"
"os"
"strings"
"github.com/mikefarah/yq/v4/pkg/yqlib"
"github.com/spf13/cobra"
"gopkg.in/op/go-logging.v1"
)
func isAutomaticOutputFormat() bool {
@@ -104,8 +104,8 @@ func configureFormats(args []string) error {
return err
}
yqlib.GetLogger().Debugf("Using input format %v", inputFormat)
yqlib.GetLogger().Debugf("Using output format %v", outputFormat)
yqlib.GetLogger().Debug("Using input format %v", inputFormat)
yqlib.GetLogger().Debug("Using output format %v", outputFormat)
return nil
}
@@ -117,7 +117,7 @@ func configureInputFormat(inputFilename string) error {
_, err := yqlib.FormatFromString(inputFormat)
if err != nil {
// unknown file type, default to yaml
yqlib.GetLogger().Debugf("Unknown file format extension '%v', defaulting to yaml", inputFormat)
yqlib.GetLogger().Debug("Unknown file format extension '%v', defaulting to yaml", inputFormat)
inputFormat = "yaml"
if isAutomaticOutputFormat() {
outputFormat = "yaml"
@@ -132,7 +132,7 @@ func configureInputFormat(inputFilename string) error {
//
outputFormat = yqlib.FormatStringFromFilename(inputFilename)
if inputFilename != "-" {
yqlib.GetLogger().Warningf("yq default output is now 'auto' (based on the filename extension). Normally yq would output '%v', but for backwards compatibility 'yaml' has been set. Please use -oy to specify yaml, or drop the -p flag.", outputFormat)
yqlib.GetLogger().Warning("yq default output is now 'auto' (based on the filename extension). Normally yq would output '%v', but for backwards compatibility 'yaml' has been set. Please use -oy to specify yaml, or drop the -p flag.", outputFormat)
}
outputFormat = "yaml"
}
@@ -235,7 +235,7 @@ func maybeFile(str string) bool {
yqlib.GetLogger().Debugf("checking '%v' is a file", str)
stat, err := os.Stat(str) // #nosec
result := err == nil && !stat.IsDir()
if yqlib.GetLogger().IsEnabledFor(slog.LevelDebug) {
if yqlib.GetLogger().IsEnabledFor(logging.DEBUG) {
if err != nil {
yqlib.GetLogger().Debugf("error: %v", err)
} else {
@@ -280,7 +280,7 @@ func processArgs(originalArgs []string) (string, []string, error) {
if expressionFile == "" && maybeFirstArgIsAFile && strings.HasSuffix(args[0], ".yq") {
// lets check if an expression file was given
yqlib.GetLogger().Debugf("Assuming arg %v is an expression file", args[0])
yqlib.GetLogger().Debug("Assuming arg %v is an expression file", args[0])
expressionFile = args[0]
args = args[1:]
}
@@ -296,7 +296,7 @@ func processArgs(originalArgs []string) (string, []string, error) {
yqlib.GetLogger().Debugf("processed args: %v", args)
if expression == "" && len(args) > 0 && args[0] != "-" && !maybeFile(args[0]) {
yqlib.GetLogger().Debugf("assuming expression is '%v'", args[0])
yqlib.GetLogger().Debug("assuming expression is '%v'", args[0])
expression = args[0]
args = args[1:]
}
+1 -1
View File
@@ -911,7 +911,7 @@ func stringsEqual(a, b []string) bool {
return false
}
for i := range a {
if a[i] != b[i] { //nolint:gosec // G602 false positive: b length equality is checked above
if a[i] != b[i] {
return false
}
}
+1 -1
View File
@@ -11,7 +11,7 @@ var (
GitDescribe string
// Version is main version number that is being run at the moment.
Version = "v4.52.5"
Version = "v4.52.2"
// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
+1 -1
View File
@@ -1,4 +1,4 @@
FROM mikefarah/yq:4@sha256:603ebff15eb308a05f1c5b8b7613179cad859aed3ec9fdd04f2ef5d32345950e
FROM mikefarah/yq:4
COPY entrypoint.sh /entrypoint.sh
+16 -13
View File
@@ -6,23 +6,24 @@ require (
github.com/alecthomas/repr v0.5.2
github.com/dimchansky/utfbom v1.1.1
github.com/elliotchance/orderedmap v1.8.0
github.com/fatih/color v1.19.0
github.com/fatih/color v1.18.0
github.com/go-ini/ini v1.67.0
github.com/goccy/go-json v0.10.6
github.com/goccy/go-json v0.10.5
github.com/goccy/go-yaml v1.19.2
github.com/hashicorp/hcl/v2 v2.24.0
github.com/jinzhu/copier v0.4.0
github.com/magiconair/properties v1.8.10
github.com/pelletier/go-toml/v2 v2.3.0
github.com/pelletier/go-toml/v2 v2.2.4
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
github.com/spf13/cobra v1.10.2
github.com/spf13/pflag v1.0.10
github.com/yuin/gopher-lua v1.1.2
github.com/zclconf/go-cty v1.18.0
go.yaml.in/yaml/v4 v4.0.0-rc.4
golang.org/x/mod v0.34.0
golang.org/x/net v0.52.0
golang.org/x/text v0.35.0
github.com/yuin/gopher-lua v1.1.1
github.com/zclconf/go-cty v1.17.0
go.yaml.in/yaml/v4 v4.0.0-rc.3
golang.org/x/mod v0.31.0
golang.org/x/net v0.49.0
golang.org/x/text v0.33.0
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
)
require (
@@ -33,9 +34,11 @@ require (
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
golang.org/x/sync v0.20.0 // indirect
golang.org/x/sys v0.42.0 // indirect
golang.org/x/tools v0.42.0 // indirect
golang.org/x/sync v0.19.0 // indirect
golang.org/x/sys v0.40.0 // indirect
golang.org/x/tools v0.40.0 // indirect
)
go 1.25.0
go 1.24.0
toolchain go1.24.1
+26 -24
View File
@@ -18,14 +18,14 @@ github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
github.com/elliotchance/orderedmap v1.8.0 h1:TrOREecvh3JbS+NCgwposXG5ZTFHtEsQiCGOhPElnMw=
github.com/elliotchance/orderedmap v1.8.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w=
github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE=
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU=
github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM=
github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
@@ -46,8 +46,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
github.com/pelletier/go-toml/v2 v2.3.0 h1:k59bC/lIZREW0/iVaQR8nDHxVq8OVlIzYCOJf421CaM=
github.com/pelletier/go-toml/v2 v2.3.0/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -61,28 +61,30 @@ github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3A
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/yuin/gopher-lua v1.1.2 h1:yF/FjE3hD65tBbt0VXLE13HWS9h34fdzJmrWRXwobGA=
github.com/yuin/gopher-lua v1.1.2/go.mod h1:7aRmXIWl37SqRf0koeyylBEzJ+aPt8A+mmkQ4f1ntR8=
github.com/zclconf/go-cty v1.18.0 h1:pJ8+HNI4gFoyRNqVE37wWbJWVw43BZczFo7KUoRczaA=
github.com/zclconf/go-cty v1.18.0/go.mod h1:qpnV6EDNgC1sns/AleL1fvatHw72j+S+nS+MJ+T2CSg=
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
github.com/zclconf/go-cty v1.17.0 h1:seZvECve6XX4tmnvRzWtJNHdscMtYEx5R7bnnVyd/d0=
github.com/zclconf/go-cty v1.17.0/go.mod h1:wqFzcImaLTI6A5HfsRwB0nj5n0MRZFwmey8YoFPPs3U=
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo=
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
go.yaml.in/yaml/v4 v4.0.0-rc.4 h1:UP4+v6fFrBIb1l934bDl//mmnoIZEDK0idg1+AIvX5U=
go.yaml.in/yaml/v4 v4.0.0-rc.4/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go=
go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI=
golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg=
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA=
golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW8s2qTSe3wGBtvo0MbVQG/c5k8RE=
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-22
View File
@@ -54,25 +54,3 @@ func TestAllAtOnceEvaluateNodes(t *testing.T) {
test.AssertResultComplex(t, tt.expected, resultsToString(t, list))
}
}
func TestTomlDecoderCanBeReinitializedAcrossDocuments(t *testing.T) {
decoder := NewTomlDecoder()
firstDocuments, err := ReadDocuments(strings.NewReader("id = \"Foobar\"\n"), decoder)
if err != nil {
t.Fatalf("failed to read first TOML document: %v", err)
}
if firstDocuments.Len() != 1 {
t.Fatalf("expected first document count to be 1, got %d", firstDocuments.Len())
}
test.AssertResult(t, "Foobar", firstDocuments.Front().Value.(*CandidateNode).Content[1].Value)
secondDocuments, err := ReadDocuments(strings.NewReader("id = \"Banana\"\n"), decoder)
if err != nil {
t.Fatalf("failed to read second TOML document: %v", err)
}
if secondDocuments.Len() != 1 {
t.Fatalf("expected second document count to be 1, got %d", secondDocuments.Len())
}
test.AssertResult(t, "Banana", secondDocuments.Front().Value.(*CandidateNode).Content[1].Value)
}
+10 -26
View File
@@ -27,22 +27,6 @@ const (
FlowStyle
)
// EncodeHint controls how a mapping node is serialised by format-specific encoders
// that distinguish between inline and block/section representations (e.g. TOML, HCL).
type EncodeHint int
const (
// EncodeHintDefault lets the encoder choose the representation (e.g. TOML block
// mappings default to [section] headers).
EncodeHintDefault EncodeHint = iota
// EncodeHintSeparateBlock forces the node to be emitted as a separate block or
// table-section header (used by TOML [section] and HCL block decoders).
EncodeHintSeparateBlock
// EncodeHintInline forces the node to be emitted as an inline / flow table
// (used by TOML inline-table decoder and TOML encoder).
EncodeHintInline
)
func createStringScalarNode(stringValue string) *CandidateNode {
var node = &CandidateNode{Kind: ScalarNode}
node.Value = stringValue
@@ -113,9 +97,9 @@ type CandidateNode struct {
// (e.g. top level cross document merge). This property does not propagate to child nodes.
EvaluateTogether bool
IsMapKey bool
// EncodeHint controls how a mapping node is serialised by format-specific encoders
// (e.g. TOML, HCL) that support both inline and block/section representations.
EncodeHint EncodeHint
// For formats like HCL and TOML: indicates that child entries should be emitted as separate blocks/tables
// rather than consolidated into nested mappings (default behaviour)
EncodeSeparate bool
}
func (n *CandidateNode) CreateChild() *CandidateNode {
@@ -296,7 +280,7 @@ func (n *CandidateNode) AddChild(rawChild *CandidateNode) {
func (n *CandidateNode) AddChildren(children []*CandidateNode) {
if n.Kind == MappingNode {
for i := 0; i < len(children)-1; i += 2 {
for i := 0; i < len(children); i += 2 {
key := children[i]
value := children[i+1]
n.AddKeyValueChild(key, value)
@@ -339,11 +323,11 @@ func (n *CandidateNode) guessTagFromCustomType() string {
dataBucket, errorReading := parseSnippet(n.Value)
if errorReading != nil {
log.Debugf("guessTagFromCustomType: could not guess underlying tag type %v", errorReading)
log.Debug("guessTagFromCustomType: could not guess underlying tag type %v", errorReading)
return n.Tag
}
guessedTag := dataBucket.Tag
log.Infof("im guessing the tag %v is a %v", n.Tag, guessedTag)
log.Info("im guessing the tag %v is a %v", n.Tag, guessedTag)
return guessedTag
}
@@ -427,7 +411,7 @@ func (n *CandidateNode) doCopy(cloneContent bool) *CandidateNode {
EvaluateTogether: n.EvaluateTogether,
IsMapKey: n.IsMapKey,
EncodeHint: n.EncodeHint,
EncodeSeparate: n.EncodeSeparate,
}
if cloneContent {
@@ -461,7 +445,7 @@ func (n *CandidateNode) UpdateFrom(other *CandidateNode, prefs assignPreferences
}
func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignPreferences) {
log.Debugf("UpdateAttributesFrom: n: %v other: %v", NodeToString(n), NodeToString(other))
log.Debug("UpdateAttributesFrom: n: %v other: %v", NodeToString(n), NodeToString(other))
if n.Kind != other.Kind {
// clear out the contents when switching to a different type
// e.g. map to array
@@ -481,8 +465,8 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignP
n.Anchor = other.Anchor
}
// Preserve EncodeHint for format-specific encoding hints
n.EncodeHint = other.EncodeHint
// Preserve EncodeSeparate flag for format-specific encoding hints
n.EncodeSeparate = other.EncodeSeparate
// merge will pickup the style of the new thing
// when autocreating nodes
+7 -7
View File
@@ -36,13 +36,13 @@ func (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap, an
switch commentMapComment.Position {
case yaml.CommentHeadPosition:
o.HeadComment = comment.String()
log.Debugf("its a head comment %v", comment.String())
log.Debug("its a head comment %v", comment.String())
case yaml.CommentLinePosition:
o.LineComment = comment.String()
log.Debugf("its a line comment %v", comment.String())
log.Debug("its a line comment %v", comment.String())
case yaml.CommentFootPosition:
o.FootComment = comment.String()
log.Debugf("its a foot comment %v", comment.String())
log.Debug("its a foot comment %v", comment.String())
}
}
}
@@ -93,8 +93,8 @@ func (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap, an
log.Debugf("folded Type %v", astLiteral.Start.Type)
o.Style = FoldedStyle
}
log.Debugf("start value: %v ", node.(*ast.LiteralNode).Start.Value)
log.Debugf("start value: %v ", node.(*ast.LiteralNode).Start.Type)
log.Debug("start value: %v ", node.(*ast.LiteralNode).Start.Value)
log.Debug("start value: %v ", node.(*ast.LiteralNode).Start.Type)
// TODO: here I could put the original value with line breaks
// to solve the multiline > problem
o.Value = astLiteral.Value.Value
@@ -187,7 +187,7 @@ func (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap, an
}
func (o *CandidateNode) goccyProcessMappingValueNode(mappingEntry *ast.MappingValueNode, cm yaml.CommentMap, anchorMap map[string]*CandidateNode) error {
log.Debugf("UnmarshalYAML MAP KEY entry %v", mappingEntry.Key)
log.Debug("UnmarshalYAML MAP KEY entry %v", mappingEntry.Key)
// AddKeyValueFirst because it clones the nodes, and we want to have the real refs when Unmarshalling
// particularly for the anchorMap
@@ -197,7 +197,7 @@ func (o *CandidateNode) goccyProcessMappingValueNode(mappingEntry *ast.MappingVa
return err
}
log.Debugf("UnmarshalYAML MAP VALUE entry %v", mappingEntry.Value)
log.Debug("UnmarshalYAML MAP VALUE entry %v", mappingEntry.Value)
if err := valueNode.UnmarshalGoccyYAML(mappingEntry.Value, cm, anchorMap); err != nil {
return err
}
+6 -6
View File
@@ -55,13 +55,13 @@ func (o *CandidateNode) copyFromYamlNode(node *yaml.Node, anchorMap map[string]*
if o.Anchor != "" {
anchorMap[o.Anchor] = o
log.Debugf("set anchor %v to %v", o.Anchor, NodeToString(o))
log.Debug("set anchor %v to %v", o.Anchor, NodeToString(o))
}
// its a single alias
if node.Alias != nil && node.Alias.Anchor != "" {
o.Alias = anchorMap[node.Alias.Anchor]
log.Debugf("set alias to %v", NodeToString(anchorMap[node.Alias.Anchor]))
log.Debug("set alias to %v", NodeToString(anchorMap[node.Alias.Anchor]))
}
o.HeadComment = node.HeadComment
o.LineComment = node.LineComment
@@ -106,7 +106,7 @@ func (o *CandidateNode) UnmarshalYAML(node *yaml.Node, anchorMap map[string]*Can
log.Debugf("UnmarshalYAML %v", node.Tag)
switch node.Kind {
case yaml.AliasNode:
log.Debugf("UnmarshalYAML - alias from yaml: %v", o.Tag)
log.Debug("UnmarshalYAML - alias from yaml: %v", o.Tag)
o.Kind = AliasNode
o.copyFromYamlNode(node, anchorMap)
return nil
@@ -176,15 +176,15 @@ func (o *CandidateNode) UnmarshalYAML(node *yaml.Node, anchorMap map[string]*Can
}
func (o *CandidateNode) MarshalYAML() (*yaml.Node, error) {
log.Debugf("MarshalYAML to yaml: %v", o.Tag)
log.Debug("MarshalYAML to yaml: %v", o.Tag)
switch o.Kind {
case AliasNode:
log.Debugf("MarshalYAML - alias to yaml: %v", o.Tag)
log.Debug("MarshalYAML - alias to yaml: %v", o.Tag)
target := &yaml.Node{Kind: yaml.AliasNode}
o.copyToYamlNode(target)
return target, nil
case ScalarNode:
log.Debugf("MarshalYAML - scalar: %v", o.Value)
log.Debug("MarshalYAML - scalar: %v", o.Value)
target := &yaml.Node{Kind: yaml.ScalarNode}
o.copyToYamlNode(target)
return target, nil
@@ -120,7 +120,7 @@ func (o *CandidateNode) UnmarshalJSON(data []byte) error {
if err != nil {
return err
}
log.Debugf("UnmarshalJSON - scalar is %v", scalar)
log.Debug("UnmarshalJSON - scalar is %v", scalar)
return o.setScalarFromJson(scalar)
+1 -1
View File
@@ -18,7 +18,7 @@ func changeOwner(info fs.FileInfo, file *os.File) error {
// this happens with snap confinement
// not really a big issue as users can chown
// the file themselves if required.
log.Infof("Skipping chown: %v", err)
log.Info("Skipping chown: %v", err)
}
}
return nil
+3 -2
View File
@@ -3,8 +3,9 @@ package yqlib
import (
"container/list"
"fmt"
"log/slog"
"time"
logging "gopkg.in/op/go-logging.v1"
)
type Context struct {
@@ -74,7 +75,7 @@ func (n *Context) ChildContext(results *list.List) Context {
}
func (n *Context) ToString() string {
if !log.IsEnabledFor(slog.LevelDebug) {
if !log.IsEnabledFor(logging.DEBUG) {
return ""
}
result := fmt.Sprintf("Context\nDontAutoCreate: %v\n", n.DontAutoCreate)
+3 -3
View File
@@ -2,11 +2,11 @@ package yqlib
import (
"container/list"
"log/slog"
"strings"
"testing"
"github.com/mikefarah/yq/v4/test"
logging "gopkg.in/op/go-logging.v1"
)
func TestChildContext(t *testing.T) {
@@ -155,8 +155,8 @@ func TestToString(t *testing.T) {
test.AssertResultComplex(t, "", result)
// Test with debug logging enabled
GetLogger().SetLevel(slog.LevelDebug)
defer GetLogger().SetLevel(slog.LevelWarn) // Reset to default
logging.SetLevel(logging.DEBUG, "")
defer logging.SetLevel(logging.INFO, "") // Reset to default
result2 := context.ToString()
test.AssertResultComplex(t, true, len(result2) > 0)
+3 -2
View File
@@ -2,7 +2,8 @@ package yqlib
import (
"fmt"
"log/slog"
logging "gopkg.in/op/go-logging.v1"
)
type DataTreeNavigator interface {
@@ -54,7 +55,7 @@ func (d *dataTreeNavigator) GetMatchingNodes(context Context, expressionNode *Ex
return context, nil
}
log.Debugf("Processing Op: %v", expressionNode.Operation.toString())
if log.IsEnabledFor(slog.LevelDebug) {
if log.IsEnabledFor(logging.DEBUG) {
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
log.Debug(NodeToString(el.Value.(*CandidateNode)))
}
+1 -1
View File
@@ -226,7 +226,7 @@ func addBlockToMapping(parent *CandidateNode, block *hclsyntax.Block, src []byte
// Mark the type node if there are multiple blocks of this type at this level
// This tells the encoder to emit them as separate blocks rather than consolidating them
if isMultipleBlocksOfType {
typeNode.EncodeHint = EncodeHintSeparateBlock
typeNode.EncodeSeparate = true
}
}
current = typeNode
+1 -1
View File
@@ -68,7 +68,7 @@ func mustProcessFormatScenario(s formatScenario, decoder Decoder, encoder Encode
result, err := processFormatScenario(s, decoder, encoder)
if err != nil {
log.Errorf("Bad scenario %v: %v", s.description, err)
log.Error("Bad scenario %v: %w", s.description, err)
return fmt.Sprintf("Bad scenario %v: %v", s.description, err.Error())
}
return result
+18 -20
View File
@@ -44,7 +44,6 @@ func (dec *tomlDecoder) Init(reader io.Reader) error {
}
dec.pendingComments = make([]string, 0)
dec.firstContentSeen = false
dec.finished = false
return nil
}
@@ -106,7 +105,7 @@ func (dec *tomlDecoder) decodeKeyValuesIntoMap(rootMap *CandidateNode, tomlNode
for dec.parser.NextExpression() {
nextItem := dec.parser.Expression()
log.Debugf("decodeKeyValuesIntoMap -- next exp, its a %v", nextItem.Kind)
log.Debug("decodeKeyValuesIntoMap -- next exp, its a %v", nextItem.Kind)
switch nextItem.Kind {
case toml.KeyValue:
@@ -118,7 +117,7 @@ func (dec *tomlDecoder) decodeKeyValuesIntoMap(rootMap *CandidateNode, tomlNode
dec.pendingComments = append(dec.pendingComments, string(nextItem.Data))
default:
// run out of key values
log.Debugf("done in decodeKeyValuesIntoMap, gota a %v", nextItem.Kind)
log.Debug("done in decodeKeyValuesIntoMap, gota a %v", nextItem.Kind)
return true, nil
}
}
@@ -150,10 +149,9 @@ func (dec *tomlDecoder) createInlineTableMap(tomlNode *toml.Node) (*CandidateNod
}
return &CandidateNode{
Kind: MappingNode,
Tag: "!!map",
EncodeHint: EncodeHintInline,
Content: content,
Kind: MappingNode,
Tag: "!!map",
Content: content,
}, nil
}
@@ -272,7 +270,7 @@ func (dec *tomlDecoder) Decode() (*CandidateNode, error) {
currentNode := dec.parser.Expression()
log.Debugf("currentNode: %v ", currentNode.Kind)
log.Debug("currentNode: %v ", currentNode.Kind)
runAgainstCurrentExp, err = dec.processTopLevelNode(currentNode)
if err != nil {
return dec.rootMap, err
@@ -299,7 +297,7 @@ func (dec *tomlDecoder) Decode() (*CandidateNode, error) {
func (dec *tomlDecoder) processTopLevelNode(currentNode *toml.Node) (bool, error) {
var runAgainstCurrentExp bool
var err error
log.Debugf("processTopLevelNode: Going to process %v state is current %v", currentNode.Kind, NodeToString(dec.rootMap))
log.Debug("processTopLevelNode: Going to process %v state is current %v", currentNode.Kind, NodeToString(dec.rootMap))
switch currentNode.Kind {
case toml.Comment:
// Collect comment to attach to next element
@@ -327,7 +325,7 @@ func (dec *tomlDecoder) processTopLevelNode(currentNode *toml.Node) (bool, error
runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(dec.rootMap, currentNode)
}
log.Debugf("processTopLevelNode: DONE Processing state is now %v", NodeToString(dec.rootMap))
log.Debug("processTopLevelNode: DONE Processing state is now %v", NodeToString(dec.rootMap))
return runAgainstCurrentExp, err
}
@@ -335,7 +333,7 @@ func (dec *tomlDecoder) processTable(currentNode *toml.Node) (bool, error) {
log.Debug("Enter processTable")
child := currentNode.Child()
fullPath := dec.getFullPath(child)
log.Debugf("fullpath: %v", fullPath)
log.Debug("fullpath: %v", fullPath)
c := Context{}
c = c.SingleChildContext(dec.rootMap)
@@ -346,10 +344,10 @@ func (dec *tomlDecoder) processTable(currentNode *toml.Node) (bool, error) {
}
tableNodeValue := &CandidateNode{
Kind: MappingNode,
Tag: "!!map",
Content: make([]*CandidateNode, 0),
EncodeHint: EncodeHintSeparateBlock,
Kind: MappingNode,
Tag: "!!map",
Content: make([]*CandidateNode, 0),
EncodeSeparate: true,
}
// Attach pending head comments to the table
@@ -403,7 +401,7 @@ func (dec *tomlDecoder) processTable(currentNode *toml.Node) (bool, error) {
}
func (dec *tomlDecoder) arrayAppend(context Context, path []interface{}, rhsNode *CandidateNode) error {
log.Debugf("arrayAppend to path: %v,%v", path, NodeToString(rhsNode))
log.Debug("arrayAppend to path: %v,%v", path, NodeToString(rhsNode))
rhsCandidateNode := &CandidateNode{
Kind: SequenceNode,
Tag: "!!seq",
@@ -428,7 +426,7 @@ func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error)
log.Debug("Enter processArrayTable")
child := currentNode.Child()
fullPath := dec.getFullPath(child)
log.Debugf("Fullpath: %v", fullPath)
log.Debug("Fullpath: %v", fullPath)
c := Context{}
c = c.SingleChildContext(dec.rootMap)
@@ -443,9 +441,9 @@ func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error)
hasValue := dec.parser.NextExpression()
tableNodeValue := &CandidateNode{
Kind: MappingNode,
Tag: "!!map",
EncodeHint: EncodeHintSeparateBlock,
Kind: MappingNode,
Tag: "!!map",
EncodeSeparate: true,
}
// Attach pending head comments to the array table
+15 -15
View File
@@ -64,7 +64,7 @@ func (dec *xmlDecoder) processComment(c string) string {
}
func (dec *xmlDecoder) createMap(n *xmlNode) (*CandidateNode, error) {
log.Debugf("createMap: headC: %v, lineC: %v, footC: %v", n.HeadComment, n.LineComment, n.FootComment)
log.Debug("createMap: headC: %v, lineC: %v, footC: %v", n.HeadComment, n.LineComment, n.FootComment)
yamlNode := &CandidateNode{Kind: MappingNode, Tag: "!!map"}
if len(n.Data) > 0 {
@@ -92,7 +92,7 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*CandidateNode, error) {
log.Debugf("label=%v, i=%v, keyValuePair.FootComment: %v", label, i, keyValuePair.FootComment)
labelNode.FootComment = dec.processComment(keyValuePair.FootComment)
log.Debugf("len of children in %v is %v", label, len(children))
log.Debug("len of children in %v is %v", label, len(children))
if len(children) > 1 {
valueNode, err = dec.createSequence(children)
if err != nil {
@@ -105,7 +105,7 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*CandidateNode, error) {
if len(children[0].Children) == 0 && children[0].HeadComment != "" {
if len(children[0].Data) > 0 {
log.Debugf("scalar comment hack, currentlabel [%v]", labelNode.HeadComment)
log.Debug("scalar comment hack, currentlabel [%v]", labelNode.HeadComment)
labelNode.HeadComment = joinComments([]string{labelNode.HeadComment, strings.TrimSpace(children[0].HeadComment)}, "\n")
children[0].HeadComment = ""
} else {
@@ -151,7 +151,7 @@ func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*CandidateNode, error) {
scalar := dec.createValueNodeFromData(n.Data)
log.Debugf("scalar (%v), headC: %v, lineC: %v, footC: %v", scalar.Tag, n.HeadComment, n.LineComment, n.FootComment)
log.Debug("scalar (%v), headC: %v, lineC: %v, footC: %v", scalar.Tag, n.HeadComment, n.LineComment, n.FootComment)
scalar.HeadComment = dec.processComment(n.HeadComment)
scalar.LineComment = dec.processComment(n.LineComment)
if scalar.Tag == "!!seq" {
@@ -211,17 +211,17 @@ func (n *xmlNode) AddChild(s string, c *xmlNode) {
if n.Children == nil {
n.Children = make([]*xmlChildrenKv, 0)
}
log.Debugf("looking for %s", s)
log.Debug("looking for %s", s)
// see if we can find an existing entry to add to
for _, childEntry := range n.Children {
if childEntry.K == s {
log.Debugf("found it, appending an entry%s", s)
log.Debug("found it, appending an entry%s", s)
childEntry.V = append(childEntry.V, c)
log.Debugf("yay len of children in %v is %v", s, len(childEntry.V))
log.Debug("yay len of children in %v is %v", s, len(childEntry.V))
return
}
}
log.Debugf("not there, making a new one %s", s)
log.Debug("not there, making a new one %s", s)
n.Children = append(n.Children, &xmlChildrenKv{K: s, V: []*xmlNode{c}})
}
@@ -267,7 +267,7 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
switch se := t.(type) {
case xml.StartElement:
log.Debugf("start element %v", se.Name.Local)
log.Debug("start element %v", se.Name.Local)
elem.state = "started"
// Build new a new current element and link it to its parent
var label = se.Name.Local
@@ -302,14 +302,14 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
if len(newBit) > 0 {
elem.n.Data = append(elem.n.Data, newBit)
elem.state = "chardata"
log.Debugf("chardata [%v] for %v", elem.n.Data, elem.label)
log.Debug("chardata [%v] for %v", elem.n.Data, elem.label)
}
case xml.EndElement:
if elem == nil {
log.Debug("no element, probably bad xml")
continue
}
log.Debugf("end element %v", elem.label)
log.Debug("end element %v", elem.label)
elem.state = "finished"
// And add it to its parent list
if elem.parent != nil {
@@ -326,10 +326,10 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
applyFootComment(elem, commentStr)
case "chardata":
log.Debugf("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
log.Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
elem.n.LineComment = joinComments([]string{elem.n.LineComment, commentStr}, " ")
default:
log.Debugf("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
log.Debug("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
elem.n.HeadComment = joinComments([]string{elem.n.HeadComment, commentStr}, " ")
}
@@ -354,7 +354,7 @@ func applyFootComment(elem *element, commentStr string) {
if len(elem.n.Children) > 0 {
lastChildIndex := len(elem.n.Children) - 1
childKv := elem.n.Children[lastChildIndex]
log.Debugf("got a foot comment, putting on last child for %v: [%v]", childKv.K, commentStr)
log.Debug("got a foot comment, putting on last child for %v: [%v]", childKv.K, commentStr)
// if it's an array of scalars, put the foot comment on the scalar itself
if len(childKv.V) > 0 && len(childKv.V[0].Children) == 0 {
nodeToUpdate := childKv.V[len(childKv.V)-1]
@@ -363,7 +363,7 @@ func applyFootComment(elem *element, commentStr string) {
childKv.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ")
}
} else {
log.Debugf("got a foot comment for %v: [%v]", elem.label, commentStr)
log.Debug("got a foot comment for %v: [%v]", elem.label, commentStr)
elem.n.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ")
}
}
@@ -1,5 +1,5 @@
# Slice Array or String
# Slice/Splice Array
The slice operator works on both arrays and strings. Like the `jq` equivalent, `.[10:15]` will return a subarray (or substring) of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array or string.
The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.
You may leave out the first or second number, which will refer to the start or end of the array or string respectively.
You may leave out the first or second number, which will refer to the start or end of the array respectively.
@@ -1,27 +0,0 @@
# System Operators
The `system` operator allows you to run an external command and use its output as a value in your expression.
**Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it.
**Note:** When enabled, the system operator can replicate the functionality of `env` and `load`
operators via external commands. Enabling it effectively overrides `--security-disable-env-ops`
and `--security-disable-file-ops`.
## Usage
```bash
yq --security-enable-system-operator --null-input '.field = system("command"; "arg1")'
```
The operator takes:
- A command string (required)
- An argument (or an array of arguments), separated from the command by `;` (optional)
The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string.
## Disabling the system operator
The system operator is disabled by default. When disabled, an error is returned instead of running the command, consistent with `--security-disable-env-ops` and `--security-disable-file-ops`.
Use `--security-enable-system-operator` flag to enable it.
+3 -81
View File
@@ -1,8 +1,8 @@
# Slice Array or String
# Slice/Splice Array
The slice operator works on both arrays and strings. Like the `jq` equivalent, `.[10:15]` will return a subarray (or substring) of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array or string.
The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.
You may leave out the first or second number, which will refer to the start or end of the array or string respectively.
You may leave out the first or second number, which will refer to the start or end of the array respectively.
## Slicing arrays
Given a sample.yml file of:
@@ -103,81 +103,3 @@ will output
- cow
```
## Slicing strings
Given a sample.yml file of:
```yaml
country: Australia
```
then
```bash
yq '.country[0:5]' sample.yml
```
will output
```yaml
Austr
```
## Slicing strings - without the second number
Finishes at the end of the string
Given a sample.yml file of:
```yaml
country: Australia
```
then
```bash
yq '.country[5:]' sample.yml
```
will output
```yaml
alia
```
## Slicing strings - without the first number
Starts from the start of the string
Given a sample.yml file of:
```yaml
country: Australia
```
then
```bash
yq '.country[:5]' sample.yml
```
will output
```yaml
Austr
```
## Slicing strings - use negative numbers to count backwards from the end
Negative indices count from the end of the string
Given a sample.yml file of:
```yaml
country: Australia
```
then
```bash
yq '.country[-5:]' sample.yml
```
will output
```yaml
ralia
```
## Slicing strings - Unicode
Indices are rune-based, so multi-byte characters are handled correctly
Given a sample.yml file of:
```yaml
greeting: héllo
```
then
```bash
yq '.greeting[1:3]' sample.yml
```
will output
```yaml
él
```
@@ -1,76 +0,0 @@
# System Operators
The `system` operator allows you to run an external command and use its output as a value in your expression.
**Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it.
**Note:** When enabled, the system operator can replicate the functionality of `env` and `load`
operators via external commands. Enabling it effectively overrides `--security-disable-env-ops`
and `--security-disable-file-ops`.
## Usage
```bash
yq --security-enable-system-operator --null-input '.field = system("command"; "arg1")'
```
The operator takes:
- A command string (required)
- An argument (or an array of arguments), separated from the command by `;` (optional)
The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string.
## Disabling the system operator
The system operator is disabled by default. When disabled, an error is returned instead of running the command, consistent with `--security-disable-env-ops` and `--security-disable-file-ops`.
Use `--security-enable-system-operator` flag to enable it.
## system operator returns error when disabled
Use `--security-enable-system-operator` to enable the system operator.
Given a sample.yml file of:
```yaml
country: Australia
```
then
```bash
yq '.country = system("/usr/bin/echo"; "test")' sample.yml
```
will output
```bash
Error: system operations are disabled, use --security-enable-system-operator to enable
```
## Run a command with an argument
Use `--security-enable-system-operator` to enable the system operator.
Given a sample.yml file of:
```yaml
country: Australia
```
then
```bash
yq --security-enable-system-operator '.country = system("/usr/bin/echo"; "test")' sample.yml
```
will output
```yaml
country: test
```
## Run a command without arguments
Omit the semicolon and args to run the command with no extra arguments.
Given a sample.yml file of:
```yaml
a: hello
```
then
```bash
yq --security-enable-system-operator '.a = system("/usr/bin/echo")' sample.yml
```
will output
```yaml
a: ""
```
-17
View File
@@ -384,20 +384,3 @@ ip = "10.0.0.2"
role = "backend"
```
## Encode: Simple mapping produces table section
Given a sample.yml file of:
```yaml
arg:
hello: foo
```
then
```bash
yq -o toml '.' sample.yml
```
will output
```toml
[arg]
hello = "foo"
```
+4 -4
View File
@@ -449,8 +449,8 @@ func (he *hclEncoder) encodeBlockIfMapping(body *hclwrite.Body, key string, valu
return false
}
// If EncodeHintSeparateBlock is set, emit children as separate blocks regardless of label extraction
if valueNode.EncodeHint == EncodeHintSeparateBlock {
// If EncodeSeparate is set, emit children as separate blocks regardless of label extraction
if valueNode.EncodeSeparate {
if handled, _ := he.encodeMappingChildrenAsBlocks(body, key, valueNode); handled {
return true
}
@@ -537,9 +537,9 @@ func (he *hclEncoder) encodeMappingChildrenAsBlocks(body *hclwrite.Body, blockTy
return false, nil
}
// Only emit as separate blocks if EncodeHintSeparateBlock is set
// Only emit as separate blocks if EncodeSeparate is true
// This allows the encoder to respect the original block structure preserved by the decoder
if valueNode.EncodeHint != EncodeHintSeparateBlock {
if !valueNode.EncodeSeparate {
return false, nil
}
+1 -1
View File
@@ -32,7 +32,7 @@ func (ke *kyamlEncoder) PrintLeadingContent(writer io.Writer, content string) er
}
func (ke *kyamlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
log.Debugf("encoderKYaml - going to print %v", NodeToString(node))
log.Debug("encoderKYaml - going to print %v", NodeToString(node))
if node.Kind == ScalarNode && ke.prefs.UnwrapScalar {
return writeString(writer, node.Value+"\n")
}
+62 -57
View File
@@ -69,27 +69,6 @@ func (te *tomlEncoder) CanHandleAliases() bool {
// ---- helpers ----
// tomlKey returns the key quoted if it contains characters that are not valid
// in a TOML bare key. TOML bare keys may only contain ASCII letters, ASCII
// digits, underscores, and dashes.
func tomlKey(key string) string {
for _, r := range key {
if (r < 'A' || r > 'Z') && (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '_' && r != '-' {
return fmt.Sprintf("%q", key)
}
}
return key
}
// tomlDottedKey joins path components, quoting any that require it.
func tomlDottedKey(path []string) string {
parts := make([]string, len(path))
for i, p := range path {
parts[i] = tomlKey(p)
}
return strings.Join(parts, ".")
}
func (te *tomlEncoder) writeComment(w io.Writer, comment string) error {
if comment == "" {
return nil
@@ -169,10 +148,9 @@ func (te *tomlEncoder) encodeTopLevelEntry(w io.Writer, path []string, node *Can
}
if allMaps {
key := path[len(path)-1]
quotedKey := tomlKey(key)
for _, it := range node.Content {
// [[key]] then body
if _, err := w.Write([]byte("[[" + quotedKey + "]]\n")); err != nil {
if _, err := w.Write([]byte("[[" + key + "]]\n")); err != nil {
return err
}
if err := te.encodeMappingBodyWithPath(w, []string{key}, it); err != nil {
@@ -184,9 +162,12 @@ func (te *tomlEncoder) encodeTopLevelEntry(w io.Writer, path []string, node *Can
// Regular array attribute
return te.writeArrayAttribute(w, path[len(path)-1], node)
case MappingNode:
// Use inline table syntax for nodes explicitly marked as TOML inline tables
// or YAML flow mappings. All other mappings become readable TOML table sections.
if node.EncodeHint == EncodeHintInline || node.Style&FlowStyle != 0 {
// Inline table if not EncodeSeparate, else emit separate tables/arrays of tables for children under this path
if !node.EncodeSeparate {
// If children contain mappings or arrays of mappings, prefer separate sections
if te.hasEncodeSeparateChild(node) || te.hasStructuralChildren(node) {
return te.encodeSeparateMapping(w, path, node)
}
return te.writeInlineTableAttribute(w, path[len(path)-1], node)
}
return te.encodeSeparateMapping(w, path, node)
@@ -204,7 +185,7 @@ func (te *tomlEncoder) writeAttribute(w io.Writer, key string, value *CandidateN
}
// Write the attribute
line := tomlKey(key) + " = " + te.formatScalar(value)
line := key + " = " + te.formatScalar(value)
// Add line comment if present
if value.LineComment != "" {
@@ -229,7 +210,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida
// Handle empty arrays
if len(seq.Content) == 0 {
line := tomlKey(key) + " = []"
line := key + " = []"
if seq.LineComment != "" {
lineComment := strings.TrimSpace(seq.LineComment)
if !strings.HasPrefix(lineComment, "#") {
@@ -252,7 +233,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida
if hasElementComments {
// Write multiline array format with comments
if _, err := w.Write([]byte(tomlKey(key) + " = [\n")); err != nil {
if _, err := w.Write([]byte(key + " = [\n")); err != nil {
return err
}
@@ -343,7 +324,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida
}
}
line := tomlKey(key) + " = [" + strings.Join(items, ", ") + "]"
line := key + " = [" + strings.Join(items, ", ") + "]"
// Add line comment if present
if seq.LineComment != "" {
@@ -391,21 +372,21 @@ func (te *tomlEncoder) mappingToInlineTable(m *CandidateNode) (string, error) {
v := m.Content[i+1]
switch v.Kind {
case ScalarNode:
parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), te.formatScalar(v)))
parts = append(parts, fmt.Sprintf("%s = %s", k, te.formatScalar(v)))
case SequenceNode:
// inline array in inline table
arr, err := te.sequenceToInlineArray(v)
if err != nil {
return "", err
}
parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), arr))
parts = append(parts, fmt.Sprintf("%s = %s", k, arr))
case MappingNode:
// nested inline table
inline, err := te.mappingToInlineTable(v)
if err != nil {
return "", err
}
parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), inline))
parts = append(parts, fmt.Sprintf("%s = %s", k, inline))
default:
return "", fmt.Errorf("unsupported inline table value kind: %v", v.Kind)
}
@@ -418,7 +399,7 @@ func (te *tomlEncoder) writeInlineTableAttribute(w io.Writer, key string, m *Can
if err != nil {
return err
}
_, err = w.Write([]byte(tomlKey(key) + " = " + inline + "\n"))
_, err = w.Write([]byte(key + " = " + inline + "\n"))
return err
}
@@ -440,7 +421,7 @@ func (te *tomlEncoder) writeTableHeader(w io.Writer, path []string, m *Candidate
}
// Write table header [a.b.c]
header := "[" + tomlDottedKey(path) + "]\n"
header := "[" + strings.Join(path, ".") + "]\n"
_, err := w.Write([]byte(header))
return err
}
@@ -448,8 +429,7 @@ func (te *tomlEncoder) writeTableHeader(w io.Writer, path []string, m *Candidate
// encodeSeparateMapping handles a mapping that should be encoded as table sections.
// It emits the table header for this mapping if it has any content, then processes children.
func (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *CandidateNode) error {
// Check if this mapping has any non-mapping, non-array-of-tables children (i.e., attributes).
// Inline mapping children also count as attributes since they render as key = { ... }.
// Check if this mapping has any non-mapping, non-array-of-tables children (i.e., attributes)
hasAttrs := false
for i := 0; i < len(m.Content); i += 2 {
v := m.Content[i+1]
@@ -457,10 +437,6 @@ func (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *Cand
hasAttrs = true
break
}
if v.Kind == MappingNode && (v.EncodeHint == EncodeHintInline || v.Style&FlowStyle != 0) {
hasAttrs = true
break
}
if v.Kind == SequenceNode {
// Check if it's NOT an array of tables
allMaps := true
@@ -488,14 +464,18 @@ func (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *Cand
return nil
}
// No attributes, just nested table structures - process children recursively
// No attributes, just nested structures - process children
for i := 0; i < len(m.Content); i += 2 {
k := m.Content[i].Value
v := m.Content[i+1]
switch v.Kind {
case MappingNode:
// Emit [path.k]
newPath := append(append([]string{}, path...), k)
if err := te.encodeSeparateMapping(w, newPath, v); err != nil {
if err := te.writeTableHeader(w, newPath, v); err != nil {
return err
}
if err := te.encodeMappingBodyWithPath(w, newPath, v); err != nil {
return err
}
case SequenceNode:
@@ -508,7 +488,7 @@ func (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *Cand
}
}
if allMaps {
key := tomlDottedKey(append(append([]string{}, path...), k))
key := strings.Join(append(append([]string{}, path...), k), ".")
for _, it := range v.Content {
if _, err := w.Write([]byte("[[" + key + "]]\n")); err != nil {
return err
@@ -533,6 +513,39 @@ func (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *Cand
return nil
}
func (te *tomlEncoder) hasEncodeSeparateChild(m *CandidateNode) bool {
for i := 0; i < len(m.Content); i += 2 {
v := m.Content[i+1]
if v.Kind == MappingNode && v.EncodeSeparate {
return true
}
}
return false
}
func (te *tomlEncoder) hasStructuralChildren(m *CandidateNode) bool {
for i := 0; i < len(m.Content); i += 2 {
v := m.Content[i+1]
// Only consider it structural if mapping has EncodeSeparate or is non-empty
if v.Kind == MappingNode && v.EncodeSeparate {
return true
}
if v.Kind == SequenceNode {
allMaps := true
for _, it := range v.Content {
if it.Kind != MappingNode {
allMaps = false
break
}
}
if allMaps {
return true
}
}
}
return false
}
// encodeMappingBodyWithPath encodes attributes and nested arrays of tables using full dotted path context
func (te *tomlEncoder) encodeMappingBodyWithPath(w io.Writer, path []string, m *CandidateNode) error {
// First, attributes (scalars and non-map arrays)
@@ -573,7 +586,7 @@ func (te *tomlEncoder) encodeMappingBodyWithPath(w io.Writer, path []string, m *
}
}
if allMaps {
dotted := tomlDottedKey(append(append([]string{}, path...), k))
dotted := strings.Join(append(append([]string{}, path...), k), ".")
for _, it := range v.Content {
if _, err := w.Write([]byte("[[" + dotted + "]]\n")); err != nil {
return err
@@ -586,21 +599,13 @@ func (te *tomlEncoder) encodeMappingBodyWithPath(w io.Writer, path []string, m *
}
}
// Finally, child mappings: inline-hint or flow-style ones become inline table attributes,
// while all others are emitted as separate sub-table sections.
// Finally, child mappings that are not marked EncodeSeparate get inlined as attributes
for i := 0; i < len(m.Content); i += 2 {
k := m.Content[i].Value
v := m.Content[i+1]
if v.Kind == MappingNode {
if v.EncodeHint == EncodeHintInline || v.Style&FlowStyle != 0 {
if err := te.writeInlineTableAttribute(w, k, v); err != nil {
return err
}
} else {
subPath := append(append([]string{}, path...), k)
if err := te.encodeSeparateMapping(w, subPath, v); err != nil {
return err
}
if v.Kind == MappingNode && !v.EncodeSeparate {
if err := te.writeInlineTableAttribute(w, k, v); err != nil {
return err
}
}
}
+3 -3
View File
@@ -59,7 +59,7 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
return err
}
if _, err := e.writer.Write([]byte("\n")); err != nil {
log.Warningf("Unable to write newline, skipping: %v", err)
log.Warning("Unable to write newline, skipping: %w", err)
}
}
}
@@ -131,7 +131,7 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *CandidateNode
return err
}
if _, err := e.writer.Write([]byte("\n")); err != nil {
log.Warningf("Unable to write newline, skipping: %v", err)
log.Warning("Unable to write newline, skipping: %w", err)
}
} else if key.Value == e.prefs.DirectiveName {
var directive xml.Directive = []byte(value.Value)
@@ -139,7 +139,7 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *CandidateNode
return err
}
if _, err := e.writer.Write([]byte("\n")); err != nil {
log.Warningf("Unable to write newline, skipping: %v", err)
log.Warning("Unable to write newline, skipping: %w", err)
}
} else {
+1 -1
View File
@@ -29,7 +29,7 @@ func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) err
}
func (ye *yamlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
log.Debugf("encoderYaml - going to print %v", NodeToString(node))
log.Debug("encoderYaml - going to print %v", NodeToString(node))
// Detect line ending style from LeadingContent
lineEnding := "\n"
if strings.Contains(node.LeadingContent, "\r\n") {
+1 -1
View File
@@ -26,7 +26,7 @@ func newExpressionParser() ExpressionParserInterface {
}
func (p *expressionParserImpl) ParseExpression(expression string) (*ExpressionNode, error) {
log.Debugf("Parsing expression: [%v]", expression)
log.Debug("Parsing expression: [%v]", expression)
tokens, err := p.pathTokeniser.Tokenise(expression)
if err != nil {
return nil, err
-1
View File
@@ -95,7 +95,6 @@ func TestParserSingleOperation(t *testing.T) {
test.AssertResultComplex(t, nil, err)
if result == nil {
t.Fatal("Expected non-nil result for single operation")
return
}
if result.Operation == nil {
t.Fatal("Expected operation to be set")
+3 -2
View File
@@ -3,7 +3,8 @@ package yqlib
import (
"errors"
"fmt"
"log/slog"
logging "gopkg.in/op/go-logging.v1"
)
type expressionPostFixer interface {
@@ -133,7 +134,7 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
return nil, fmt.Errorf("bad expression - probably missing close bracket on %v", opStack[len(opStack)-1].toString(false))
}
if log.IsEnabledFor(slog.LevelDebug) {
if log.IsEnabledFor(logging.DEBUG) {
log.Debugf("PostFix Result:")
for _, currentToken := range result {
log.Debugf("> %v", currentToken.toString())
+3 -2
View File
@@ -30,7 +30,7 @@ func tryRenameFile(from string, to string) error {
}
func tryRemoveTempFile(filename string) {
log.Debugf("Removing temp file: %v", filename)
log.Debug("Removing temp file: %v", filename)
removeErr := os.Remove(filename)
if removeErr != nil {
log.Errorf("Failed to remove temp file: %v", filename)
@@ -68,7 +68,8 @@ func SafelyCloseReader(reader io.Reader) {
func safelyCloseFile(file *os.File) {
err := file.Close()
if err != nil {
log.Errorf("Error closing file %v: %v", file.Name(), err)
log.Error("Error closing file!")
log.Error(err.Error())
}
}
+1 -1
View File
@@ -58,7 +58,7 @@ func (f *frontMatterHandlerImpl) Split() error {
return err
}
f.yamlFrontMatterFilename = yamlTempFile.Name()
log.Debugf("yamlTempFile: %v", yamlTempFile.Name())
log.Debug("yamlTempFile: %v", yamlTempFile.Name())
lineCount := 0
-49
View File
@@ -108,55 +108,6 @@ yaml: doc
fmHandler.CleanUp()
}
func TestFrontMatterFilenamePreserved(t *testing.T) {
// Regression test for https://github.com/mikefarah/yq/issues/2538
// When using --front-matter, the filename operator should return
// the original filename, not the path to the temporary file.
file := createTestFile(`---
name: john
---
Some content
`)
originalFilename := "/path/to/original/file.md"
fmHandler := NewFrontMatterHandler(file)
err := fmHandler.Split()
if err != nil {
panic(err)
}
tempFilename := fmHandler.GetYamlFrontMatterFilename()
// Register the alias (as the command code does)
SetFilenameAlias(tempFilename, originalFilename)
defer ClearFilenameAliases()
// Verify resolveFilename returns the original name
resolved := resolveFilename(tempFilename)
test.AssertResult(t, originalFilename, resolved)
// Read documents using the temp file, verify they get the original filename
reader, err := readStream(tempFilename)
if err != nil {
panic(err)
}
decoder := NewYamlDecoder(ConfiguredYamlPreferences)
docs, err := readDocuments(reader, tempFilename, 0, decoder)
if err != nil {
panic(err)
}
if docs.Len() == 0 {
t.Fatal("expected at least one document")
}
firstDoc := docs.Front().Value.(*CandidateNode)
test.AssertResult(t, originalFilename, firstDoc.filename)
tryRemoveTempFile(file)
fmHandler.CleanUp()
}
func TestFrontMatterSplitWithArray(t *testing.T) {
file := createTestFile(`[1,2,3]
---
+1 -6
View File
@@ -105,7 +105,7 @@ func handleToken(tokens []*token, index int, postProcessedTokens []*token) (toke
skipNextToken = false
currentToken := tokens[index]
log.Debugf("processing %v", currentToken.toString(true))
log.Debug("processing %v", currentToken.toString(true))
if currentToken.TokenType == traverseArrayCollect {
// `.[exp]`` works by creating a traversal array of [self, exp] and piping that into the traverse array operator
@@ -131,11 +131,6 @@ func handleToken(tokens []*token, index int, postProcessedTokens []*token) (toke
log.Debugf("previous token is : traverseArrayOpType")
// need to put the number 0 before this token, as that is implied
postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: createValueOperation(0, "0")})
} else if index >= 2 && tokens[index-1].TokenType == openCollect &&
(tokens[index-2].TokenType == operationToken || tokens[index-2].TokenType == closeCollect || tokens[index-2].TokenType == closeCollectObject) {
log.Debugf("previous token is : openCollect following a traversal, implying 0 start")
// need to put the number 0 before this token, as that is implied
postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: createValueOperation(0, "0")})
}
}
+5 -7
View File
@@ -96,8 +96,6 @@ var participleYqRules = []*participleYqRule{
simpleOp("load_?str|str_?load", loadStringOpType),
{"LoadYaml", `load`, loadOp(NewYamlDecoder(LoadYamlPreferences)), 0},
simpleOp("system", systemOpType),
{"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0},
simpleOp("select", selectOpType),
@@ -285,7 +283,7 @@ func pathToken(wrapped bool) yqAction {
if wrapped {
value = unwrap(value)
}
log.Debugf("PathToken %v", value)
log.Debug("PathToken %v", value)
op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: prefs}
return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
}
@@ -338,7 +336,7 @@ func flattenWithDepth() yqAction {
func assignAllCommentsOp(updateAssign bool) yqAction {
return func(rawToken lexer.Token) (*token, error) {
log.Debugf("assignAllCommentsOp %v", rawToken.Value)
log.Debug("assignAllCommentsOp %v", rawToken.Value)
value := rawToken.Value
op := &Operation{
OperationType: assignCommentOpType,
@@ -353,7 +351,7 @@ func assignAllCommentsOp(updateAssign bool) yqAction {
func assignOpToken(updateAssign bool) yqAction {
return func(rawToken lexer.Token) (*token, error) {
log.Debugf("assignOpToken %v", rawToken.Value)
log.Debug("assignOpToken %v", rawToken.Value)
value := rawToken.Value
prefs := assignPreferences{DontOverWriteAnchor: true}
if strings.Contains(value, "c") {
@@ -378,9 +376,9 @@ func nullValue() yqAction {
func stringValue() yqAction {
return func(rawToken lexer.Token) (*token, error) {
log.Debugf("rawTokenvalue: %v", rawToken.Value)
log.Debug("rawTokenvalue: %v", rawToken.Value)
value := unwrap(rawToken.Value)
log.Debugf("unwrapped: %v", value)
log.Debug("unwrapped: %v", value)
value = processEscapeCharacters(value)
return &token{TokenType: operationToken, Operation: &Operation{
OperationType: stringInterpolationOpType,
+10 -9
View File
@@ -4,10 +4,11 @@ package yqlib
import (
"container/list"
"fmt"
"log/slog"
"math"
"strconv"
"strings"
logging "gopkg.in/op/go-logging.v1"
)
var ExpressionParser ExpressionParserInterface
@@ -18,17 +19,17 @@ func InitExpressionParser() {
}
}
var log = newLogger()
var log = logging.MustGetLogger("yq-lib")
var PrettyPrintExp = `(... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not)) ) style=""`
// GetLogger returns the yq logger instance.
func GetLogger() *Logger {
func GetLogger() *logging.Logger {
return log
}
func getContentValueByKey(content []*CandidateNode, key string) *CandidateNode {
for index := 0; index < len(content)-1; index = index + 2 {
for index := 0; index < len(content); index = index + 2 {
keyNode := content[index]
valueNode := content[index+1]
if keyNode.Value == key {
@@ -80,7 +81,7 @@ func recurseNodeObjectEqual(lhs *CandidateNode, rhs *CandidateNode) bool {
key := lhs.Content[index]
value := lhs.Content[index+1]
indexInRHS := findKeyInMap(rhs, key)
indexInRHS := findInArray(rhs, key)
if indexInRHS == -1 || !recursiveNodeEqual(value, rhs.Content[indexInRHS+1]) {
return false
@@ -250,7 +251,7 @@ func processEscapeCharacters(original string) string {
value := result.String()
if value != original {
log.Debugf("processEscapeCharacters from [%v] to [%v]", original, value)
log.Debug("processEscapeCharacters from [%v] to [%v]", original, value)
}
return value
}
@@ -273,7 +274,7 @@ func footComment(node *CandidateNode) string {
// use for debugging only
func NodesToString(collection *list.List) string {
if !log.IsEnabledFor(slog.LevelDebug) {
if !log.IsEnabledFor(logging.DEBUG) {
return ""
}
@@ -285,7 +286,7 @@ func NodesToString(collection *list.List) string {
}
func NodeToString(node *CandidateNode) string {
if !log.IsEnabledFor(slog.LevelDebug) {
if !log.IsEnabledFor(logging.DEBUG) {
return ""
}
if node == nil {
@@ -303,7 +304,7 @@ func NodeToString(node *CandidateNode) string {
}
func NodeContentToString(node *CandidateNode, depth int) string {
if !log.IsEnabledFor(slog.LevelDebug) {
if !log.IsEnabledFor(logging.DEBUG) {
return ""
}
var sb strings.Builder
+1 -19
View File
@@ -24,7 +24,7 @@ type parseSnippetScenario struct {
var parseSnippetScenarios = []parseSnippetScenario{
{
snippet: ":",
expectedError: "yaml: while parsing a block mapping at <unknown position>: did not find expected key",
expectedError: "yaml: did not find expected key",
},
{
snippet: "",
@@ -300,24 +300,6 @@ func TestRecurseNodeObjectEqual(t *testing.T) {
test.AssertResult(t, true, recurseNodeObjectEqual(obj1, obj2))
test.AssertResult(t, false, recurseNodeObjectEqual(obj1, obj3))
test.AssertResult(t, false, recurseNodeObjectEqual(obj1, obj4))
// A null key must not match a null value in the other map.
// Regression test for https://issues.oss-fuzz.com/issues/383860504
nullKey := &CandidateNode{Kind: ScalarNode, Tag: "!!null"}
nullVal := &CandidateNode{Kind: ScalarNode, Tag: "!!null"}
intKey := createScalarNode(2, "2")
intKey.Tag = "!!int"
intVal := &CandidateNode{Kind: ScalarNode, Tag: "!!null"}
mapWithNullKey := &CandidateNode{
Kind: MappingNode,
Content: []*CandidateNode{nullKey, nullVal},
}
mapWithIntKey := &CandidateNode{
Kind: MappingNode,
Content: []*CandidateNode{intKey, intVal},
}
test.AssertResult(t, false, recurseNodeObjectEqual(mapWithNullKey, mapWithIntKey))
}
func TestParseInt(t *testing.T) {
-77
View File
@@ -1,77 +0,0 @@
package yqlib
import (
"fmt"
"log/slog"
"os"
)
// Logger wraps log/slog providing a printf-style interface used throughout yq.
type Logger struct {
levelVar slog.LevelVar
slogger *slog.Logger
}
func newLogger() *Logger {
l := &Logger{}
l.levelVar.Set(slog.LevelWarn)
handler := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: &l.levelVar})
l.slogger = slog.New(handler)
return l
}
// SetLevel sets the minimum log level.
func (l *Logger) SetLevel(level slog.Level) {
l.levelVar.Set(level)
}
// GetLevel returns the current log level.
func (l *Logger) GetLevel() slog.Level {
return l.levelVar.Level()
}
// IsEnabledFor returns true if the given level is enabled.
func (l *Logger) IsEnabledFor(level slog.Level) bool {
return l.levelVar.Level() <= level
}
// SetSlogger replaces the underlying slog.Logger (e.g. to configure output format).
func (l *Logger) SetSlogger(logger *slog.Logger) {
l.slogger = logger
}
func (l *Logger) Debug(msg string) {
if l.IsEnabledFor(slog.LevelDebug) {
l.slogger.Debug(msg)
}
}
func (l *Logger) Debugf(format string, args ...interface{}) {
if l.IsEnabledFor(slog.LevelDebug) {
l.slogger.Debug(fmt.Sprintf(format, args...))
}
}
func (l *Logger) Info(msg string) {
l.slogger.Info(msg)
}
func (l *Logger) Infof(format string, args ...interface{}) {
l.slogger.Info(fmt.Sprintf(format, args...))
}
func (l *Logger) Warning(msg string) {
l.slogger.Warn(msg)
}
func (l *Logger) Warningf(format string, args ...interface{}) {
l.slogger.Warn(fmt.Sprintf(format, args...))
}
func (l *Logger) Error(msg string) {
l.slogger.Error(msg)
}
func (l *Logger) Errorf(format string, args ...interface{}) {
l.slogger.Error(fmt.Sprintf(format, args...))
}
+1 -3
View File
@@ -25,7 +25,7 @@ var valueToStringFunc = func(p *Operation) string {
}
func createValueOperation(value interface{}, stringValue string) *Operation {
log.Debugf("creating value op for string %v", stringValue)
log.Debug("creating value op for string %v", stringValue)
var node = createScalarNode(value, stringValue)
return &Operation{
@@ -164,8 +164,6 @@ var stringInterpolationOpType = &operationType{Type: "STRING_INT", NumArgs: 0, P
var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 52, Handler: loadOperator, CheckForPostTraverse: true}
var loadStringOpType = &operationType{Type: "LOAD_STRING", NumArgs: 1, Precedence: 52, Handler: loadStringOperator}
var systemOpType = &operationType{Type: "SYSTEM", NumArgs: 1, Precedence: 50, Handler: systemOperator}
var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 52, Handler: keysOperator, CheckForPostTraverse: true}
var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator}
+2 -2
View File
@@ -195,9 +195,9 @@ func addMaps(target *CandidateNode, lhsC *CandidateNode, rhsC *CandidateNode) {
for index := 0; index < len(rhs.Content); index = index + 2 {
key := rhs.Content[index]
value := rhs.Content[index+1]
log.Debugf("finding %v", key.Value)
log.Debug("finding %v", key.Value)
indexInLHS := findKeyInMap(target, key)
log.Debugf("indexInLhs %v", indexInLHS)
log.Debug("indexInLhs %v", indexInLHS)
if indexInLHS < 0 {
// not in there, append it
target.AddKeyValueChild(key, value)
-12
View File
@@ -527,18 +527,6 @@ var addOperatorScenarios = []expressionScenario{
expression: `.a += [2]`,
expectedError: "!!seq () cannot be added to a !!str (a)",
},
{
// Regression test for https://issues.oss-fuzz.com/issues/383860504
// Adding a map to itself must not panic when sequence keys contain
// single-entry mappings with a null key in one and a non-null key
// in the other.
skipDoc: true,
document: "? [{~: ~}]\n: v1\n? [{2: ~}]\n: v2",
expression: `. += .`,
expected: []string{
"D0, P[], (!!map)::? [{~: ~}]\n: v1\n? [{2: ~}]\n: v2\n",
},
},
}
func TestAddOperatorScenarios(t *testing.T) {
+2 -2
View File
@@ -256,7 +256,7 @@ func explodeNode(node *CandidateNode, context Context) error {
node.Value = node.Alias.Value
node.Alias = nil
}
log.Debugf("now I'm %v", NodeToString(node))
log.Debug("now I'm %v", NodeToString(node))
return nil
case MappingNode:
// //check the map has an alias in it
@@ -304,7 +304,7 @@ func applyAlias(node *CandidateNode, alias *CandidateNode, aliasIndex int, newCo
if alias == nil {
return nil
}
log.Debugf("alias: %v", NodeToString(alias))
log.Debug("alias: %v", NodeToString(alias))
if alias.Kind != MappingNode {
return fmt.Errorf("can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing %v", alias.Tag)
}
+1 -1
View File
@@ -37,7 +37,7 @@ func assignUpdateOperator(d *dataTreeNavigator, context Context, expressionNode
prefs := getAssignPreferences(expressionNode.Operation.Preferences)
log.Debugf("assignUpdateOperator prefs: %v", prefs)
log.Debug("assignUpdateOperator prefs: %v", prefs)
if !expressionNode.Operation.UpdateAssign {
// this works because we already ran against LHS with an editable context.
+1 -1
View File
@@ -142,7 +142,7 @@ func notOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Cont
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
log.Debugf("notOperation checking %v", candidate)
log.Debug("notOperation checking %v", candidate)
truthy := isTruthyNode(candidate)
result := createBooleanCandidate(candidate, !truthy)
results.PushBack(result)
+2 -2
View File
@@ -46,9 +46,9 @@ func containsObject(lhs *CandidateNode, rhs *CandidateNode) (bool, error) {
rhsKey := rhs.Content[index]
rhsValue := rhs.Content[index+1]
log.Debugf("Looking for %v in the lhs", rhsKey.Value)
lhsKeyIndex := findKeyInMap(lhs, rhsKey)
lhsKeyIndex := findInArray(lhs, rhsKey)
log.Debugf("index is %v", lhsKeyIndex)
if lhsKeyIndex < 0 {
if lhsKeyIndex < 0 || lhsKeyIndex%2 != 0 {
return false, nil
}
lhsValue := lhs.Content[lhsKeyIndex+1]
-10
View File
@@ -65,16 +65,6 @@ var containsOperatorScenarios = []expressionScenario{
"D0, P[], (!!bool)::false\n",
},
},
{
// Regression: findInArray could match a null key against a null
// value at an earlier odd index, producing a false negative.
skipDoc: true,
document: "? 1\n: ~\n? ~\n: x",
expression: `contains({~: "x"})`,
expected: []string{
"D0, P[], (!!bool)::true\n",
},
},
{
description: "String contains substring",
document: `"foobar"`,
+1 -1
View File
@@ -80,7 +80,7 @@ func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *Expre
node, errorReading := parseSnippet(formattedTimeStr)
if errorReading != nil {
log.Debugf("could not parse %v - lets just leave it as a string: %v", formattedTimeStr, errorReading)
log.Debugf("could not parse %v - lets just leave it as a string: %w", formattedTimeStr, errorReading)
node = &CandidateNode{
Kind: ScalarNode,
Tag: "!!str",
+1 -1
View File
@@ -45,7 +45,7 @@ func removeFromContext(context Context, candidate *CandidateNode) (Context, erro
if nodeInContext != candidate {
newResults.PushBack(nodeInContext)
} else {
log.Infof("Need to delete this %v", NodeToString(nodeInContext))
log.Info("Need to delete this %v", NodeToString(nodeInContext))
}
}
return context.ChildContext(newResults), nil
+1 -1
View File
@@ -33,7 +33,7 @@ func configureEncoder(format *Format, indent int) Encoder {
func encodeToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) {
var output bytes.Buffer
log.Debugf("printing with indent: %v", prefs.indent)
log.Debug("printing with indent: %v", prefs.indent)
encoder := configureEncoder(prefs.format, prefs.indent)
if encoder == nil {
+3 -3
View File
@@ -157,10 +157,10 @@ func withEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *
if err != nil {
return Context{}, err
}
log.Debugf("candidate %v", NodeToString(candidate))
log.Debugf("candidate leading content: %v", candidate.LeadingContent)
log.Debug("candidate %v", NodeToString(candidate))
log.Debug("candidate leading content: %v", candidate.LeadingContent)
collected.LeadingContent = candidate.LeadingContent
log.Debugf("candidate FootComment: [%v]", candidate.FootComment)
log.Debug("candidate FootComment: [%v]", candidate.FootComment)
collected.HeadComment = candidate.HeadComment
collected.FootComment = candidate.FootComment
+5 -5
View File
@@ -21,7 +21,7 @@ func envOperator(_ *dataTreeNavigator, context Context, expressionNode *Expressi
return Context{}, fmt.Errorf("env operations have been disabled")
}
envName := expressionNode.Operation.CandidateNode.Value
log.Debugf("EnvOperator, env name: %v", envName)
log.Debug("EnvOperator, env name:", envName)
rawValue := os.Getenv(envName)
@@ -49,9 +49,9 @@ func envOperator(_ *dataTreeNavigator, context Context, expressionNode *Expressi
}
}
log.Debugf("ENV tag: %v", node.Tag)
log.Debugf("ENV value: %v", node.Value)
log.Debugf("ENV Kind: %v", node.Kind)
log.Debug("ENV tag", node.Tag)
log.Debug("ENV value", node.Value)
log.Debug("ENV Kind", node.Kind)
return context.SingleChildContext(node), nil
}
@@ -78,7 +78,7 @@ func envsubstOperator(_ *dataTreeNavigator, context Context, expressionNode *Exp
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
node := el.Value.(*CandidateNode)
if node.Tag != "!!str" {
log.Warningf("EnvSubstOperator, env name: %v %v", node.Tag, node.Value)
log.Warning("EnvSubstOperator, env name:", node.Tag, node.Value)
return Context{}, fmt.Errorf("cannot substitute with %v, can only substitute strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
}
+2 -2
View File
@@ -47,8 +47,8 @@ func mapOperator(d *dataTreeNavigator, context Context, expressionNode *Expressi
}
result, err := d.GetMatchingNodes(splatted, expressionNode.RHS)
log.Debugf("expressionNode.Rhs %v", expressionNode.RHS.Operation.OperationType)
log.Debugf("result %v", result)
log.Debug("expressionNode.Rhs %v", expressionNode.RHS.Operation.OperationType)
log.Debug("result %v", result)
if err != nil {
return Context{}, err
}
+2 -4
View File
@@ -155,10 +155,8 @@ func repeatString(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error
return nil, err
} else if count < 0 {
return nil, fmt.Errorf("cannot repeat string by a negative number (%v)", count)
}
maxResultLen := 10 * 1024 * 1024 // 10 MiB
if count > 0 && len(stringNode.Value) > maxResultLen/count {
return nil, fmt.Errorf("result of repeating string (%v bytes) by %v would exceed %v bytes", len(stringNode.Value), count, maxResultLen)
} else if count > 10000000 {
return nil, fmt.Errorf("cannot repeat string by more than 100 million (%v)", count)
}
target.Value = strings.Repeat(stringNode.Value, count)
+4 -24
View File
@@ -237,11 +237,12 @@ var multiplyOperatorScenarios = []expressionScenario{
expectedError: "cannot repeat string by a negative number (-4)",
},
{
description: "Multiply string by count that exceeds result size limit",
description: "Multiply string X by more than 100 million",
// very large string.repeats causes a panic
skipDoc: true,
document: `n: 100000001`,
expression: `"banana" * .n`,
expectedError: "result of repeating string (6 bytes) by 100000001 would exceed 10485760 bytes",
expectedError: "cannot repeat string by more than 100 million (100000001)",
},
{
description: "Multiply int node X string",
@@ -553,7 +554,7 @@ var multiplyOperatorScenarios = []expressionScenario{
document: document,
expression: `.b * .c`,
expected: []string{
"D0, P[b], (!!map)::{name: dog, \"<<\": *cat}\n",
"D0, P[b], (!!map)::{name: dog, <<: *cat}\n",
},
},
{
@@ -692,27 +693,6 @@ var multiplyOperatorScenarios = []expressionScenario{
"D0, P[], (!!null)::null\n",
},
},
{
// Regression test for https://issues.oss-fuzz.com/issues/418818862
// Large repeat count with a long string must not panic.
skipDoc: true,
expression: `"abc" * 99999999`,
expectedError: "result of repeating string (3 bytes) by 99999999 would exceed 10485760 bytes",
},
{
// Regression test for https://issues.oss-fuzz.com/issues/383195001
// Product of string length * repeat count must be bounded.
skipDoc: true,
expression: `"x" * 99999999`,
expectedError: "result of repeating string (1 bytes) by 99999999 would exceed 10485760 bytes",
},
{
// The size guard must not overflow: len * count can wrap to
// a negative or small value on 64-bit, bypassing the check.
skipDoc: true,
expression: `"ab" * 4611686018427387904`,
expectedError: "result of repeating string (2 bytes) by 4611686018427387904 would exceed 10485760 bytes",
},
}
func TestMultiplyOperatorScenarios(t *testing.T) {
+13 -42
View File
@@ -16,44 +16,11 @@ func getSliceNumber(d *dataTreeNavigator, context Context, node *CandidateNode,
return parseInt(result.MatchingNodes.Front().Value.(*CandidateNode).Value)
}
// clampSliceIndex resolves a possibly-negative slice index against
// length and clamps the result to [0, length].
func clampSliceIndex(index, length int) int {
if index < 0 {
index += length
}
if index < 0 {
return 0
}
if index > length {
return length
}
return index
}
func sliceStringNode(lhsNode *CandidateNode, firstNumber int, secondNumber int) *CandidateNode {
runes := []rune(lhsNode.Value)
length := len(runes)
relativeFirstNumber := clampSliceIndex(firstNumber, length)
relativeSecondNumber := clampSliceIndex(secondNumber, length)
if relativeSecondNumber < relativeFirstNumber {
relativeSecondNumber = relativeFirstNumber
}
log.Debugf("sliceStringNode: slice from %v to %v", relativeFirstNumber, relativeSecondNumber)
slicedString := string(runes[relativeFirstNumber:relativeSecondNumber])
replacement := lhsNode.CreateReplacement(ScalarNode, lhsNode.Tag, slicedString)
replacement.Style = lhsNode.Style
return replacement
}
func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debug("slice array operator!")
log.Debugf("lhs: %v", expressionNode.LHS.Operation.toString())
log.Debugf("rhs: %v", expressionNode.RHS.Operation.toString())
log.Debug("lhs: %v", expressionNode.LHS.Operation.toString())
log.Debug("rhs: %v", expressionNode.RHS.Operation.toString())
results := list.New()
@@ -61,24 +28,28 @@ func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *E
lhsNode := el.Value.(*CandidateNode)
firstNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.LHS)
if err != nil {
return Context{}, err
}
relativeFirstNumber := firstNumber
if relativeFirstNumber < 0 {
relativeFirstNumber = len(lhsNode.Content) + firstNumber
}
secondNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.RHS)
if err != nil {
return Context{}, err
}
if lhsNode.Kind == ScalarNode && lhsNode.guessTagFromCustomType() == "!!str" {
results.PushBack(sliceStringNode(lhsNode, firstNumber, secondNumber))
continue
relativeSecondNumber := secondNumber
if relativeSecondNumber < 0 {
relativeSecondNumber = len(lhsNode.Content) + secondNumber
} else if relativeSecondNumber > len(lhsNode.Content) {
relativeSecondNumber = len(lhsNode.Content)
}
relativeFirstNumber := clampSliceIndex(firstNumber, len(lhsNode.Content))
relativeSecondNumber := clampSliceIndex(secondNumber, len(lhsNode.Content))
log.Debugf("calculateIndicesToTraverse: slice from %v to %v", relativeFirstNumber, relativeSecondNumber)
log.Debug("calculateIndicesToTraverse: slice from %v to %v", relativeFirstNumber, relativeSecondNumber)
var newResults []*CandidateNode
for i := relativeFirstNumber; i < relativeSecondNumber; i++ {
-109
View File
@@ -98,115 +98,6 @@ var sliceArrayScenarios = []expressionScenario{
"D0, P[], (!!seq)::- cat1\n",
},
},
{
// Regression test for https://issues.oss-fuzz.com/issues/438776028
// Negative second index that underflows after adjustment must
// clamp to zero, yielding an empty sequence.
skipDoc: true,
document: `[a, b, c]`,
expression: `.[0:-99999]`,
expected: []string{
"D0, P[], (!!seq)::[]\n",
},
},
{
// First-index underflow: without clamping, the loop starts at a
// negative index and panics on Content access.
skipDoc: true,
document: `[a, b, c]`,
expression: `.[-99999:3]`,
expected: []string{
"D0, P[], (!!seq)::- a\n- b\n- c\n",
},
},
{
// Both indices underflow: both clamp to zero, yielding an empty
// sequence.
skipDoc: true,
document: `[a, b, c]`,
expression: `.[-99999:-99998]`,
expected: []string{
"D0, P[], (!!seq)::[]\n",
},
},
{
description: "Slicing strings",
document: `country: Australia`,
expression: `.country[0:5]`,
expected: []string{
"D0, P[country], (!!str)::Austr\n",
},
},
{
description: "Slicing strings - without the second number",
subdescription: "Finishes at the end of the string",
document: `country: Australia`,
expression: `.country[5:]`,
expected: []string{
"D0, P[country], (!!str)::alia\n",
},
},
{
description: "Slicing strings - without the first number",
subdescription: "Starts from the start of the string",
document: `country: Australia`,
expression: `.country[:5]`,
expected: []string{
"D0, P[country], (!!str)::Austr\n",
},
},
{
description: "Slicing strings - use negative numbers to count backwards from the end",
subdescription: "Negative indices count from the end of the string",
document: `country: Australia`,
expression: `.country[-5:]`,
expected: []string{
"D0, P[country], (!!str)::ralia\n",
},
},
{
skipDoc: true,
document: `country: Australia`,
expression: `.country[1:-1]`,
expected: []string{
"D0, P[country], (!!str)::ustrali\n",
},
},
{
skipDoc: true,
document: `country: Australia`,
expression: `.country[:]`,
expected: []string{
"D0, P[country], (!!str)::Australia\n",
},
},
{
skipDoc: true,
description: "second index beyond string length clamps",
document: `country: Australia`,
expression: `.country[:100]`,
expected: []string{
"D0, P[country], (!!str)::Australia\n",
},
},
{
skipDoc: true,
description: "first index beyond string length returns empty string",
document: `country: Australia`,
expression: `.country[100:]`,
expected: []string{
"D0, P[country], (!!str)::\n",
},
},
{
description: "Slicing strings - Unicode",
subdescription: "Indices are rune-based, so multi-byte characters are handled correctly",
document: `greeting: héllo`,
expression: `.greeting[1:3]`,
expected: []string{
"D0, P[greeting], (!!str)::él\n",
},
},
}
func TestSliceOperatorScenarios(t *testing.T) {
+2 -2
View File
@@ -142,12 +142,12 @@ func (a sortableNodeArray) compare(lhs *CandidateNode, rhs *CandidateNode, dateT
} else if isDateTime {
lhsTime, err := parseDateTime(layout, lhs.Value)
if err != nil {
log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %v", lhs.Value, layout, err)
log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %w", lhs.Value, layout, err)
return strings.Compare(lhs.Value, rhs.Value)
}
rhsTime, err := parseDateTime(layout, rhs.Value)
if err != nil {
log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %v", rhs.Value, layout, err)
log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %w", rhs.Value, layout, err)
return strings.Compare(lhs.Value, rhs.Value)
}
if lhsTime.Equal(rhsTime) {
+4 -4
View File
@@ -220,7 +220,7 @@ func getSubstituteParameters(d *dataTreeNavigator, block *ExpressionNode, contex
regEx = regExNodes.MatchingNodes.Front().Value.(*CandidateNode).Value
}
log.Debugf("regEx %v", regEx)
log.Debug("regEx %v", regEx)
replacementNodes, err := d.GetMatchingNodes(context, block.RHS)
if err != nil {
@@ -318,7 +318,7 @@ func getMatches(matchPrefs matchPreferences, regEx *regexp.Regexp, value string)
allIndices = [][]int{regEx.FindStringSubmatchIndex(value)}
}
log.Debugf("allMatches, %v", allMatches)
log.Debug("allMatches, %v", allMatches)
return allMatches, allIndices
}
@@ -423,7 +423,7 @@ func extractMatchArguments(d *dataTreeNavigator, context Context, expressionNode
if regExNodes.MatchingNodes.Front() != nil {
regExStr = regExNodes.MatchingNodes.Front().Value.(*CandidateNode).Value
}
log.Debugf("regEx %v", regExStr)
log.Debug("regEx %v", regExStr)
regEx, err := regexp.Compile(regExStr)
return regEx, matchPrefs, err
}
@@ -564,7 +564,7 @@ func split(value string, spltStr string) (Kind, string, []*CandidateNode) {
var contents []*CandidateNode
if value != "" {
log.Debugf("going to spltStr[%v]", spltStr)
log.Debug("going to spltStr[%v]", spltStr)
var newStrings = strings.Split(value, spltStr)
contents = make([]*CandidateNode, len(newStrings))
+1 -1
View File
@@ -26,7 +26,7 @@ func parseStyle(customStyle string) (Style, error) {
func assignStyleOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("AssignStyleOperator")
log.Debugf("AssignStyleOperator: %v")
var style Style
if !expressionNode.Operation.UpdateAssign {
rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS)
-146
View File
@@ -1,146 +0,0 @@
package yqlib
import (
"bytes"
"container/list"
"fmt"
"os/exec"
"strings"
)
func resolveSystemArgs(argsNode *CandidateNode) ([]string, error) {
if argsNode == nil {
return nil, nil
}
if argsNode.Kind == SequenceNode {
args := make([]string, 0, len(argsNode.Content))
for _, child := range argsNode.Content {
// Only non-null scalar children are valid arguments.
if child == nil {
continue
}
if child.Kind != ScalarNode || child.Tag == "!!null" {
return nil, fmt.Errorf("system operator: argument must be a non-null scalar; got kind=%v tag=%v", child.Kind, child.Tag)
}
args = append(args, child.Value)
}
if len(args) == 0 {
return nil, nil
}
return args, nil
}
// Single-argument case: only accept a non-null scalar node.
if argsNode.Tag == "!!null" {
return nil, nil
}
if argsNode.Kind != ScalarNode {
return nil, fmt.Errorf("system operator: args must be a non-null scalar or sequence of non-null scalars; got kind=%v tag=%v", argsNode.Kind, argsNode.Tag)
}
return []string{argsNode.Value}, nil
}
func resolveCommandNode(commandNodes Context) (string, error) {
if commandNodes.MatchingNodes.Front() == nil {
return "", fmt.Errorf("system operator: command expression returned no results")
}
if commandNodes.MatchingNodes.Len() > 1 {
log.Debugf("system operator: command expression returned %d results, using first", commandNodes.MatchingNodes.Len())
}
cmdNode := commandNodes.MatchingNodes.Front().Value.(*CandidateNode)
if cmdNode.Kind != ScalarNode || cmdNode.guessTagFromCustomType() != "!!str" {
return "", fmt.Errorf("system operator: command must be a string scalar")
}
if cmdNode.Value == "" {
return "", fmt.Errorf("system operator: command must be a non-empty string")
}
return cmdNode.Value, nil
}
func systemOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
if !ConfiguredSecurityPreferences.EnableSystemOps {
return Context{}, fmt.Errorf("system operations are disabled, use --security-enable-system-operator to enable")
}
// determine at parse time whether we have (command; args) or just (command)
hasArgs := expressionNode.RHS.Operation.OperationType == blockOpType
var results = list.New()
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
nodeContext := context.SingleReadonlyChildContext(candidate)
var command string
var args []string
if hasArgs {
block := expressionNode.RHS
commandNodes, err := d.GetMatchingNodes(nodeContext, block.LHS)
if err != nil {
return Context{}, err
}
command, err = resolveCommandNode(commandNodes)
if err != nil {
return Context{}, err
}
argsNodes, err := d.GetMatchingNodes(nodeContext, block.RHS)
if err != nil {
return Context{}, err
}
if argsNodes.MatchingNodes.Len() > 1 {
log.Debugf("system operator: args expression returned %d results, using first", argsNodes.MatchingNodes.Len())
}
if argsNodes.MatchingNodes.Front() != nil {
args, err = resolveSystemArgs(argsNodes.MatchingNodes.Front().Value.(*CandidateNode))
if err != nil {
return Context{}, err
}
}
} else {
commandNodes, err := d.GetMatchingNodes(nodeContext, expressionNode.RHS)
if err != nil {
return Context{}, err
}
command, err = resolveCommandNode(commandNodes)
if err != nil {
return Context{}, err
}
}
var stdin bytes.Buffer
encoded, err := encodeToYamlString(candidate)
if err != nil {
return Context{}, err
}
stdin.WriteString(encoded)
// #nosec G204 - intentional: user must explicitly enable this operator
cmd := exec.Command(command, args...)
cmd.Stdin = &stdin
var stderr bytes.Buffer
cmd.Stderr = &stderr
output, err := cmd.Output()
if err != nil {
stderrStr := strings.TrimSpace(stderr.String())
if stderrStr != "" {
return Context{}, fmt.Errorf("system command '%v' failed: %w\nstderr: %v", command, err, stderrStr)
}
return Context{}, fmt.Errorf("system command '%v' failed: %w", command, err)
}
result := string(output)
if strings.HasSuffix(result, "\r\n") {
result = result[:len(result)-2]
} else if strings.HasSuffix(result, "\n") {
result = result[:len(result)-1]
}
newNode := candidate.CreateReplacement(ScalarNode, "!!str", result)
results.PushBack(newNode)
}
return context.ChildContext(results), nil
}
-123
View File
@@ -1,123 +0,0 @@
package yqlib
import (
"os/exec"
"testing"
)
func findExec(t *testing.T, name string) string {
t.Helper()
path, err := exec.LookPath(name)
if err != nil {
t.Skipf("skipping: %v not found: %v", name, err)
}
return path
}
var systemOperatorDisabledScenarios = []expressionScenario{
{
description: "system operator returns error when disabled",
subdescription: "Use `--security-enable-system-operator` to enable the system operator.",
document: "country: Australia",
expression: `.country = system("/usr/bin/echo"; "test")`,
expectedError: "system operations are disabled, use --security-enable-system-operator to enable",
},
}
func TestSystemOperatorDisabledScenarios(t *testing.T) {
originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps
defer func() {
ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps
}()
ConfiguredSecurityPreferences.EnableSystemOps = false
for _, tt := range systemOperatorDisabledScenarios {
testScenario(t, &tt)
}
documentOperatorScenarios(t, "system-operators", systemOperatorDisabledScenarios)
}
func TestSystemOperatorEnabledScenarios(t *testing.T) {
echoPath := findExec(t, "echo")
falsePath := findExec(t, "false")
originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps
defer func() {
ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps
}()
ConfiguredSecurityPreferences.EnableSystemOps = true
scenarios := []expressionScenario{
{
description: "Run a command with an argument",
subdescription: "Use `--security-enable-system-operator` to enable the system operator.",
yqFlags: "--security-enable-system-operator",
document: "country: Australia",
expression: `.country = system("` + echoPath + `"; "test")`,
expected: []string{
"D0, P[], (!!map)::country: test\n",
},
},
{
description: "Run a command without arguments",
subdescription: "Omit the semicolon and args to run the command with no extra arguments.",
yqFlags: "--security-enable-system-operator",
document: "a: hello",
expression: `.a = system("` + echoPath + `")`,
expected: []string{
"D0, P[], (!!map)::a: \"\"\n",
},
},
{
description: "Run a command with multiple arguments",
subdescription: "Pass an array of arguments.",
skipDoc: true,
document: "a: hello",
expression: `.a = system("` + echoPath + `"; ["foo", "bar"])`,
expected: []string{
"D0, P[], (!!map)::a: foo bar\n",
},
},
{
description: "Command and args are evaluated per matched node",
skipDoc: true,
document: "cmd: " + echoPath + "\narg: hello",
expression: `.result = system(.cmd; .arg)`,
expected: []string{
"D0, P[], (!!map)::cmd: " + echoPath + "\narg: hello\nresult: hello\n",
},
},
{
description: "Command failure returns error",
skipDoc: true,
document: "a: hello",
expression: `.a = system("` + falsePath + `")`,
expectedError: "system command '" + falsePath + "' failed: exit status 1",
},
{
description: "Null command returns error",
skipDoc: true,
document: "a: hello",
expression: `.a = system(null)`,
expectedError: "system operator: command must be a string scalar",
},
{
description: "System operator processes multiple matched nodes",
skipDoc: true,
document: "a: first",
document2: "a: second",
expression: `.a = system("` + echoPath + `"; "replaced")`,
expected: []string{
"D0, P[], (!!map)::a: replaced\n",
"D0, P[], (!!map)::a: replaced\n",
},
},
}
for _, tt := range scenarios {
testScenario(t, &tt)
}
appendOperatorDocumentScenario(t, "system-operators", scenarios)
}
+1 -1
View File
@@ -6,7 +6,7 @@ import (
func assignTagOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("AssignTagOperator")
log.Debugf("AssignTagOperator: %v")
tag := ""
if !expressionNode.Operation.UpdateAssign {
+15 -42
View File
@@ -36,32 +36,8 @@ func traversePathOperator(_ *dataTreeNavigator, context Context, expressionNode
return context.ChildContext(matches), nil
}
// resolveAliasChain follows an alias chain iteratively, returning the
// first non-alias node. Returns an error if a cycle is detected.
func resolveAliasChain(node *CandidateNode) (*CandidateNode, error) {
if node.Kind != AliasNode {
return node, nil
}
visited := map[*CandidateNode]bool{}
for node.Kind == AliasNode {
if visited[node] {
return nil, fmt.Errorf("alias cycle detected")
}
visited[node] = true
log.Debug("its an alias!")
node = node.Alias
}
return node, nil
}
func traverse(context Context, matchingNode *CandidateNode, operation *Operation) (*list.List, error) {
log.Debugf("Traversing %v", NodeToString(matchingNode))
var err error
matchingNode, err = resolveAliasChain(matchingNode)
if err != nil {
return nil, err
}
log.Debug("Traversing %v", NodeToString(matchingNode))
if matchingNode.Tag == "!!null" && operation.Value != "[]" && !context.DontAutoCreate {
log.Debugf("Guessing kind")
@@ -79,13 +55,17 @@ func traverse(context Context, matchingNode *CandidateNode, operation *Operation
switch matchingNode.Kind {
case MappingNode:
log.Debugf("its a map with %v entries", len(matchingNode.Content)/2)
log.Debug("its a map with %v entries", len(matchingNode.Content)/2)
return traverseMap(context, matchingNode, createStringScalarNode(operation.StringValue), operation.Preferences.(traversePreferences), false)
case SequenceNode:
log.Debugf("its a sequence of %v things!", len(matchingNode.Content))
log.Debug("its a sequence of %v things!", len(matchingNode.Content))
return traverseArray(matchingNode, operation, operation.Preferences.(traversePreferences))
case AliasNode:
log.Debug("its an alias!")
matchingNode = matchingNode.Alias
return traverse(context, matchingNode, operation)
default:
return list.New(), nil
}
@@ -99,11 +79,7 @@ func traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode
log.Debugf("--traverseArrayOperator")
if expressionNode.RHS != nil && expressionNode.RHS.RHS != nil && expressionNode.RHS.RHS.Operation.OperationType == createMapOpType {
lhsContext, err := d.GetMatchingNodes(context, expressionNode.LHS)
if err != nil {
return Context{}, err
}
return sliceArrayOperator(d, lhsContext, expressionNode.RHS.RHS)
return sliceArrayOperator(d, context, expressionNode.RHS.RHS)
}
lhs, err := d.GetMatchingNodes(context, expressionNode.LHS)
@@ -149,13 +125,7 @@ func traverseNodesWithArrayIndices(context Context, indicesToTraverse []*Candida
return context.ChildContext(matchingNodeMap), nil
}
func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesToTraverse []*CandidateNode, prefs traversePreferences) (*list.List, error) {
var err error
matchingNode, err = resolveAliasChain(matchingNode)
if err != nil {
return nil, err
}
func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesToTraverse []*CandidateNode, prefs traversePreferences) (*list.List, error) { // call this if doc / alias like the other traverse
if matchingNode.Tag == "!!null" {
log.Debugf("OperatorArrayTraverse got a null - turning it into an empty array")
// auto vivification
@@ -168,6 +138,9 @@ func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesT
}
switch matchingNode.Kind {
case AliasNode:
matchingNode = matchingNode.Alias
return traverseArrayIndices(context, matchingNode, indicesToTraverse, prefs)
case SequenceNode:
return traverseArrayWithIndices(matchingNode, indicesToTraverse, prefs)
case MappingNode:
@@ -185,7 +158,7 @@ func traverseMapWithIndices(context Context, candidate *CandidateNode, indices [
var matchingNodeMap = list.New()
for _, indexNode := range indices {
log.Debugf("traverseMapWithIndices: %v", indexNode.Value)
log.Debug("traverseMapWithIndices: %v", indexNode.Value)
newNodes, err := traverseMap(context, candidate, indexNode, prefs, false)
if err != nil {
return nil, err
@@ -210,7 +183,7 @@ func traverseArrayWithIndices(node *CandidateNode, indices []*CandidateNode, pre
}
for _, indexNode := range indices {
log.Debugf("traverseArrayWithIndices: '%v'", indexNode.Value)
log.Debug("traverseArrayWithIndices: '%v'", indexNode.Value)
index, err := parseInt(indexNode.Value)
if err != nil && prefs.OptionalTraverse {
continue
@@ -393,7 +366,7 @@ func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, merge *CandidateNode
}
func traverseArray(candidate *CandidateNode, operation *Operation, prefs traversePreferences) (*list.List, error) {
log.Debugf("operation Value %v", operation.Value)
log.Debug("operation Value %v", operation.Value)
indices := []*CandidateNode{{Value: operation.StringValue}}
return traverseArrayWithIndices(candidate, indices, prefs)
}
-65
View File
@@ -665,16 +665,6 @@ var traversePathOperatorScenarios = []expressionScenario{
"D0, P[a], (!!null)::null\n",
},
},
{
// Regression test for https://issues.oss-fuzz.com/issues/390467412
// go-yaml accepts cross-document alias references (invalid per
// YAML spec). A nested assignment on such an alias can create a
// circular alias node, which must not cause a stack overflow.
skipDoc: true,
document: "&-- a\n---\n*--",
expression: ". = (.x = 1)",
expectedError: "alias cycle detected",
},
}
func TestTraversePathOperatorScenarios(t *testing.T) {
@@ -692,58 +682,3 @@ func TestTraversePathOperatorAlignedToSpecScenarios(t *testing.T) {
appendOperatorDocumentScenario(t, "traverse-read", fixedTraversePathOperatorScenarios)
ConfiguredYamlPreferences.FixMergeAnchorToSpec = false
}
// Regression test for https://issues.oss-fuzz.com/issues/390467412
// A circular alias (alias pointing back to itself) must not cause a
// stack overflow. resolveAliasChain should detect the cycle and return
// an error; both traverse() and traverseArrayIndices() use it.
func TestTraverseAliasCycle(t *testing.T) {
aliasNode := &CandidateNode{
Kind: AliasNode,
}
aliasNode.Alias = aliasNode // A -> A
op := &Operation{
OperationType: traversePathOpType,
Value: "key",
StringValue: "key",
Preferences: traversePreferences{},
}
_, err := traverse(Context{}, aliasNode, op)
if err == nil {
t.Fatal("expected error for alias cycle, got nil")
}
if err.Error() != "alias cycle detected" {
t.Fatalf("expected 'alias cycle detected', got %q", err.Error())
}
// Same cycle must be caught through the array traversal path.
_, err = traverseArrayIndices(Context{}, aliasNode, nil, traversePreferences{})
if err == nil {
t.Fatal("expected error for alias cycle via traverseArrayIndices, got nil")
}
if err.Error() != "alias cycle detected" {
t.Fatalf("expected 'alias cycle detected', got %q", err.Error())
}
}
func TestTraverseAliasCycleChain(t *testing.T) {
nodeA := &CandidateNode{Kind: AliasNode}
nodeB := &CandidateNode{Kind: AliasNode}
nodeA.Alias = nodeB
nodeB.Alias = nodeA // A -> B -> A
op := &Operation{
OperationType: traversePathOpType,
Value: "key",
StringValue: "key",
Preferences: traversePreferences{},
}
_, err := traverse(Context{}, nodeA, op)
if err == nil {
t.Fatal("expected error for alias cycle chain, got nil")
}
if err.Error() != "alias cycle detected" {
t.Fatalf("expected 'alias cycle detected', got %q", err.Error())
}
}
+8 -8
View File
@@ -4,21 +4,21 @@ import "container/list"
func unionOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debug("unionOperator--")
log.Debugf("unionOperator: context: %v", NodesToString(context.MatchingNodes))
log.Debug("unionOperator: context: %v", NodesToString(context.MatchingNodes))
lhs, err := d.GetMatchingNodes(context, expressionNode.LHS)
if err != nil {
return Context{}, err
}
log.Debugf("unionOperator: lhs: %v", NodesToString(lhs.MatchingNodes))
log.Debugf("unionOperator: rhs input: %v", NodesToString(context.MatchingNodes))
log.Debugf("unionOperator: rhs: %v", expressionNode.RHS.Operation.toString())
log.Debug("unionOperator: lhs: %v", NodesToString(lhs.MatchingNodes))
log.Debug("unionOperator: rhs input: %v", NodesToString(context.MatchingNodes))
log.Debug("unionOperator: rhs: %v", expressionNode.RHS.Operation.toString())
rhs, err := d.GetMatchingNodes(context, expressionNode.RHS)
if err != nil {
return Context{}, err
}
log.Debugf("unionOperator: lhs: %v", lhs.ToString())
log.Debugf("unionOperator: rhs: %v", rhs.ToString())
log.Debug("unionOperator: lhs: %v", lhs.ToString())
log.Debug("unionOperator: rhs: %v", rhs.ToString())
results := lhs.ChildContext(list.New())
for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
@@ -33,11 +33,11 @@ func unionOperator(d *dataTreeNavigator, context Context, expressionNode *Expres
for el := rhs.MatchingNodes.Front(); el != nil; el = el.Next() {
node := el.Value.(*CandidateNode)
log.Debugf("union operator rhs: processing %v", NodeToString(node))
log.Debug("union operator rhs: processing %v", NodeToString(node))
results.MatchingNodes.PushBack(node)
}
}
log.Debugf("union operator: all together: %v", results.ToString())
log.Debug("union operator: all together: %v", results.ToString())
return results, nil
}
+1 -1
View File
@@ -7,7 +7,7 @@ func referenceOperator(_ *dataTreeNavigator, context Context, expressionNode *Ex
}
func valueOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("value = %v", expressionNode.Operation.CandidateNode.Value)
log.Debug("value = %v", expressionNode.Operation.CandidateNode.Value)
if context.MatchingNodes.Len() == 0 {
clone := expressionNode.Operation.CandidateNode.Copy()
return context.SingleChildContext(clone), nil
+3 -3
View File
@@ -7,7 +7,7 @@ import (
func getVariableOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
variableName := expressionNode.Operation.StringValue
log.Debugf("getVariableOperator %v", variableName)
log.Debug("getVariableOperator %v", variableName)
result := context.GetVariable(variableName)
if result == nil {
result = list.New()
@@ -67,7 +67,7 @@ func variableLoopSingleChild(d *dataTreeNavigator, context Context, originalExp
// now we loop over lhs, set variable to each result and calculate originalExp.Rhs
for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
log.Debugf("PROCESSING VARIABLE: %v", NodeToString(el.Value.(*CandidateNode)))
log.Debug("PROCESSING VARIABLE: ", NodeToString(el.Value.(*CandidateNode)))
var variableValue = list.New()
if prefs.IsReference {
variableValue.PushBack(el.Value)
@@ -83,7 +83,7 @@ func variableLoopSingleChild(d *dataTreeNavigator, context Context, originalExp
if err != nil {
return Context{}, err
}
log.Debugf("PROCESSING VARIABLE DONE, got back: %v", rhs.MatchingNodes.Len())
log.Debug("PROCESSING VARIABLE DONE, got back: ", rhs.MatchingNodes.Len())
results.PushBackList(rhs.MatchingNodes)
}
+2 -2
View File
@@ -3,9 +3,9 @@ package yqlib
import (
"container/list"
"fmt"
"log/slog"
"github.com/jinzhu/copier"
logging "gopkg.in/op/go-logging.v1"
)
type operatorHandler func(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error)
@@ -88,7 +88,7 @@ func resultsForRHS(d *dataTreeNavigator, context Context, lhsCandidate *Candidat
for rightEl := rhs.MatchingNodes.Front(); rightEl != nil; rightEl = rightEl.Next() {
rhsCandidate := rightEl.Value.(*CandidateNode)
if !log.IsEnabledFor(slog.LevelDebug) {
if !log.IsEnabledFor(logging.DEBUG) {
log.Debugf("Applying lhs: %v, rhsCandidate, %v", NodeToString(lhsCandidate), NodeToString(rhsCandidate))
}
resultCandidate, err := prefs.Calculation(d, context, lhsCandidate, rhsCandidate)
+6 -15
View File
@@ -7,7 +7,6 @@ import (
"fmt"
"io"
"io/fs"
"log/slog"
"os"
"sort"
"strings"
@@ -15,6 +14,7 @@ import (
"time"
"github.com/mikefarah/yq/v4/test"
logging "gopkg.in/op/go-logging.v1"
)
type expressionScenario struct {
@@ -31,7 +31,6 @@ type expressionScenario struct {
dontFormatInputForDoc bool // dont format input doc for documentation generation
requiresFormat string
skipForGoccy bool
yqFlags string // extra yq flags to include in generated doc command snippets
}
var goccyTesting = false
@@ -39,9 +38,9 @@ var goccyTesting = false
var testingDecoder = NewYamlDecoder(ConfiguredYamlPreferences)
func TestMain(m *testing.M) {
GetLogger().SetLevel(slog.LevelWarn)
logging.SetLevel(logging.WARNING, "")
if os.Getenv("DEBUG") == "true" {
GetLogger().SetLevel(slog.LevelDebug)
logging.SetLevel(logging.DEBUG, "")
}
ConfiguredYamlPreferences.ColorsEnabled = false
ConfiguredJSONPreferences.ColorsEnabled = false
@@ -357,22 +356,14 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
writeOrPanic(w, "then\n")
flagsPrefix := ""
if s.yqFlags != "" {
flagsPrefix = s.yqFlags + " "
}
if s.expression != "" {
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v'%v' %v\n```\n", envCommand, flagsPrefix, command, strings.ReplaceAll(s.expression, "'", `'\''`), files))
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v'%v' %v\n```\n", envCommand, command, strings.ReplaceAll(s.expression, "'", `'\''`), files))
} else {
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v%v\n```\n", envCommand, flagsPrefix, command, files))
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v\n```\n", envCommand, command, files))
}
} else {
writeOrPanic(w, "Running\n")
flagsPrefix := ""
if s.yqFlags != "" {
flagsPrefix = s.yqFlags + " "
}
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v--null-input '%v'\n```\n", envCommand, flagsPrefix, command, s.expression))
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v--null-input '%v'\n```\n", envCommand, command, s.expression))
}
return formattedDoc, formattedDoc2
}
+3 -3
View File
@@ -70,7 +70,7 @@ func removeLastEOL(b *bytes.Buffer) {
}
func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
log.Debugf("PrintResults for %v matches", matchingNodes.Len())
log.Debug("PrintResults for %v matches", matchingNodes.Len())
if matchingNodes.Len() == 0 {
log.Debug("no matching results, nothing to print")
@@ -97,8 +97,8 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
for el := matchingNodes.Front(); el != nil; el = el.Next() {
mappedDoc := el.Value.(*CandidateNode)
log.Debugf("print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v", p.firstTimePrinting, p.previousDocIndex)
log.Debugf("%v", NodeToString(mappedDoc))
log.Debug("print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v", p.firstTimePrinting, p.previousDocIndex)
log.Debug("%v", NodeToString(mappedDoc))
writer, errorWriting := p.printerWriter.GetWriter(mappedDoc)
if errorWriting != nil {
return errorWriting
+4 -6
View File
@@ -1,13 +1,11 @@
package yqlib
type SecurityPreferences struct {
DisableEnvOps bool
DisableFileOps bool
EnableSystemOps bool
DisableEnvOps bool
DisableFileOps bool
}
var ConfiguredSecurityPreferences = SecurityPreferences{
DisableEnvOps: false,
DisableFileOps: false,
EnableSystemOps: false,
DisableEnvOps: false,
DisableFileOps: false,
}
-1
View File
@@ -76,7 +76,6 @@ func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, p
}
func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, decoder Decoder) (uint, error) {
filename = resolveFilename(filename)
var currentIndex uint
err := decoder.Init(reader)
-90
View File
@@ -287,18 +287,6 @@ var expectedSubArrays = `array:
- {}
`
// Keys with special characters that require quoting in TOML
var rtSpecialKeyInlineTable = `host = { "http://sealos.hub:5000" = { capabilities = ["pull", "resolve", "push"], skip_verify = true } }
`
var rtSpecialKeyTableSection = `["/tmp/blah"]
value = "hello"
`
var rtSpecialKeyDottedTableSection = `[servers."http://localhost:8080"]
ip = "127.0.0.1"
`
var tomlScenarios = []formatScenario{
{
skipDoc: true,
@@ -626,58 +614,6 @@ var tomlScenarios = []formatScenario{
expected: tomlTableWithComments,
scenarioType: "roundtrip",
},
// Encode (YAML → TOML) scenarios - verify readable table sections are produced
{
description: "Encode: Simple mapping produces table section",
input: "arg:\n hello: foo\n",
expected: "[arg]\nhello = \"foo\"\n",
scenarioType: "encode",
},
{
skipDoc: true,
description: "Encode: Nested mappings produce nested table sections",
input: "a:\n b:\n c: val\n",
expected: "[a.b]\nc = \"val\"\n",
scenarioType: "encode",
},
{
skipDoc: true,
description: "Encode: Mixed scalars and nested mapping",
input: "a:\n hello: foo\n nested:\n key: val\n",
expected: "[a]\nhello = \"foo\"\n\n[a.nested]\nkey = \"val\"\n",
scenarioType: "encode",
},
{
skipDoc: true,
description: "Encode: YAML flow mapping stays inline",
input: "arg: {hello: foo}\n",
expected: "arg = { hello = \"foo\" }\n",
scenarioType: "encode",
},
{
skipDoc: true,
description: "Roundtrip: key with special characters in inline table",
input: rtSpecialKeyInlineTable,
expression: ".",
expected: rtSpecialKeyInlineTable,
scenarioType: "roundtrip",
},
{
skipDoc: true,
description: "Roundtrip: key with special characters in table section",
input: rtSpecialKeyTableSection,
expression: ".",
expected: rtSpecialKeyTableSection,
scenarioType: "roundtrip",
},
{
skipDoc: true,
description: "Roundtrip: special character key in dotted table section header",
input: rtSpecialKeyDottedTableSection,
expression: ".",
expected: rtSpecialKeyDottedTableSection,
scenarioType: "roundtrip",
},
}
func testTomlScenario(t *testing.T, s formatScenario) {
@@ -693,8 +629,6 @@ func testTomlScenario(t *testing.T, s formatScenario) {
}
case "roundtrip":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewTomlDecoder(), NewTomlEncoder()), s.description)
case "encode":
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewTomlEncoder()), s.description)
}
}
@@ -720,28 +654,6 @@ func documentTomlDecodeScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(ConfiguredYamlPreferences))))
}
func documentTomlEncodeScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
if s.subdescription != "" {
writeOrPanic(w, s.subdescription)
writeOrPanic(w, "\n\n")
}
writeOrPanic(w, "Given a sample.yml file of:\n")
writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input))
writeOrPanic(w, "then\n")
expression := s.expression
if expression == "" {
expression = "."
}
writeOrPanic(w, fmt.Sprintf("```bash\nyq -o toml '%v' sample.yml\n```\n", expression))
writeOrPanic(w, "will output\n")
writeOrPanic(w, fmt.Sprintf("```toml\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewTomlEncoder())))
}
func documentTomlRoundtripScenario(w *bufio.Writer, s formatScenario) {
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
@@ -775,8 +687,6 @@ func documentTomlScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
documentTomlDecodeScenario(w, s)
case "roundtrip":
documentTomlRoundtripScenario(w, s)
case "encode":
documentTomlEncodeScenario(w, s)
default:
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
-24
View File
@@ -9,29 +9,6 @@ import (
"os"
)
// filenameAliases maps real file paths to display names.
// Used by front matter handling to preserve original filenames
// when the actual content is read from temporary files.
var filenameAliases = map[string]string{}
// SetFilenameAlias registers a display name for a file path so that
// the filename operator returns the original name instead of a temp path.
func SetFilenameAlias(realPath string, displayName string) {
filenameAliases[realPath] = displayName
}
// ClearFilenameAliases removes all filename aliases.
func ClearFilenameAliases() {
filenameAliases = map[string]string{}
}
func resolveFilename(filename string) string {
if alias, ok := filenameAliases[filename]; ok {
return alias
}
return filename
}
func readStream(filename string) (io.Reader, error) {
var reader *bufio.Reader
if filename == "-" {
@@ -59,7 +36,6 @@ func ReadDocuments(reader io.Reader, decoder Decoder) (*list.List, error) {
}
func readDocuments(reader io.Reader, filename string, fileIndex int, decoder Decoder) (*list.List, error) {
filename = resolveFilename(filename)
err := decoder.Init(reader)
if err != nil {
return nil, err
+2 -2
View File
@@ -38,13 +38,13 @@ func (w *writeInPlaceHandlerImpl) CreateTempFile() (*os.File, error) {
if err = changeOwner(info, file); err != nil {
return nil, err
}
log.Debugf("WriteInPlaceHandler: writing to tempfile: %v", file.Name())
log.Debug("WriteInPlaceHandler: writing to tempfile: %v", file.Name())
w.tempFile = file
return file, err
}
func (w *writeInPlaceHandlerImpl) FinishWriteInPlace(evaluatedSuccessfully bool) error {
log.Debugf("Going to write in place, evaluatedSuccessfully=%v, target=%v", evaluatedSuccessfully, w.inputFilename)
log.Debug("Going to write in place, evaluatedSuccessfully=%v, target=%v", evaluatedSuccessfully, w.inputFilename)
safelyCloseFile(w.tempFile)
if evaluatedSuccessfully {
log.Debug("Moving temp file to target")
+1 -7
View File
@@ -109,7 +109,6 @@ hostpath
hotdog
howdy
incase
Infof
inlinetables
inplace
ints
@@ -298,9 +297,4 @@ subsubarray
Ffile
Fquery
coverpkg
gsub
ralia
Austr
ustrali
héllo
alia
gsub
-9
View File
@@ -8,15 +8,6 @@
- git push --tags
- use github actions to publish docker and make github release
- check github updated yq action in marketplace
- update github-action/Dockerfile to pin the newly published docker image digest:
skopeo inspect docker://docker.io/mikefarah/yq:4 | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['Digest'])"
then update the FROM line in github-action/Dockerfile with the new digest:
FROM mikefarah/yq:4@sha256:<new-digest>
// release artifacts are signed with cosign keyless signing (Sigstore)
// users can verify with:
// cosign verify-blob --bundle checksums.bundle checksums
// install cosign: brew install cosign OR go install github.com/sigstore/cosign/v2/cmd/cosign@latest
- snapcraft
-14
View File
@@ -1,17 +1,3 @@
4.52.5:
- Fix: reset TOML decoder state between files (#2634) thanks @terminalchai
- Fix: preserve original filename when using --front-matter (#2613) thanks @cobyfrombrooklyn-bot
- Fix typo in filename (#2611) thanks @alexandear
- Bumped dependencies
4.52.4:
- Dropping windows/arm - no longer supported in cross-compile
4.52.3:
- Fixing comments in TOML arrays (#2592)
- Bumped dependencies
4.52.2:
- Fixed bad instructions file breaking go-install (#2587) Thanks @theyoprst
- Fixed TOML table scope after comments (#2588) Thanks @tomers
+2
View File
@@ -3,6 +3,8 @@
set -o errexit
set -o pipefail
# TODO: Check if the found golangci-lint version matches the expected version (e.g., v1.62.0), especially if falling back to PATH version.
GOPATH_LINT="$(go env GOPATH)/bin/golangci-lint"
BIN_LINT="./bin/golangci-lint"
LINT_CMD=""
+1 -1
View File
@@ -1,5 +1,5 @@
#!/bin/sh
set -ex
go mod download golang.org/x/tools@latest
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.11.3
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.1.5
curl -sSfL https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.22.11
+2 -2
View File
@@ -1,5 +1,5 @@
name: yq
version: 'v4.52.5'
version: 'v4.52.2'
summary: A lightweight and portable command-line data file processor
description: |
`yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml, json, xml, csv, properties and TOML files.
@@ -32,6 +32,6 @@ parts:
build-environment:
- CGO_ENABLED: 0
source: https://github.com/mikefarah/yq.git
source-tag: v4.52.5
source-tag: v4.52.2
build-snaps:
- go/latest/stable
-2
View File
@@ -14,7 +14,6 @@ func TestMainFunction(t *testing.T) {
cmd := command.New()
if cmd == nil {
t.Fatal("command.New() returned nil")
return
}
if cmd.Use != "yq" {
@@ -100,7 +99,6 @@ func TestMainFunctionExecution(t *testing.T) {
// the command structure is correct and can be configured
if cmd == nil {
t.Fatal("Command should not be nil")
return
}
if cmd.Use != "yq" {