mirror of
https://github.com/mikefarah/yq.git
synced 2026-08-24 16:39:58 +08:00
Compare commits
43
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c8f6c1a042 | ||
|
|
0e803833fb | ||
|
|
30ca9ffde7 | ||
|
|
2927a28283 | ||
|
|
c47fe40a30 | ||
|
|
8c018da9c9 | ||
|
|
44c55c8a54 | ||
|
|
22e609b2d9 | ||
|
|
3b2423e871 | ||
|
|
68f0322ba3 | ||
|
|
d69c7d1a36 | ||
|
|
b0ba9589d7 | ||
|
|
80139ae1cc | ||
|
|
0374ad6b4b | ||
|
|
2ef934281e | ||
|
|
17f66dc6c6 | ||
|
|
dcb9c2a543 | ||
|
|
8f5d876bf0 | ||
|
|
7d8d3ab902 | ||
|
|
11f4dc1a03 | ||
|
|
0f4fb8d35e | ||
|
|
80c319aa0c | ||
|
|
b25ae78545 | ||
|
|
b151522485 | ||
|
|
c5cbf9760b | ||
|
|
b5cb9a2f20 | ||
|
|
133ba767a6 | ||
|
|
5db3dcf394 | ||
|
|
4c148178e2 | ||
|
|
4df6e46f95 | ||
|
|
6a965bc39a | ||
|
|
34d3a29308 | ||
|
|
16e4df2304 | ||
|
|
79a92d0478 | ||
|
|
88a31ae8c6 | ||
|
|
5a7e72a743 | ||
|
|
562531d936 | ||
|
|
2c471b6498 | ||
|
|
f4ef6ef3cf | ||
|
|
f49f2bd2d8 | ||
|
|
6ccc7b7797 | ||
|
|
b3e1fbb7d1 | ||
|
|
288ca2d114 |
@@ -38,11 +38,11 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
- name: Initialize CodeQL
|
||||
uses: github/codeql-action/init@v4
|
||||
uses: github/codeql-action/init@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1
|
||||
with:
|
||||
languages: ${{ matrix.language }}
|
||||
# If you wish to specify custom queries, you can do so here or in a config file.
|
||||
@@ -53,7 +53,7 @@ jobs:
|
||||
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
|
||||
# If this step fails, then you should remove it and run the build manually (see below)
|
||||
- name: Autobuild
|
||||
uses: github/codeql-action/autobuild@v4
|
||||
uses: github/codeql-action/autobuild@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1
|
||||
|
||||
# ℹ️ Command-line programs to run using the OS shell.
|
||||
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
|
||||
@@ -67,4 +67,4 @@ jobs:
|
||||
# make release
|
||||
|
||||
- name: Perform CodeQL Analysis
|
||||
uses: github/codeql-action/analyze@v4
|
||||
uses: github/codeql-action/analyze@c10b8064de6f491fea524254123dbe5e09572f13 # v4.35.1
|
||||
|
||||
@@ -14,16 +14,16 @@ jobs:
|
||||
IMAGE_NAME: mikefarah/yq
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v3
|
||||
uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0
|
||||
with:
|
||||
platforms: all
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
with:
|
||||
version: latest
|
||||
|
||||
@@ -31,13 +31,13 @@ jobs:
|
||||
run: echo ${{ steps.buildx.outputs.platforms }} && docker version
|
||||
|
||||
- name: Login to Docker Hub
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
|
||||
- name: Login to GitHub Container Registry
|
||||
uses: docker/login-action@v3
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
|
||||
@@ -11,13 +11,13 @@ jobs:
|
||||
steps:
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v6
|
||||
uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version: '^1.20'
|
||||
id: go
|
||||
|
||||
- name: Check out code into the Go module directory
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
|
||||
- name: Get dependencies
|
||||
run: |
|
||||
|
||||
@@ -8,9 +8,12 @@ on:
|
||||
jobs:
|
||||
publishGitRelease:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/setup-go@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version: '^1.20'
|
||||
check-latest: true
|
||||
@@ -37,14 +40,22 @@ jobs:
|
||||
--output=yq.1
|
||||
man.md
|
||||
|
||||
- name: Install cosign
|
||||
uses: sigstore/cosign-installer@v3
|
||||
|
||||
- name: Cross compile
|
||||
run: |
|
||||
sudo apt-get install rhash -y
|
||||
go install github.com/goreleaser/goreleaser/v2@latest
|
||||
./scripts/xcompile.sh
|
||||
|
||||
- name: Sign checksums
|
||||
run: |
|
||||
cosign sign-blob --yes --output-bundle build/checksums.bundle build/checksums
|
||||
cosign sign-blob --yes --output-bundle build/checksums-bsd.bundle build/checksums-bsd
|
||||
|
||||
- name: Release
|
||||
uses: softprops/action-gh-release@v1
|
||||
uses: softprops/action-gh-release@3bb12739c298aeb8a4eeaf626c5b8d85266b0e65 # v2.6.2
|
||||
with:
|
||||
files: build/*
|
||||
draft: true
|
||||
|
||||
@@ -12,14 +12,14 @@ jobs:
|
||||
environment: snap
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: snapcore/action-build@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- uses: snapcore/action-build@3bdaa03e1ba6bf59a65f84a751d943d549a54e79 # v1.3.0
|
||||
id: build
|
||||
env:
|
||||
SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.STORE_LOGIN }}
|
||||
with:
|
||||
snapcraft-args: "remote-build --launchpad-accept-public-upload"
|
||||
- uses: snapcore/action-publish@v1
|
||||
- uses: snapcore/action-publish@214b86e5ca036ead1668c79afb81e550e6c54d40 # v1.2.0
|
||||
env:
|
||||
SNAPCRAFT_STORE_CREDENTIALS: ${{ secrets.STORE_LOGIN }}
|
||||
with:
|
||||
|
||||
@@ -13,7 +13,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
- name: Get test
|
||||
id: get_value
|
||||
uses: mikefarah/yq@master
|
||||
|
||||
@@ -39,7 +39,6 @@ builds:
|
||||
- openbsd_amd64
|
||||
- windows_386
|
||||
- windows_amd64
|
||||
- windows_arm
|
||||
- windows_arm64
|
||||
|
||||
no_unique_dist_dir: true
|
||||
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.25.6 AS builder
|
||||
FROM golang:1.26.2@sha256:2a2b4b5791cea8ae09caecba7bad0bd9631def96e5fe362e4a5e67009fe4ae61 AS builder
|
||||
|
||||
WORKDIR /go/src/mikefarah/yq
|
||||
|
||||
@@ -10,7 +10,7 @@ RUN ./scripts/acceptance.sh
|
||||
|
||||
# Choose alpine as a base image to make this useful for CI, as many
|
||||
# CI tools expect an interactive shell inside the container
|
||||
FROM alpine:3 AS production
|
||||
FROM alpine:3@sha256:25109184c71bdad752c8312a8623239686a9a2071e8825f20acb8f2198c3f659 AS production
|
||||
LABEL maintainer="Mike Farah <mikefarah@users.noreply.github.com>"
|
||||
|
||||
COPY --from=builder /go/src/mikefarah/yq/yq /usr/bin/yq
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
FROM golang:1.25.6
|
||||
FROM golang:1.26.2@sha256:2a2b4b5791cea8ae09caecba7bad0bd9631def96e5fe362e4a5e67009fe4ae61
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y npm && \
|
||||
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
# Security Policy
|
||||
|
||||
## Reporting a Vulnerability
|
||||
|
||||
Please **do not** report security vulnerabilities through public GitHub issues.
|
||||
|
||||
Instead, use GitHub's private vulnerability reporting feature:
|
||||
👉 https://github.com/mikefarah/yq/security
|
||||
|
||||
This allows vulnerabilities to be triaged and addressed confidentially before any public disclosure.
|
||||
|
||||
## Scope
|
||||
|
||||
### HTTP / TLS / Network vulnerabilities
|
||||
|
||||
yq is a command-line YAML/JSON/TOML processor that reads from files or standard input and writes to standard output. **yq does not include any HTTP or network libraries** and makes no network connections at runtime. CVEs related to HTTP, TLS, or networking are therefore **not applicable** to yq.
|
||||
|
||||
### Dependency version bumps
|
||||
|
||||
yq uses [Dependabot](https://docs.github.com/en/code-security/dependabot) to automatically raise pull requests for:
|
||||
|
||||
- Go module dependencies
|
||||
- Go toolchain version
|
||||
- Docker base images
|
||||
|
||||
Please **do not** raise pull requests or issues solely to bump dependency or Go versions — Dependabot handles this automatically and the maintainers merge those PRs regularly.
|
||||
@@ -101,12 +101,15 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
|
||||
}
|
||||
|
||||
if frontMatter != "" {
|
||||
originalFilename := args[0]
|
||||
frontMatterHandler := yqlib.NewFrontMatterHandler(args[0])
|
||||
err = frontMatterHandler.Split()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
args[0] = frontMatterHandler.GetYamlFrontMatterFilename()
|
||||
yqlib.SetFilenameAlias(args[0], originalFilename)
|
||||
defer yqlib.ClearFilenameAliases()
|
||||
|
||||
if frontMatter == "process" {
|
||||
reader := frontMatterHandler.GetContentReader()
|
||||
|
||||
@@ -13,6 +13,7 @@ func TestCreateEvaluateAllCommand(t *testing.T) {
|
||||
|
||||
if cmd == nil {
|
||||
t.Fatal("createEvaluateAllCommand returned nil")
|
||||
return
|
||||
}
|
||||
|
||||
// Test basic command properties
|
||||
|
||||
@@ -122,12 +122,15 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
|
||||
|
||||
if frontMatter != "" {
|
||||
yqlib.GetLogger().Debug("using front matter handler")
|
||||
originalFilename := args[0]
|
||||
frontMatterHandler := yqlib.NewFrontMatterHandler(args[0])
|
||||
err = frontMatterHandler.Split()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
args[0] = frontMatterHandler.GetYamlFrontMatterFilename()
|
||||
yqlib.SetFilenameAlias(args[0], originalFilename)
|
||||
defer yqlib.ClearFilenameAliases()
|
||||
|
||||
if frontMatter == "process" {
|
||||
reader := frontMatterHandler.GetContentReader()
|
||||
|
||||
@@ -13,6 +13,7 @@ func TestCreateEvaluateSequenceCommand(t *testing.T) {
|
||||
|
||||
if cmd == nil {
|
||||
t.Fatal("createEvaluateSequenceCommand returned nil")
|
||||
return
|
||||
}
|
||||
|
||||
// Test basic command properties
|
||||
|
||||
+9
-17
@@ -2,12 +2,12 @@ package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/mikefarah/yq/v4/pkg/yqlib"
|
||||
"github.com/spf13/cobra"
|
||||
logging "gopkg.in/op/go-logging.v1"
|
||||
)
|
||||
|
||||
type runeValue rune
|
||||
@@ -68,30 +68,21 @@ yq -P -oy sample.json
|
||||
},
|
||||
PersistentPreRunE: func(cmd *cobra.Command, _ []string) error {
|
||||
cmd.SetOut(cmd.OutOrStdout())
|
||||
level := logging.WARNING
|
||||
stringFormat := `[%{level}] %{color}%{time:15:04:05}%{color:reset} %{message}`
|
||||
|
||||
// when NO_COLOR environment variable presents and not an empty string the coloured output should be disabled;
|
||||
// refer to no-color.org
|
||||
forceNoColor = forceNoColor || os.Getenv("NO_COLOR") != ""
|
||||
|
||||
if verbose && forceNoColor {
|
||||
level = logging.DEBUG
|
||||
stringFormat = `[%{level:5.5s}] %{time:15:04:05} %{shortfile:-33s} %{shortfunc:-25s} %{message}`
|
||||
} else if verbose {
|
||||
level = logging.DEBUG
|
||||
stringFormat = `[%{level:5.5s}] %{color}%{time:15:04:05}%{color:bold} %{shortfile:-33s} %{shortfunc:-25s}%{color:reset} %{message}`
|
||||
} else if forceNoColor {
|
||||
stringFormat = `[%{level}] %{time:15:04:05} %{message}`
|
||||
level := slog.LevelWarn
|
||||
if verbose {
|
||||
level = slog.LevelDebug
|
||||
}
|
||||
|
||||
var format = logging.MustStringFormatter(stringFormat)
|
||||
var backend = logging.AddModuleLevel(
|
||||
logging.NewBackendFormatter(logging.NewLogBackend(os.Stderr, "", 0), format))
|
||||
yqlib.GetLogger().SetLevel(level)
|
||||
opts := &slog.HandlerOptions{Level: level, AddSource: verbose}
|
||||
handler := slog.NewTextHandler(os.Stderr, opts)
|
||||
yqlib.GetLogger().SetSlogger(slog.New(handler))
|
||||
|
||||
backend.SetLevel(level, "")
|
||||
|
||||
logging.SetBackend(backend)
|
||||
yqlib.InitExpressionParser()
|
||||
|
||||
return nil
|
||||
@@ -221,6 +212,7 @@ yq -P -oy sample.json
|
||||
|
||||
rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableEnvOps, "security-disable-env-ops", "", false, "Disable env related operations.")
|
||||
rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableFileOps, "security-disable-file-ops", "", false, "Disable file related operations (e.g. load)")
|
||||
rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.EnableSystemOps, "security-enable-system-operator", "", false, "Enable system operator to allow execution of external commands.")
|
||||
|
||||
rootCmd.AddCommand(
|
||||
createEvaluateSequenceCommand(),
|
||||
|
||||
@@ -195,6 +195,7 @@ func TestNew(t *testing.T) {
|
||||
|
||||
if rootCmd == nil {
|
||||
t.Fatal("New() returned nil")
|
||||
return
|
||||
}
|
||||
|
||||
// Test basic command properties
|
||||
|
||||
+8
-8
@@ -3,12 +3,12 @@ package cmd
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/mikefarah/yq/v4/pkg/yqlib"
|
||||
"github.com/spf13/cobra"
|
||||
"gopkg.in/op/go-logging.v1"
|
||||
)
|
||||
|
||||
func isAutomaticOutputFormat() bool {
|
||||
@@ -104,8 +104,8 @@ func configureFormats(args []string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
yqlib.GetLogger().Debug("Using input format %v", inputFormat)
|
||||
yqlib.GetLogger().Debug("Using output format %v", outputFormat)
|
||||
yqlib.GetLogger().Debugf("Using input format %v", inputFormat)
|
||||
yqlib.GetLogger().Debugf("Using output format %v", outputFormat)
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -117,7 +117,7 @@ func configureInputFormat(inputFilename string) error {
|
||||
_, err := yqlib.FormatFromString(inputFormat)
|
||||
if err != nil {
|
||||
// unknown file type, default to yaml
|
||||
yqlib.GetLogger().Debug("Unknown file format extension '%v', defaulting to yaml", inputFormat)
|
||||
yqlib.GetLogger().Debugf("Unknown file format extension '%v', defaulting to yaml", inputFormat)
|
||||
inputFormat = "yaml"
|
||||
if isAutomaticOutputFormat() {
|
||||
outputFormat = "yaml"
|
||||
@@ -132,7 +132,7 @@ func configureInputFormat(inputFilename string) error {
|
||||
//
|
||||
outputFormat = yqlib.FormatStringFromFilename(inputFilename)
|
||||
if inputFilename != "-" {
|
||||
yqlib.GetLogger().Warning("yq default output is now 'auto' (based on the filename extension). Normally yq would output '%v', but for backwards compatibility 'yaml' has been set. Please use -oy to specify yaml, or drop the -p flag.", outputFormat)
|
||||
yqlib.GetLogger().Warningf("yq default output is now 'auto' (based on the filename extension). Normally yq would output '%v', but for backwards compatibility 'yaml' has been set. Please use -oy to specify yaml, or drop the -p flag.", outputFormat)
|
||||
}
|
||||
outputFormat = "yaml"
|
||||
}
|
||||
@@ -235,7 +235,7 @@ func maybeFile(str string) bool {
|
||||
yqlib.GetLogger().Debugf("checking '%v' is a file", str)
|
||||
stat, err := os.Stat(str) // #nosec
|
||||
result := err == nil && !stat.IsDir()
|
||||
if yqlib.GetLogger().IsEnabledFor(logging.DEBUG) {
|
||||
if yqlib.GetLogger().IsEnabledFor(slog.LevelDebug) {
|
||||
if err != nil {
|
||||
yqlib.GetLogger().Debugf("error: %v", err)
|
||||
} else {
|
||||
@@ -280,7 +280,7 @@ func processArgs(originalArgs []string) (string, []string, error) {
|
||||
|
||||
if expressionFile == "" && maybeFirstArgIsAFile && strings.HasSuffix(args[0], ".yq") {
|
||||
// lets check if an expression file was given
|
||||
yqlib.GetLogger().Debug("Assuming arg %v is an expression file", args[0])
|
||||
yqlib.GetLogger().Debugf("Assuming arg %v is an expression file", args[0])
|
||||
expressionFile = args[0]
|
||||
args = args[1:]
|
||||
}
|
||||
@@ -296,7 +296,7 @@ func processArgs(originalArgs []string) (string, []string, error) {
|
||||
|
||||
yqlib.GetLogger().Debugf("processed args: %v", args)
|
||||
if expression == "" && len(args) > 0 && args[0] != "-" && !maybeFile(args[0]) {
|
||||
yqlib.GetLogger().Debug("assuming expression is '%v'", args[0])
|
||||
yqlib.GetLogger().Debugf("assuming expression is '%v'", args[0])
|
||||
expression = args[0]
|
||||
args = args[1:]
|
||||
}
|
||||
|
||||
+1
-1
@@ -911,7 +911,7 @@ func stringsEqual(a, b []string) bool {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
if a[i] != b[i] { //nolint:gosec // G602 false positive: b length equality is checked above
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -11,7 +11,7 @@ var (
|
||||
GitDescribe string
|
||||
|
||||
// Version is main version number that is being run at the moment.
|
||||
Version = "v4.52.2"
|
||||
Version = "v4.52.5"
|
||||
|
||||
// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
|
||||
// then it means that it is a final release. Otherwise, this is a pre-release
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
FROM mikefarah/yq:4
|
||||
FROM mikefarah/yq:4@sha256:603ebff15eb308a05f1c5b8b7613179cad859aed3ec9fdd04f2ef5d32345950e
|
||||
|
||||
COPY entrypoint.sh /entrypoint.sh
|
||||
|
||||
|
||||
@@ -6,24 +6,23 @@ require (
|
||||
github.com/alecthomas/repr v0.5.2
|
||||
github.com/dimchansky/utfbom v1.1.1
|
||||
github.com/elliotchance/orderedmap v1.8.0
|
||||
github.com/fatih/color v1.18.0
|
||||
github.com/fatih/color v1.19.0
|
||||
github.com/go-ini/ini v1.67.0
|
||||
github.com/goccy/go-json v0.10.5
|
||||
github.com/goccy/go-json v0.10.6
|
||||
github.com/goccy/go-yaml v1.19.2
|
||||
github.com/hashicorp/hcl/v2 v2.24.0
|
||||
github.com/jinzhu/copier v0.4.0
|
||||
github.com/magiconair/properties v1.8.10
|
||||
github.com/pelletier/go-toml/v2 v2.2.4
|
||||
github.com/pelletier/go-toml/v2 v2.3.0
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/spf13/pflag v1.0.10
|
||||
github.com/yuin/gopher-lua v1.1.1
|
||||
github.com/zclconf/go-cty v1.17.0
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.3
|
||||
golang.org/x/mod v0.31.0
|
||||
golang.org/x/net v0.49.0
|
||||
golang.org/x/text v0.33.0
|
||||
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
|
||||
github.com/yuin/gopher-lua v1.1.2
|
||||
github.com/zclconf/go-cty v1.18.0
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.4
|
||||
golang.org/x/mod v0.34.0
|
||||
golang.org/x/net v0.52.0
|
||||
golang.org/x/text v0.35.0
|
||||
)
|
||||
|
||||
require (
|
||||
@@ -34,11 +33,9 @@ require (
|
||||
github.com/mattn/go-colorable v0.1.14 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.40.0 // indirect
|
||||
golang.org/x/tools v0.40.0 // indirect
|
||||
golang.org/x/sync v0.20.0 // indirect
|
||||
golang.org/x/sys v0.42.0 // indirect
|
||||
golang.org/x/tools v0.42.0 // indirect
|
||||
)
|
||||
|
||||
go 1.24.0
|
||||
|
||||
toolchain go1.24.1
|
||||
go 1.25.0
|
||||
|
||||
@@ -18,14 +18,14 @@ github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi
|
||||
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
|
||||
github.com/elliotchance/orderedmap v1.8.0 h1:TrOREecvh3JbS+NCgwposXG5ZTFHtEsQiCGOhPElnMw=
|
||||
github.com/elliotchance/orderedmap v1.8.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
|
||||
github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM=
|
||||
github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU=
|
||||
github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w=
|
||||
github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE=
|
||||
github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A=
|
||||
github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
|
||||
github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
|
||||
github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
|
||||
github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/goccy/go-json v0.10.6 h1:p8HrPJzOakx/mn/bQtjgNjdTcN+/S6FcG2CTtQOrHVU=
|
||||
github.com/goccy/go-json v0.10.6/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
|
||||
github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM=
|
||||
github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
@@ -46,8 +46,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0=
|
||||
github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pelletier/go-toml/v2 v2.3.0 h1:k59bC/lIZREW0/iVaQR8nDHxVq8OVlIzYCOJf421CaM=
|
||||
github.com/pelletier/go-toml/v2 v2.3.0/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
@@ -61,30 +61,28 @@ github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3A
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
|
||||
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
|
||||
github.com/zclconf/go-cty v1.17.0 h1:seZvECve6XX4tmnvRzWtJNHdscMtYEx5R7bnnVyd/d0=
|
||||
github.com/zclconf/go-cty v1.17.0/go.mod h1:wqFzcImaLTI6A5HfsRwB0nj5n0MRZFwmey8YoFPPs3U=
|
||||
github.com/yuin/gopher-lua v1.1.2 h1:yF/FjE3hD65tBbt0VXLE13HWS9h34fdzJmrWRXwobGA=
|
||||
github.com/yuin/gopher-lua v1.1.2/go.mod h1:7aRmXIWl37SqRf0koeyylBEzJ+aPt8A+mmkQ4f1ntR8=
|
||||
github.com/zclconf/go-cty v1.18.0 h1:pJ8+HNI4gFoyRNqVE37wWbJWVw43BZczFo7KUoRczaA=
|
||||
github.com/zclconf/go-cty v1.18.0/go.mod h1:qpnV6EDNgC1sns/AleL1fvatHw72j+S+nS+MJ+T2CSg=
|
||||
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo=
|
||||
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||
golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI=
|
||||
golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg=
|
||||
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
|
||||
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
|
||||
golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
|
||||
golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.4 h1:UP4+v6fFrBIb1l934bDl//mmnoIZEDK0idg1+AIvX5U=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.4/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||
golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
|
||||
golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
|
||||
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
|
||||
golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
|
||||
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
|
||||
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
|
||||
golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
|
||||
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
|
||||
golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA=
|
||||
golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc=
|
||||
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
|
||||
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
|
||||
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
|
||||
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
|
||||
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW8s2qTSe3wGBtvo0MbVQG/c5k8RE=
|
||||
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
@@ -54,3 +54,25 @@ func TestAllAtOnceEvaluateNodes(t *testing.T) {
|
||||
test.AssertResultComplex(t, tt.expected, resultsToString(t, list))
|
||||
}
|
||||
}
|
||||
|
||||
func TestTomlDecoderCanBeReinitializedAcrossDocuments(t *testing.T) {
|
||||
decoder := NewTomlDecoder()
|
||||
|
||||
firstDocuments, err := ReadDocuments(strings.NewReader("id = \"Foobar\"\n"), decoder)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read first TOML document: %v", err)
|
||||
}
|
||||
if firstDocuments.Len() != 1 {
|
||||
t.Fatalf("expected first document count to be 1, got %d", firstDocuments.Len())
|
||||
}
|
||||
test.AssertResult(t, "Foobar", firstDocuments.Front().Value.(*CandidateNode).Content[1].Value)
|
||||
|
||||
secondDocuments, err := ReadDocuments(strings.NewReader("id = \"Banana\"\n"), decoder)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read second TOML document: %v", err)
|
||||
}
|
||||
if secondDocuments.Len() != 1 {
|
||||
t.Fatalf("expected second document count to be 1, got %d", secondDocuments.Len())
|
||||
}
|
||||
test.AssertResult(t, "Banana", secondDocuments.Front().Value.(*CandidateNode).Content[1].Value)
|
||||
}
|
||||
|
||||
+26
-10
@@ -27,6 +27,22 @@ const (
|
||||
FlowStyle
|
||||
)
|
||||
|
||||
// EncodeHint controls how a mapping node is serialised by format-specific encoders
|
||||
// that distinguish between inline and block/section representations (e.g. TOML, HCL).
|
||||
type EncodeHint int
|
||||
|
||||
const (
|
||||
// EncodeHintDefault lets the encoder choose the representation (e.g. TOML block
|
||||
// mappings default to [section] headers).
|
||||
EncodeHintDefault EncodeHint = iota
|
||||
// EncodeHintSeparateBlock forces the node to be emitted as a separate block or
|
||||
// table-section header (used by TOML [section] and HCL block decoders).
|
||||
EncodeHintSeparateBlock
|
||||
// EncodeHintInline forces the node to be emitted as an inline / flow table
|
||||
// (used by TOML inline-table decoder and TOML encoder).
|
||||
EncodeHintInline
|
||||
)
|
||||
|
||||
func createStringScalarNode(stringValue string) *CandidateNode {
|
||||
var node = &CandidateNode{Kind: ScalarNode}
|
||||
node.Value = stringValue
|
||||
@@ -97,9 +113,9 @@ type CandidateNode struct {
|
||||
// (e.g. top level cross document merge). This property does not propagate to child nodes.
|
||||
EvaluateTogether bool
|
||||
IsMapKey bool
|
||||
// For formats like HCL and TOML: indicates that child entries should be emitted as separate blocks/tables
|
||||
// rather than consolidated into nested mappings (default behaviour)
|
||||
EncodeSeparate bool
|
||||
// EncodeHint controls how a mapping node is serialised by format-specific encoders
|
||||
// (e.g. TOML, HCL) that support both inline and block/section representations.
|
||||
EncodeHint EncodeHint
|
||||
}
|
||||
|
||||
func (n *CandidateNode) CreateChild() *CandidateNode {
|
||||
@@ -280,7 +296,7 @@ func (n *CandidateNode) AddChild(rawChild *CandidateNode) {
|
||||
|
||||
func (n *CandidateNode) AddChildren(children []*CandidateNode) {
|
||||
if n.Kind == MappingNode {
|
||||
for i := 0; i < len(children); i += 2 {
|
||||
for i := 0; i < len(children)-1; i += 2 {
|
||||
key := children[i]
|
||||
value := children[i+1]
|
||||
n.AddKeyValueChild(key, value)
|
||||
@@ -323,11 +339,11 @@ func (n *CandidateNode) guessTagFromCustomType() string {
|
||||
dataBucket, errorReading := parseSnippet(n.Value)
|
||||
|
||||
if errorReading != nil {
|
||||
log.Debug("guessTagFromCustomType: could not guess underlying tag type %v", errorReading)
|
||||
log.Debugf("guessTagFromCustomType: could not guess underlying tag type %v", errorReading)
|
||||
return n.Tag
|
||||
}
|
||||
guessedTag := dataBucket.Tag
|
||||
log.Info("im guessing the tag %v is a %v", n.Tag, guessedTag)
|
||||
log.Infof("im guessing the tag %v is a %v", n.Tag, guessedTag)
|
||||
return guessedTag
|
||||
}
|
||||
|
||||
@@ -411,7 +427,7 @@ func (n *CandidateNode) doCopy(cloneContent bool) *CandidateNode {
|
||||
EvaluateTogether: n.EvaluateTogether,
|
||||
IsMapKey: n.IsMapKey,
|
||||
|
||||
EncodeSeparate: n.EncodeSeparate,
|
||||
EncodeHint: n.EncodeHint,
|
||||
}
|
||||
|
||||
if cloneContent {
|
||||
@@ -445,7 +461,7 @@ func (n *CandidateNode) UpdateFrom(other *CandidateNode, prefs assignPreferences
|
||||
}
|
||||
|
||||
func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignPreferences) {
|
||||
log.Debug("UpdateAttributesFrom: n: %v other: %v", NodeToString(n), NodeToString(other))
|
||||
log.Debugf("UpdateAttributesFrom: n: %v other: %v", NodeToString(n), NodeToString(other))
|
||||
if n.Kind != other.Kind {
|
||||
// clear out the contents when switching to a different type
|
||||
// e.g. map to array
|
||||
@@ -465,8 +481,8 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignP
|
||||
n.Anchor = other.Anchor
|
||||
}
|
||||
|
||||
// Preserve EncodeSeparate flag for format-specific encoding hints
|
||||
n.EncodeSeparate = other.EncodeSeparate
|
||||
// Preserve EncodeHint for format-specific encoding hints
|
||||
n.EncodeHint = other.EncodeHint
|
||||
|
||||
// merge will pickup the style of the new thing
|
||||
// when autocreating nodes
|
||||
|
||||
@@ -36,13 +36,13 @@ func (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap, an
|
||||
switch commentMapComment.Position {
|
||||
case yaml.CommentHeadPosition:
|
||||
o.HeadComment = comment.String()
|
||||
log.Debug("its a head comment %v", comment.String())
|
||||
log.Debugf("its a head comment %v", comment.String())
|
||||
case yaml.CommentLinePosition:
|
||||
o.LineComment = comment.String()
|
||||
log.Debug("its a line comment %v", comment.String())
|
||||
log.Debugf("its a line comment %v", comment.String())
|
||||
case yaml.CommentFootPosition:
|
||||
o.FootComment = comment.String()
|
||||
log.Debug("its a foot comment %v", comment.String())
|
||||
log.Debugf("its a foot comment %v", comment.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -93,8 +93,8 @@ func (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap, an
|
||||
log.Debugf("folded Type %v", astLiteral.Start.Type)
|
||||
o.Style = FoldedStyle
|
||||
}
|
||||
log.Debug("start value: %v ", node.(*ast.LiteralNode).Start.Value)
|
||||
log.Debug("start value: %v ", node.(*ast.LiteralNode).Start.Type)
|
||||
log.Debugf("start value: %v ", node.(*ast.LiteralNode).Start.Value)
|
||||
log.Debugf("start value: %v ", node.(*ast.LiteralNode).Start.Type)
|
||||
// TODO: here I could put the original value with line breaks
|
||||
// to solve the multiline > problem
|
||||
o.Value = astLiteral.Value.Value
|
||||
@@ -187,7 +187,7 @@ func (o *CandidateNode) UnmarshalGoccyYAML(node ast.Node, cm yaml.CommentMap, an
|
||||
}
|
||||
|
||||
func (o *CandidateNode) goccyProcessMappingValueNode(mappingEntry *ast.MappingValueNode, cm yaml.CommentMap, anchorMap map[string]*CandidateNode) error {
|
||||
log.Debug("UnmarshalYAML MAP KEY entry %v", mappingEntry.Key)
|
||||
log.Debugf("UnmarshalYAML MAP KEY entry %v", mappingEntry.Key)
|
||||
|
||||
// AddKeyValueFirst because it clones the nodes, and we want to have the real refs when Unmarshalling
|
||||
// particularly for the anchorMap
|
||||
@@ -197,7 +197,7 @@ func (o *CandidateNode) goccyProcessMappingValueNode(mappingEntry *ast.MappingVa
|
||||
return err
|
||||
}
|
||||
|
||||
log.Debug("UnmarshalYAML MAP VALUE entry %v", mappingEntry.Value)
|
||||
log.Debugf("UnmarshalYAML MAP VALUE entry %v", mappingEntry.Value)
|
||||
if err := valueNode.UnmarshalGoccyYAML(mappingEntry.Value, cm, anchorMap); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -120,7 +120,7 @@ func (o *CandidateNode) UnmarshalJSON(data []byte) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Debug("UnmarshalJSON - scalar is %v", scalar)
|
||||
log.Debugf("UnmarshalJSON - scalar is %v", scalar)
|
||||
|
||||
return o.setScalarFromJson(scalar)
|
||||
|
||||
@@ -55,13 +55,13 @@ func (o *CandidateNode) copyFromYamlNode(node *yaml.Node, anchorMap map[string]*
|
||||
|
||||
if o.Anchor != "" {
|
||||
anchorMap[o.Anchor] = o
|
||||
log.Debug("set anchor %v to %v", o.Anchor, NodeToString(o))
|
||||
log.Debugf("set anchor %v to %v", o.Anchor, NodeToString(o))
|
||||
}
|
||||
|
||||
// its a single alias
|
||||
if node.Alias != nil && node.Alias.Anchor != "" {
|
||||
o.Alias = anchorMap[node.Alias.Anchor]
|
||||
log.Debug("set alias to %v", NodeToString(anchorMap[node.Alias.Anchor]))
|
||||
log.Debugf("set alias to %v", NodeToString(anchorMap[node.Alias.Anchor]))
|
||||
}
|
||||
o.HeadComment = node.HeadComment
|
||||
o.LineComment = node.LineComment
|
||||
@@ -106,7 +106,7 @@ func (o *CandidateNode) UnmarshalYAML(node *yaml.Node, anchorMap map[string]*Can
|
||||
log.Debugf("UnmarshalYAML %v", node.Tag)
|
||||
switch node.Kind {
|
||||
case yaml.AliasNode:
|
||||
log.Debug("UnmarshalYAML - alias from yaml: %v", o.Tag)
|
||||
log.Debugf("UnmarshalYAML - alias from yaml: %v", o.Tag)
|
||||
o.Kind = AliasNode
|
||||
o.copyFromYamlNode(node, anchorMap)
|
||||
return nil
|
||||
@@ -176,15 +176,15 @@ func (o *CandidateNode) UnmarshalYAML(node *yaml.Node, anchorMap map[string]*Can
|
||||
}
|
||||
|
||||
func (o *CandidateNode) MarshalYAML() (*yaml.Node, error) {
|
||||
log.Debug("MarshalYAML to yaml: %v", o.Tag)
|
||||
log.Debugf("MarshalYAML to yaml: %v", o.Tag)
|
||||
switch o.Kind {
|
||||
case AliasNode:
|
||||
log.Debug("MarshalYAML - alias to yaml: %v", o.Tag)
|
||||
log.Debugf("MarshalYAML - alias to yaml: %v", o.Tag)
|
||||
target := &yaml.Node{Kind: yaml.AliasNode}
|
||||
o.copyToYamlNode(target)
|
||||
return target, nil
|
||||
case ScalarNode:
|
||||
log.Debug("MarshalYAML - scalar: %v", o.Value)
|
||||
log.Debugf("MarshalYAML - scalar: %v", o.Value)
|
||||
target := &yaml.Node{Kind: yaml.ScalarNode}
|
||||
o.copyToYamlNode(target)
|
||||
return target, nil
|
||||
|
||||
@@ -18,7 +18,7 @@ func changeOwner(info fs.FileInfo, file *os.File) error {
|
||||
// this happens with snap confinement
|
||||
// not really a big issue as users can chown
|
||||
// the file themselves if required.
|
||||
log.Info("Skipping chown: %v", err)
|
||||
log.Infof("Skipping chown: %v", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
@@ -3,9 +3,8 @@ package yqlib
|
||||
import (
|
||||
"container/list"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
logging "gopkg.in/op/go-logging.v1"
|
||||
)
|
||||
|
||||
type Context struct {
|
||||
@@ -75,7 +74,7 @@ func (n *Context) ChildContext(results *list.List) Context {
|
||||
}
|
||||
|
||||
func (n *Context) ToString() string {
|
||||
if !log.IsEnabledFor(logging.DEBUG) {
|
||||
if !log.IsEnabledFor(slog.LevelDebug) {
|
||||
return ""
|
||||
}
|
||||
result := fmt.Sprintf("Context\nDontAutoCreate: %v\n", n.DontAutoCreate)
|
||||
|
||||
@@ -2,11 +2,11 @@ package yqlib
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/mikefarah/yq/v4/test"
|
||||
logging "gopkg.in/op/go-logging.v1"
|
||||
)
|
||||
|
||||
func TestChildContext(t *testing.T) {
|
||||
@@ -155,8 +155,8 @@ func TestToString(t *testing.T) {
|
||||
test.AssertResultComplex(t, "", result)
|
||||
|
||||
// Test with debug logging enabled
|
||||
logging.SetLevel(logging.DEBUG, "")
|
||||
defer logging.SetLevel(logging.INFO, "") // Reset to default
|
||||
GetLogger().SetLevel(slog.LevelDebug)
|
||||
defer GetLogger().SetLevel(slog.LevelWarn) // Reset to default
|
||||
|
||||
result2 := context.ToString()
|
||||
test.AssertResultComplex(t, true, len(result2) > 0)
|
||||
|
||||
@@ -2,8 +2,7 @@ package yqlib
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
logging "gopkg.in/op/go-logging.v1"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
type DataTreeNavigator interface {
|
||||
@@ -55,7 +54,7 @@ func (d *dataTreeNavigator) GetMatchingNodes(context Context, expressionNode *Ex
|
||||
return context, nil
|
||||
}
|
||||
log.Debugf("Processing Op: %v", expressionNode.Operation.toString())
|
||||
if log.IsEnabledFor(logging.DEBUG) {
|
||||
if log.IsEnabledFor(slog.LevelDebug) {
|
||||
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
|
||||
log.Debug(NodeToString(el.Value.(*CandidateNode)))
|
||||
}
|
||||
|
||||
@@ -226,7 +226,7 @@ func addBlockToMapping(parent *CandidateNode, block *hclsyntax.Block, src []byte
|
||||
// Mark the type node if there are multiple blocks of this type at this level
|
||||
// This tells the encoder to emit them as separate blocks rather than consolidating them
|
||||
if isMultipleBlocksOfType {
|
||||
typeNode.EncodeSeparate = true
|
||||
typeNode.EncodeHint = EncodeHintSeparateBlock
|
||||
}
|
||||
}
|
||||
current = typeNode
|
||||
|
||||
@@ -68,7 +68,7 @@ func mustProcessFormatScenario(s formatScenario, decoder Decoder, encoder Encode
|
||||
|
||||
result, err := processFormatScenario(s, decoder, encoder)
|
||||
if err != nil {
|
||||
log.Error("Bad scenario %v: %w", s.description, err)
|
||||
log.Errorf("Bad scenario %v: %v", s.description, err)
|
||||
return fmt.Sprintf("Bad scenario %v: %v", s.description, err.Error())
|
||||
}
|
||||
return result
|
||||
|
||||
+20
-18
@@ -44,6 +44,7 @@ func (dec *tomlDecoder) Init(reader io.Reader) error {
|
||||
}
|
||||
dec.pendingComments = make([]string, 0)
|
||||
dec.firstContentSeen = false
|
||||
dec.finished = false
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -105,7 +106,7 @@ func (dec *tomlDecoder) decodeKeyValuesIntoMap(rootMap *CandidateNode, tomlNode
|
||||
|
||||
for dec.parser.NextExpression() {
|
||||
nextItem := dec.parser.Expression()
|
||||
log.Debug("decodeKeyValuesIntoMap -- next exp, its a %v", nextItem.Kind)
|
||||
log.Debugf("decodeKeyValuesIntoMap -- next exp, its a %v", nextItem.Kind)
|
||||
|
||||
switch nextItem.Kind {
|
||||
case toml.KeyValue:
|
||||
@@ -117,7 +118,7 @@ func (dec *tomlDecoder) decodeKeyValuesIntoMap(rootMap *CandidateNode, tomlNode
|
||||
dec.pendingComments = append(dec.pendingComments, string(nextItem.Data))
|
||||
default:
|
||||
// run out of key values
|
||||
log.Debug("done in decodeKeyValuesIntoMap, gota a %v", nextItem.Kind)
|
||||
log.Debugf("done in decodeKeyValuesIntoMap, gota a %v", nextItem.Kind)
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
@@ -149,9 +150,10 @@ func (dec *tomlDecoder) createInlineTableMap(tomlNode *toml.Node) (*CandidateNod
|
||||
}
|
||||
|
||||
return &CandidateNode{
|
||||
Kind: MappingNode,
|
||||
Tag: "!!map",
|
||||
Content: content,
|
||||
Kind: MappingNode,
|
||||
Tag: "!!map",
|
||||
EncodeHint: EncodeHintInline,
|
||||
Content: content,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -270,7 +272,7 @@ func (dec *tomlDecoder) Decode() (*CandidateNode, error) {
|
||||
|
||||
currentNode := dec.parser.Expression()
|
||||
|
||||
log.Debug("currentNode: %v ", currentNode.Kind)
|
||||
log.Debugf("currentNode: %v ", currentNode.Kind)
|
||||
runAgainstCurrentExp, err = dec.processTopLevelNode(currentNode)
|
||||
if err != nil {
|
||||
return dec.rootMap, err
|
||||
@@ -297,7 +299,7 @@ func (dec *tomlDecoder) Decode() (*CandidateNode, error) {
|
||||
func (dec *tomlDecoder) processTopLevelNode(currentNode *toml.Node) (bool, error) {
|
||||
var runAgainstCurrentExp bool
|
||||
var err error
|
||||
log.Debug("processTopLevelNode: Going to process %v state is current %v", currentNode.Kind, NodeToString(dec.rootMap))
|
||||
log.Debugf("processTopLevelNode: Going to process %v state is current %v", currentNode.Kind, NodeToString(dec.rootMap))
|
||||
switch currentNode.Kind {
|
||||
case toml.Comment:
|
||||
// Collect comment to attach to next element
|
||||
@@ -325,7 +327,7 @@ func (dec *tomlDecoder) processTopLevelNode(currentNode *toml.Node) (bool, error
|
||||
runAgainstCurrentExp, err = dec.decodeKeyValuesIntoMap(dec.rootMap, currentNode)
|
||||
}
|
||||
|
||||
log.Debug("processTopLevelNode: DONE Processing state is now %v", NodeToString(dec.rootMap))
|
||||
log.Debugf("processTopLevelNode: DONE Processing state is now %v", NodeToString(dec.rootMap))
|
||||
return runAgainstCurrentExp, err
|
||||
}
|
||||
|
||||
@@ -333,7 +335,7 @@ func (dec *tomlDecoder) processTable(currentNode *toml.Node) (bool, error) {
|
||||
log.Debug("Enter processTable")
|
||||
child := currentNode.Child()
|
||||
fullPath := dec.getFullPath(child)
|
||||
log.Debug("fullpath: %v", fullPath)
|
||||
log.Debugf("fullpath: %v", fullPath)
|
||||
|
||||
c := Context{}
|
||||
c = c.SingleChildContext(dec.rootMap)
|
||||
@@ -344,10 +346,10 @@ func (dec *tomlDecoder) processTable(currentNode *toml.Node) (bool, error) {
|
||||
}
|
||||
|
||||
tableNodeValue := &CandidateNode{
|
||||
Kind: MappingNode,
|
||||
Tag: "!!map",
|
||||
Content: make([]*CandidateNode, 0),
|
||||
EncodeSeparate: true,
|
||||
Kind: MappingNode,
|
||||
Tag: "!!map",
|
||||
Content: make([]*CandidateNode, 0),
|
||||
EncodeHint: EncodeHintSeparateBlock,
|
||||
}
|
||||
|
||||
// Attach pending head comments to the table
|
||||
@@ -401,7 +403,7 @@ func (dec *tomlDecoder) processTable(currentNode *toml.Node) (bool, error) {
|
||||
}
|
||||
|
||||
func (dec *tomlDecoder) arrayAppend(context Context, path []interface{}, rhsNode *CandidateNode) error {
|
||||
log.Debug("arrayAppend to path: %v,%v", path, NodeToString(rhsNode))
|
||||
log.Debugf("arrayAppend to path: %v,%v", path, NodeToString(rhsNode))
|
||||
rhsCandidateNode := &CandidateNode{
|
||||
Kind: SequenceNode,
|
||||
Tag: "!!seq",
|
||||
@@ -426,7 +428,7 @@ func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error)
|
||||
log.Debug("Enter processArrayTable")
|
||||
child := currentNode.Child()
|
||||
fullPath := dec.getFullPath(child)
|
||||
log.Debug("Fullpath: %v", fullPath)
|
||||
log.Debugf("Fullpath: %v", fullPath)
|
||||
|
||||
c := Context{}
|
||||
c = c.SingleChildContext(dec.rootMap)
|
||||
@@ -441,9 +443,9 @@ func (dec *tomlDecoder) processArrayTable(currentNode *toml.Node) (bool, error)
|
||||
hasValue := dec.parser.NextExpression()
|
||||
|
||||
tableNodeValue := &CandidateNode{
|
||||
Kind: MappingNode,
|
||||
Tag: "!!map",
|
||||
EncodeSeparate: true,
|
||||
Kind: MappingNode,
|
||||
Tag: "!!map",
|
||||
EncodeHint: EncodeHintSeparateBlock,
|
||||
}
|
||||
|
||||
// Attach pending head comments to the array table
|
||||
|
||||
+15
-15
@@ -64,7 +64,7 @@ func (dec *xmlDecoder) processComment(c string) string {
|
||||
}
|
||||
|
||||
func (dec *xmlDecoder) createMap(n *xmlNode) (*CandidateNode, error) {
|
||||
log.Debug("createMap: headC: %v, lineC: %v, footC: %v", n.HeadComment, n.LineComment, n.FootComment)
|
||||
log.Debugf("createMap: headC: %v, lineC: %v, footC: %v", n.HeadComment, n.LineComment, n.FootComment)
|
||||
yamlNode := &CandidateNode{Kind: MappingNode, Tag: "!!map"}
|
||||
|
||||
if len(n.Data) > 0 {
|
||||
@@ -92,7 +92,7 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*CandidateNode, error) {
|
||||
log.Debugf("label=%v, i=%v, keyValuePair.FootComment: %v", label, i, keyValuePair.FootComment)
|
||||
labelNode.FootComment = dec.processComment(keyValuePair.FootComment)
|
||||
|
||||
log.Debug("len of children in %v is %v", label, len(children))
|
||||
log.Debugf("len of children in %v is %v", label, len(children))
|
||||
if len(children) > 1 {
|
||||
valueNode, err = dec.createSequence(children)
|
||||
if err != nil {
|
||||
@@ -105,7 +105,7 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*CandidateNode, error) {
|
||||
if len(children[0].Children) == 0 && children[0].HeadComment != "" {
|
||||
if len(children[0].Data) > 0 {
|
||||
|
||||
log.Debug("scalar comment hack, currentlabel [%v]", labelNode.HeadComment)
|
||||
log.Debugf("scalar comment hack, currentlabel [%v]", labelNode.HeadComment)
|
||||
labelNode.HeadComment = joinComments([]string{labelNode.HeadComment, strings.TrimSpace(children[0].HeadComment)}, "\n")
|
||||
children[0].HeadComment = ""
|
||||
} else {
|
||||
@@ -151,7 +151,7 @@ func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*CandidateNode, error) {
|
||||
|
||||
scalar := dec.createValueNodeFromData(n.Data)
|
||||
|
||||
log.Debug("scalar (%v), headC: %v, lineC: %v, footC: %v", scalar.Tag, n.HeadComment, n.LineComment, n.FootComment)
|
||||
log.Debugf("scalar (%v), headC: %v, lineC: %v, footC: %v", scalar.Tag, n.HeadComment, n.LineComment, n.FootComment)
|
||||
scalar.HeadComment = dec.processComment(n.HeadComment)
|
||||
scalar.LineComment = dec.processComment(n.LineComment)
|
||||
if scalar.Tag == "!!seq" {
|
||||
@@ -211,17 +211,17 @@ func (n *xmlNode) AddChild(s string, c *xmlNode) {
|
||||
if n.Children == nil {
|
||||
n.Children = make([]*xmlChildrenKv, 0)
|
||||
}
|
||||
log.Debug("looking for %s", s)
|
||||
log.Debugf("looking for %s", s)
|
||||
// see if we can find an existing entry to add to
|
||||
for _, childEntry := range n.Children {
|
||||
if childEntry.K == s {
|
||||
log.Debug("found it, appending an entry%s", s)
|
||||
log.Debugf("found it, appending an entry%s", s)
|
||||
childEntry.V = append(childEntry.V, c)
|
||||
log.Debug("yay len of children in %v is %v", s, len(childEntry.V))
|
||||
log.Debugf("yay len of children in %v is %v", s, len(childEntry.V))
|
||||
return
|
||||
}
|
||||
}
|
||||
log.Debug("not there, making a new one %s", s)
|
||||
log.Debugf("not there, making a new one %s", s)
|
||||
n.Children = append(n.Children, &xmlChildrenKv{K: s, V: []*xmlNode{c}})
|
||||
}
|
||||
|
||||
@@ -267,7 +267,7 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
|
||||
|
||||
switch se := t.(type) {
|
||||
case xml.StartElement:
|
||||
log.Debug("start element %v", se.Name.Local)
|
||||
log.Debugf("start element %v", se.Name.Local)
|
||||
elem.state = "started"
|
||||
// Build new a new current element and link it to its parent
|
||||
var label = se.Name.Local
|
||||
@@ -302,14 +302,14 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
|
||||
if len(newBit) > 0 {
|
||||
elem.n.Data = append(elem.n.Data, newBit)
|
||||
elem.state = "chardata"
|
||||
log.Debug("chardata [%v] for %v", elem.n.Data, elem.label)
|
||||
log.Debugf("chardata [%v] for %v", elem.n.Data, elem.label)
|
||||
}
|
||||
case xml.EndElement:
|
||||
if elem == nil {
|
||||
log.Debug("no element, probably bad xml")
|
||||
continue
|
||||
}
|
||||
log.Debug("end element %v", elem.label)
|
||||
log.Debugf("end element %v", elem.label)
|
||||
elem.state = "finished"
|
||||
// And add it to its parent list
|
||||
if elem.parent != nil {
|
||||
@@ -326,10 +326,10 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
|
||||
applyFootComment(elem, commentStr)
|
||||
|
||||
case "chardata":
|
||||
log.Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
|
||||
log.Debugf("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
|
||||
elem.n.LineComment = joinComments([]string{elem.n.LineComment, commentStr}, " ")
|
||||
default:
|
||||
log.Debug("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
|
||||
log.Debugf("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
|
||||
elem.n.HeadComment = joinComments([]string{elem.n.HeadComment, commentStr}, " ")
|
||||
}
|
||||
|
||||
@@ -354,7 +354,7 @@ func applyFootComment(elem *element, commentStr string) {
|
||||
if len(elem.n.Children) > 0 {
|
||||
lastChildIndex := len(elem.n.Children) - 1
|
||||
childKv := elem.n.Children[lastChildIndex]
|
||||
log.Debug("got a foot comment, putting on last child for %v: [%v]", childKv.K, commentStr)
|
||||
log.Debugf("got a foot comment, putting on last child for %v: [%v]", childKv.K, commentStr)
|
||||
// if it's an array of scalars, put the foot comment on the scalar itself
|
||||
if len(childKv.V) > 0 && len(childKv.V[0].Children) == 0 {
|
||||
nodeToUpdate := childKv.V[len(childKv.V)-1]
|
||||
@@ -363,7 +363,7 @@ func applyFootComment(elem *element, commentStr string) {
|
||||
childKv.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ")
|
||||
}
|
||||
} else {
|
||||
log.Debug("got a foot comment for %v: [%v]", elem.label, commentStr)
|
||||
log.Debugf("got a foot comment for %v: [%v]", elem.label, commentStr)
|
||||
elem.n.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# Slice/Splice Array
|
||||
# Slice Array or String
|
||||
|
||||
The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.
|
||||
The slice operator works on both arrays and strings. Like the `jq` equivalent, `.[10:15]` will return a subarray (or substring) of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array or string.
|
||||
|
||||
You may leave out the first or second number, which will refer to the start or end of the array respectively.
|
||||
You may leave out the first or second number, which will refer to the start or end of the array or string respectively.
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
# System Operators
|
||||
|
||||
The `system` operator allows you to run an external command and use its output as a value in your expression.
|
||||
|
||||
**Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it.
|
||||
|
||||
**Note:** When enabled, the system operator can replicate the functionality of `env` and `load`
|
||||
operators via external commands. Enabling it effectively overrides `--security-disable-env-ops`
|
||||
and `--security-disable-file-ops`.
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
yq --security-enable-system-operator --null-input '.field = system("command"; "arg1")'
|
||||
```
|
||||
|
||||
The operator takes:
|
||||
- A command string (required)
|
||||
- An argument (or an array of arguments), separated from the command by `;` (optional)
|
||||
|
||||
The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string.
|
||||
|
||||
## Disabling the system operator
|
||||
|
||||
The system operator is disabled by default. When disabled, an error is returned instead of running the command, consistent with `--security-disable-env-ops` and `--security-disable-file-ops`.
|
||||
|
||||
Use `--security-enable-system-operator` flag to enable it.
|
||||
@@ -1,8 +1,8 @@
|
||||
# Slice/Splice Array
|
||||
# Slice Array or String
|
||||
|
||||
The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.
|
||||
The slice operator works on both arrays and strings. Like the `jq` equivalent, `.[10:15]` will return a subarray (or substring) of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array or string.
|
||||
|
||||
You may leave out the first or second number, which will refer to the start or end of the array respectively.
|
||||
You may leave out the first or second number, which will refer to the start or end of the array or string respectively.
|
||||
|
||||
## Slicing arrays
|
||||
Given a sample.yml file of:
|
||||
@@ -103,3 +103,81 @@ will output
|
||||
- cow
|
||||
```
|
||||
|
||||
## Slicing strings
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
country: Australia
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.country[0:5]' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
Austr
|
||||
```
|
||||
|
||||
## Slicing strings - without the second number
|
||||
Finishes at the end of the string
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
country: Australia
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.country[5:]' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
alia
|
||||
```
|
||||
|
||||
## Slicing strings - without the first number
|
||||
Starts from the start of the string
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
country: Australia
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.country[:5]' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
Austr
|
||||
```
|
||||
|
||||
## Slicing strings - use negative numbers to count backwards from the end
|
||||
Negative indices count from the end of the string
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
country: Australia
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.country[-5:]' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
ralia
|
||||
```
|
||||
|
||||
## Slicing strings - Unicode
|
||||
Indices are rune-based, so multi-byte characters are handled correctly
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
greeting: héllo
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.greeting[1:3]' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
él
|
||||
```
|
||||
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
# System Operators
|
||||
|
||||
The `system` operator allows you to run an external command and use its output as a value in your expression.
|
||||
|
||||
**Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it.
|
||||
|
||||
**Note:** When enabled, the system operator can replicate the functionality of `env` and `load`
|
||||
operators via external commands. Enabling it effectively overrides `--security-disable-env-ops`
|
||||
and `--security-disable-file-ops`.
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
yq --security-enable-system-operator --null-input '.field = system("command"; "arg1")'
|
||||
```
|
||||
|
||||
The operator takes:
|
||||
- A command string (required)
|
||||
- An argument (or an array of arguments), separated from the command by `;` (optional)
|
||||
|
||||
The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string.
|
||||
|
||||
## Disabling the system operator
|
||||
|
||||
The system operator is disabled by default. When disabled, an error is returned instead of running the command, consistent with `--security-disable-env-ops` and `--security-disable-file-ops`.
|
||||
|
||||
Use `--security-enable-system-operator` flag to enable it.
|
||||
|
||||
## system operator returns error when disabled
|
||||
Use `--security-enable-system-operator` to enable the system operator.
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
country: Australia
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.country = system("/usr/bin/echo"; "test")' sample.yml
|
||||
```
|
||||
will output
|
||||
```bash
|
||||
Error: system operations are disabled, use --security-enable-system-operator to enable
|
||||
```
|
||||
|
||||
## Run a command with an argument
|
||||
Use `--security-enable-system-operator` to enable the system operator.
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
country: Australia
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq --security-enable-system-operator '.country = system("/usr/bin/echo"; "test")' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
country: test
|
||||
```
|
||||
|
||||
## Run a command without arguments
|
||||
Omit the semicolon and args to run the command with no extra arguments.
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
a: hello
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq --security-enable-system-operator '.a = system("/usr/bin/echo")' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
a: ""
|
||||
```
|
||||
|
||||
@@ -384,3 +384,20 @@ ip = "10.0.0.2"
|
||||
role = "backend"
|
||||
```
|
||||
|
||||
## Encode: Simple mapping produces table section
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
arg:
|
||||
hello: foo
|
||||
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq -o toml '.' sample.yml
|
||||
```
|
||||
will output
|
||||
```toml
|
||||
[arg]
|
||||
hello = "foo"
|
||||
```
|
||||
|
||||
|
||||
@@ -449,8 +449,8 @@ func (he *hclEncoder) encodeBlockIfMapping(body *hclwrite.Body, key string, valu
|
||||
return false
|
||||
}
|
||||
|
||||
// If EncodeSeparate is set, emit children as separate blocks regardless of label extraction
|
||||
if valueNode.EncodeSeparate {
|
||||
// If EncodeHintSeparateBlock is set, emit children as separate blocks regardless of label extraction
|
||||
if valueNode.EncodeHint == EncodeHintSeparateBlock {
|
||||
if handled, _ := he.encodeMappingChildrenAsBlocks(body, key, valueNode); handled {
|
||||
return true
|
||||
}
|
||||
@@ -537,9 +537,9 @@ func (he *hclEncoder) encodeMappingChildrenAsBlocks(body *hclwrite.Body, blockTy
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// Only emit as separate blocks if EncodeSeparate is true
|
||||
// Only emit as separate blocks if EncodeHintSeparateBlock is set
|
||||
// This allows the encoder to respect the original block structure preserved by the decoder
|
||||
if !valueNode.EncodeSeparate {
|
||||
if valueNode.EncodeHint != EncodeHintSeparateBlock {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ func (ke *kyamlEncoder) PrintLeadingContent(writer io.Writer, content string) er
|
||||
}
|
||||
|
||||
func (ke *kyamlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
|
||||
log.Debug("encoderKYaml - going to print %v", NodeToString(node))
|
||||
log.Debugf("encoderKYaml - going to print %v", NodeToString(node))
|
||||
if node.Kind == ScalarNode && ke.prefs.UnwrapScalar {
|
||||
return writeString(writer, node.Value+"\n")
|
||||
}
|
||||
|
||||
+57
-62
@@ -69,6 +69,27 @@ func (te *tomlEncoder) CanHandleAliases() bool {
|
||||
|
||||
// ---- helpers ----
|
||||
|
||||
// tomlKey returns the key quoted if it contains characters that are not valid
|
||||
// in a TOML bare key. TOML bare keys may only contain ASCII letters, ASCII
|
||||
// digits, underscores, and dashes.
|
||||
func tomlKey(key string) string {
|
||||
for _, r := range key {
|
||||
if (r < 'A' || r > 'Z') && (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '_' && r != '-' {
|
||||
return fmt.Sprintf("%q", key)
|
||||
}
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// tomlDottedKey joins path components, quoting any that require it.
|
||||
func tomlDottedKey(path []string) string {
|
||||
parts := make([]string, len(path))
|
||||
for i, p := range path {
|
||||
parts[i] = tomlKey(p)
|
||||
}
|
||||
return strings.Join(parts, ".")
|
||||
}
|
||||
|
||||
func (te *tomlEncoder) writeComment(w io.Writer, comment string) error {
|
||||
if comment == "" {
|
||||
return nil
|
||||
@@ -148,9 +169,10 @@ func (te *tomlEncoder) encodeTopLevelEntry(w io.Writer, path []string, node *Can
|
||||
}
|
||||
if allMaps {
|
||||
key := path[len(path)-1]
|
||||
quotedKey := tomlKey(key)
|
||||
for _, it := range node.Content {
|
||||
// [[key]] then body
|
||||
if _, err := w.Write([]byte("[[" + key + "]]\n")); err != nil {
|
||||
if _, err := w.Write([]byte("[[" + quotedKey + "]]\n")); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := te.encodeMappingBodyWithPath(w, []string{key}, it); err != nil {
|
||||
@@ -162,12 +184,9 @@ func (te *tomlEncoder) encodeTopLevelEntry(w io.Writer, path []string, node *Can
|
||||
// Regular array attribute
|
||||
return te.writeArrayAttribute(w, path[len(path)-1], node)
|
||||
case MappingNode:
|
||||
// Inline table if not EncodeSeparate, else emit separate tables/arrays of tables for children under this path
|
||||
if !node.EncodeSeparate {
|
||||
// If children contain mappings or arrays of mappings, prefer separate sections
|
||||
if te.hasEncodeSeparateChild(node) || te.hasStructuralChildren(node) {
|
||||
return te.encodeSeparateMapping(w, path, node)
|
||||
}
|
||||
// Use inline table syntax for nodes explicitly marked as TOML inline tables
|
||||
// or YAML flow mappings. All other mappings become readable TOML table sections.
|
||||
if node.EncodeHint == EncodeHintInline || node.Style&FlowStyle != 0 {
|
||||
return te.writeInlineTableAttribute(w, path[len(path)-1], node)
|
||||
}
|
||||
return te.encodeSeparateMapping(w, path, node)
|
||||
@@ -185,7 +204,7 @@ func (te *tomlEncoder) writeAttribute(w io.Writer, key string, value *CandidateN
|
||||
}
|
||||
|
||||
// Write the attribute
|
||||
line := key + " = " + te.formatScalar(value)
|
||||
line := tomlKey(key) + " = " + te.formatScalar(value)
|
||||
|
||||
// Add line comment if present
|
||||
if value.LineComment != "" {
|
||||
@@ -210,7 +229,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida
|
||||
|
||||
// Handle empty arrays
|
||||
if len(seq.Content) == 0 {
|
||||
line := key + " = []"
|
||||
line := tomlKey(key) + " = []"
|
||||
if seq.LineComment != "" {
|
||||
lineComment := strings.TrimSpace(seq.LineComment)
|
||||
if !strings.HasPrefix(lineComment, "#") {
|
||||
@@ -233,7 +252,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida
|
||||
|
||||
if hasElementComments {
|
||||
// Write multiline array format with comments
|
||||
if _, err := w.Write([]byte(key + " = [\n")); err != nil {
|
||||
if _, err := w.Write([]byte(tomlKey(key) + " = [\n")); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -324,7 +343,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida
|
||||
}
|
||||
}
|
||||
|
||||
line := key + " = [" + strings.Join(items, ", ") + "]"
|
||||
line := tomlKey(key) + " = [" + strings.Join(items, ", ") + "]"
|
||||
|
||||
// Add line comment if present
|
||||
if seq.LineComment != "" {
|
||||
@@ -372,21 +391,21 @@ func (te *tomlEncoder) mappingToInlineTable(m *CandidateNode) (string, error) {
|
||||
v := m.Content[i+1]
|
||||
switch v.Kind {
|
||||
case ScalarNode:
|
||||
parts = append(parts, fmt.Sprintf("%s = %s", k, te.formatScalar(v)))
|
||||
parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), te.formatScalar(v)))
|
||||
case SequenceNode:
|
||||
// inline array in inline table
|
||||
arr, err := te.sequenceToInlineArray(v)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
parts = append(parts, fmt.Sprintf("%s = %s", k, arr))
|
||||
parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), arr))
|
||||
case MappingNode:
|
||||
// nested inline table
|
||||
inline, err := te.mappingToInlineTable(v)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
parts = append(parts, fmt.Sprintf("%s = %s", k, inline))
|
||||
parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), inline))
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported inline table value kind: %v", v.Kind)
|
||||
}
|
||||
@@ -399,7 +418,7 @@ func (te *tomlEncoder) writeInlineTableAttribute(w io.Writer, key string, m *Can
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = w.Write([]byte(key + " = " + inline + "\n"))
|
||||
_, err = w.Write([]byte(tomlKey(key) + " = " + inline + "\n"))
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -421,7 +440,7 @@ func (te *tomlEncoder) writeTableHeader(w io.Writer, path []string, m *Candidate
|
||||
}
|
||||
|
||||
// Write table header [a.b.c]
|
||||
header := "[" + strings.Join(path, ".") + "]\n"
|
||||
header := "[" + tomlDottedKey(path) + "]\n"
|
||||
_, err := w.Write([]byte(header))
|
||||
return err
|
||||
}
|
||||
@@ -429,7 +448,8 @@ func (te *tomlEncoder) writeTableHeader(w io.Writer, path []string, m *Candidate
|
||||
// encodeSeparateMapping handles a mapping that should be encoded as table sections.
|
||||
// It emits the table header for this mapping if it has any content, then processes children.
|
||||
func (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *CandidateNode) error {
|
||||
// Check if this mapping has any non-mapping, non-array-of-tables children (i.e., attributes)
|
||||
// Check if this mapping has any non-mapping, non-array-of-tables children (i.e., attributes).
|
||||
// Inline mapping children also count as attributes since they render as key = { ... }.
|
||||
hasAttrs := false
|
||||
for i := 0; i < len(m.Content); i += 2 {
|
||||
v := m.Content[i+1]
|
||||
@@ -437,6 +457,10 @@ func (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *Cand
|
||||
hasAttrs = true
|
||||
break
|
||||
}
|
||||
if v.Kind == MappingNode && (v.EncodeHint == EncodeHintInline || v.Style&FlowStyle != 0) {
|
||||
hasAttrs = true
|
||||
break
|
||||
}
|
||||
if v.Kind == SequenceNode {
|
||||
// Check if it's NOT an array of tables
|
||||
allMaps := true
|
||||
@@ -464,18 +488,14 @@ func (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *Cand
|
||||
return nil
|
||||
}
|
||||
|
||||
// No attributes, just nested structures - process children
|
||||
// No attributes, just nested table structures - process children recursively
|
||||
for i := 0; i < len(m.Content); i += 2 {
|
||||
k := m.Content[i].Value
|
||||
v := m.Content[i+1]
|
||||
switch v.Kind {
|
||||
case MappingNode:
|
||||
// Emit [path.k]
|
||||
newPath := append(append([]string{}, path...), k)
|
||||
if err := te.writeTableHeader(w, newPath, v); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := te.encodeMappingBodyWithPath(w, newPath, v); err != nil {
|
||||
if err := te.encodeSeparateMapping(w, newPath, v); err != nil {
|
||||
return err
|
||||
}
|
||||
case SequenceNode:
|
||||
@@ -488,7 +508,7 @@ func (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *Cand
|
||||
}
|
||||
}
|
||||
if allMaps {
|
||||
key := strings.Join(append(append([]string{}, path...), k), ".")
|
||||
key := tomlDottedKey(append(append([]string{}, path...), k))
|
||||
for _, it := range v.Content {
|
||||
if _, err := w.Write([]byte("[[" + key + "]]\n")); err != nil {
|
||||
return err
|
||||
@@ -513,39 +533,6 @@ func (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *Cand
|
||||
return nil
|
||||
}
|
||||
|
||||
func (te *tomlEncoder) hasEncodeSeparateChild(m *CandidateNode) bool {
|
||||
for i := 0; i < len(m.Content); i += 2 {
|
||||
v := m.Content[i+1]
|
||||
if v.Kind == MappingNode && v.EncodeSeparate {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (te *tomlEncoder) hasStructuralChildren(m *CandidateNode) bool {
|
||||
for i := 0; i < len(m.Content); i += 2 {
|
||||
v := m.Content[i+1]
|
||||
// Only consider it structural if mapping has EncodeSeparate or is non-empty
|
||||
if v.Kind == MappingNode && v.EncodeSeparate {
|
||||
return true
|
||||
}
|
||||
if v.Kind == SequenceNode {
|
||||
allMaps := true
|
||||
for _, it := range v.Content {
|
||||
if it.Kind != MappingNode {
|
||||
allMaps = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if allMaps {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// encodeMappingBodyWithPath encodes attributes and nested arrays of tables using full dotted path context
|
||||
func (te *tomlEncoder) encodeMappingBodyWithPath(w io.Writer, path []string, m *CandidateNode) error {
|
||||
// First, attributes (scalars and non-map arrays)
|
||||
@@ -586,7 +573,7 @@ func (te *tomlEncoder) encodeMappingBodyWithPath(w io.Writer, path []string, m *
|
||||
}
|
||||
}
|
||||
if allMaps {
|
||||
dotted := strings.Join(append(append([]string{}, path...), k), ".")
|
||||
dotted := tomlDottedKey(append(append([]string{}, path...), k))
|
||||
for _, it := range v.Content {
|
||||
if _, err := w.Write([]byte("[[" + dotted + "]]\n")); err != nil {
|
||||
return err
|
||||
@@ -599,13 +586,21 @@ func (te *tomlEncoder) encodeMappingBodyWithPath(w io.Writer, path []string, m *
|
||||
}
|
||||
}
|
||||
|
||||
// Finally, child mappings that are not marked EncodeSeparate get inlined as attributes
|
||||
// Finally, child mappings: inline-hint or flow-style ones become inline table attributes,
|
||||
// while all others are emitted as separate sub-table sections.
|
||||
for i := 0; i < len(m.Content); i += 2 {
|
||||
k := m.Content[i].Value
|
||||
v := m.Content[i+1]
|
||||
if v.Kind == MappingNode && !v.EncodeSeparate {
|
||||
if err := te.writeInlineTableAttribute(w, k, v); err != nil {
|
||||
return err
|
||||
if v.Kind == MappingNode {
|
||||
if v.EncodeHint == EncodeHintInline || v.Style&FlowStyle != 0 {
|
||||
if err := te.writeInlineTableAttribute(w, k, v); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
subPath := append(append([]string{}, path...), k)
|
||||
if err := te.encodeSeparateMapping(w, subPath, v); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,7 +59,7 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
|
||||
return err
|
||||
}
|
||||
if _, err := e.writer.Write([]byte("\n")); err != nil {
|
||||
log.Warning("Unable to write newline, skipping: %w", err)
|
||||
log.Warningf("Unable to write newline, skipping: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -131,7 +131,7 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *CandidateNode
|
||||
return err
|
||||
}
|
||||
if _, err := e.writer.Write([]byte("\n")); err != nil {
|
||||
log.Warning("Unable to write newline, skipping: %w", err)
|
||||
log.Warningf("Unable to write newline, skipping: %v", err)
|
||||
}
|
||||
} else if key.Value == e.prefs.DirectiveName {
|
||||
var directive xml.Directive = []byte(value.Value)
|
||||
@@ -139,7 +139,7 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *CandidateNode
|
||||
return err
|
||||
}
|
||||
if _, err := e.writer.Write([]byte("\n")); err != nil {
|
||||
log.Warning("Unable to write newline, skipping: %w", err)
|
||||
log.Warningf("Unable to write newline, skipping: %v", err)
|
||||
}
|
||||
} else {
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) err
|
||||
}
|
||||
|
||||
func (ye *yamlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
|
||||
log.Debug("encoderYaml - going to print %v", NodeToString(node))
|
||||
log.Debugf("encoderYaml - going to print %v", NodeToString(node))
|
||||
// Detect line ending style from LeadingContent
|
||||
lineEnding := "\n"
|
||||
if strings.Contains(node.LeadingContent, "\r\n") {
|
||||
|
||||
@@ -26,7 +26,7 @@ func newExpressionParser() ExpressionParserInterface {
|
||||
}
|
||||
|
||||
func (p *expressionParserImpl) ParseExpression(expression string) (*ExpressionNode, error) {
|
||||
log.Debug("Parsing expression: [%v]", expression)
|
||||
log.Debugf("Parsing expression: [%v]", expression)
|
||||
tokens, err := p.pathTokeniser.Tokenise(expression)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -95,6 +95,7 @@ func TestParserSingleOperation(t *testing.T) {
|
||||
test.AssertResultComplex(t, nil, err)
|
||||
if result == nil {
|
||||
t.Fatal("Expected non-nil result for single operation")
|
||||
return
|
||||
}
|
||||
if result.Operation == nil {
|
||||
t.Fatal("Expected operation to be set")
|
||||
|
||||
@@ -3,8 +3,7 @@ package yqlib
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
logging "gopkg.in/op/go-logging.v1"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
type expressionPostFixer interface {
|
||||
@@ -134,7 +133,7 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
|
||||
return nil, fmt.Errorf("bad expression - probably missing close bracket on %v", opStack[len(opStack)-1].toString(false))
|
||||
}
|
||||
|
||||
if log.IsEnabledFor(logging.DEBUG) {
|
||||
if log.IsEnabledFor(slog.LevelDebug) {
|
||||
log.Debugf("PostFix Result:")
|
||||
for _, currentToken := range result {
|
||||
log.Debugf("> %v", currentToken.toString())
|
||||
|
||||
@@ -30,7 +30,7 @@ func tryRenameFile(from string, to string) error {
|
||||
}
|
||||
|
||||
func tryRemoveTempFile(filename string) {
|
||||
log.Debug("Removing temp file: %v", filename)
|
||||
log.Debugf("Removing temp file: %v", filename)
|
||||
removeErr := os.Remove(filename)
|
||||
if removeErr != nil {
|
||||
log.Errorf("Failed to remove temp file: %v", filename)
|
||||
@@ -68,8 +68,7 @@ func SafelyCloseReader(reader io.Reader) {
|
||||
func safelyCloseFile(file *os.File) {
|
||||
err := file.Close()
|
||||
if err != nil {
|
||||
log.Error("Error closing file!")
|
||||
log.Error(err.Error())
|
||||
log.Errorf("Error closing file %v: %v", file.Name(), err)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -58,7 +58,7 @@ func (f *frontMatterHandlerImpl) Split() error {
|
||||
return err
|
||||
}
|
||||
f.yamlFrontMatterFilename = yamlTempFile.Name()
|
||||
log.Debug("yamlTempFile: %v", yamlTempFile.Name())
|
||||
log.Debugf("yamlTempFile: %v", yamlTempFile.Name())
|
||||
|
||||
lineCount := 0
|
||||
|
||||
|
||||
@@ -108,6 +108,55 @@ yaml: doc
|
||||
fmHandler.CleanUp()
|
||||
}
|
||||
|
||||
func TestFrontMatterFilenamePreserved(t *testing.T) {
|
||||
// Regression test for https://github.com/mikefarah/yq/issues/2538
|
||||
// When using --front-matter, the filename operator should return
|
||||
// the original filename, not the path to the temporary file.
|
||||
file := createTestFile(`---
|
||||
name: john
|
||||
---
|
||||
Some content
|
||||
`)
|
||||
originalFilename := "/path/to/original/file.md"
|
||||
|
||||
fmHandler := NewFrontMatterHandler(file)
|
||||
err := fmHandler.Split()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
tempFilename := fmHandler.GetYamlFrontMatterFilename()
|
||||
|
||||
// Register the alias (as the command code does)
|
||||
SetFilenameAlias(tempFilename, originalFilename)
|
||||
defer ClearFilenameAliases()
|
||||
|
||||
// Verify resolveFilename returns the original name
|
||||
resolved := resolveFilename(tempFilename)
|
||||
test.AssertResult(t, originalFilename, resolved)
|
||||
|
||||
// Read documents using the temp file, verify they get the original filename
|
||||
reader, err := readStream(tempFilename)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
decoder := NewYamlDecoder(ConfiguredYamlPreferences)
|
||||
docs, err := readDocuments(reader, tempFilename, 0, decoder)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
if docs.Len() == 0 {
|
||||
t.Fatal("expected at least one document")
|
||||
}
|
||||
|
||||
firstDoc := docs.Front().Value.(*CandidateNode)
|
||||
test.AssertResult(t, originalFilename, firstDoc.filename)
|
||||
|
||||
tryRemoveTempFile(file)
|
||||
fmHandler.CleanUp()
|
||||
}
|
||||
|
||||
func TestFrontMatterSplitWithArray(t *testing.T) {
|
||||
file := createTestFile(`[1,2,3]
|
||||
---
|
||||
|
||||
+6
-1
@@ -105,7 +105,7 @@ func handleToken(tokens []*token, index int, postProcessedTokens []*token) (toke
|
||||
skipNextToken = false
|
||||
currentToken := tokens[index]
|
||||
|
||||
log.Debug("processing %v", currentToken.toString(true))
|
||||
log.Debugf("processing %v", currentToken.toString(true))
|
||||
|
||||
if currentToken.TokenType == traverseArrayCollect {
|
||||
// `.[exp]`` works by creating a traversal array of [self, exp] and piping that into the traverse array operator
|
||||
@@ -131,6 +131,11 @@ func handleToken(tokens []*token, index int, postProcessedTokens []*token) (toke
|
||||
log.Debugf("previous token is : traverseArrayOpType")
|
||||
// need to put the number 0 before this token, as that is implied
|
||||
postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: createValueOperation(0, "0")})
|
||||
} else if index >= 2 && tokens[index-1].TokenType == openCollect &&
|
||||
(tokens[index-2].TokenType == operationToken || tokens[index-2].TokenType == closeCollect || tokens[index-2].TokenType == closeCollectObject) {
|
||||
log.Debugf("previous token is : openCollect following a traversal, implying 0 start")
|
||||
// need to put the number 0 before this token, as that is implied
|
||||
postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: createValueOperation(0, "0")})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -96,6 +96,8 @@ var participleYqRules = []*participleYqRule{
|
||||
simpleOp("load_?str|str_?load", loadStringOpType),
|
||||
{"LoadYaml", `load`, loadOp(NewYamlDecoder(LoadYamlPreferences)), 0},
|
||||
|
||||
simpleOp("system", systemOpType),
|
||||
|
||||
{"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0},
|
||||
|
||||
simpleOp("select", selectOpType),
|
||||
@@ -283,7 +285,7 @@ func pathToken(wrapped bool) yqAction {
|
||||
if wrapped {
|
||||
value = unwrap(value)
|
||||
}
|
||||
log.Debug("PathToken %v", value)
|
||||
log.Debugf("PathToken %v", value)
|
||||
op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: prefs}
|
||||
return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
|
||||
}
|
||||
@@ -336,7 +338,7 @@ func flattenWithDepth() yqAction {
|
||||
|
||||
func assignAllCommentsOp(updateAssign bool) yqAction {
|
||||
return func(rawToken lexer.Token) (*token, error) {
|
||||
log.Debug("assignAllCommentsOp %v", rawToken.Value)
|
||||
log.Debugf("assignAllCommentsOp %v", rawToken.Value)
|
||||
value := rawToken.Value
|
||||
op := &Operation{
|
||||
OperationType: assignCommentOpType,
|
||||
@@ -351,7 +353,7 @@ func assignAllCommentsOp(updateAssign bool) yqAction {
|
||||
|
||||
func assignOpToken(updateAssign bool) yqAction {
|
||||
return func(rawToken lexer.Token) (*token, error) {
|
||||
log.Debug("assignOpToken %v", rawToken.Value)
|
||||
log.Debugf("assignOpToken %v", rawToken.Value)
|
||||
value := rawToken.Value
|
||||
prefs := assignPreferences{DontOverWriteAnchor: true}
|
||||
if strings.Contains(value, "c") {
|
||||
@@ -376,9 +378,9 @@ func nullValue() yqAction {
|
||||
|
||||
func stringValue() yqAction {
|
||||
return func(rawToken lexer.Token) (*token, error) {
|
||||
log.Debug("rawTokenvalue: %v", rawToken.Value)
|
||||
log.Debugf("rawTokenvalue: %v", rawToken.Value)
|
||||
value := unwrap(rawToken.Value)
|
||||
log.Debug("unwrapped: %v", value)
|
||||
log.Debugf("unwrapped: %v", value)
|
||||
value = processEscapeCharacters(value)
|
||||
return &token{TokenType: operationToken, Operation: &Operation{
|
||||
OperationType: stringInterpolationOpType,
|
||||
|
||||
+9
-10
@@ -4,11 +4,10 @@ package yqlib
|
||||
import (
|
||||
"container/list"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
logging "gopkg.in/op/go-logging.v1"
|
||||
)
|
||||
|
||||
var ExpressionParser ExpressionParserInterface
|
||||
@@ -19,17 +18,17 @@ func InitExpressionParser() {
|
||||
}
|
||||
}
|
||||
|
||||
var log = logging.MustGetLogger("yq-lib")
|
||||
var log = newLogger()
|
||||
|
||||
var PrettyPrintExp = `(... | (select(tag != "!!str"), select(tag == "!!str") | select(test("(?i)^(y|yes|n|no|on|off)$") | not)) ) style=""`
|
||||
|
||||
// GetLogger returns the yq logger instance.
|
||||
func GetLogger() *logging.Logger {
|
||||
func GetLogger() *Logger {
|
||||
return log
|
||||
}
|
||||
|
||||
func getContentValueByKey(content []*CandidateNode, key string) *CandidateNode {
|
||||
for index := 0; index < len(content); index = index + 2 {
|
||||
for index := 0; index < len(content)-1; index = index + 2 {
|
||||
keyNode := content[index]
|
||||
valueNode := content[index+1]
|
||||
if keyNode.Value == key {
|
||||
@@ -81,7 +80,7 @@ func recurseNodeObjectEqual(lhs *CandidateNode, rhs *CandidateNode) bool {
|
||||
key := lhs.Content[index]
|
||||
value := lhs.Content[index+1]
|
||||
|
||||
indexInRHS := findInArray(rhs, key)
|
||||
indexInRHS := findKeyInMap(rhs, key)
|
||||
|
||||
if indexInRHS == -1 || !recursiveNodeEqual(value, rhs.Content[indexInRHS+1]) {
|
||||
return false
|
||||
@@ -251,7 +250,7 @@ func processEscapeCharacters(original string) string {
|
||||
|
||||
value := result.String()
|
||||
if value != original {
|
||||
log.Debug("processEscapeCharacters from [%v] to [%v]", original, value)
|
||||
log.Debugf("processEscapeCharacters from [%v] to [%v]", original, value)
|
||||
}
|
||||
return value
|
||||
}
|
||||
@@ -274,7 +273,7 @@ func footComment(node *CandidateNode) string {
|
||||
|
||||
// use for debugging only
|
||||
func NodesToString(collection *list.List) string {
|
||||
if !log.IsEnabledFor(logging.DEBUG) {
|
||||
if !log.IsEnabledFor(slog.LevelDebug) {
|
||||
return ""
|
||||
}
|
||||
|
||||
@@ -286,7 +285,7 @@ func NodesToString(collection *list.List) string {
|
||||
}
|
||||
|
||||
func NodeToString(node *CandidateNode) string {
|
||||
if !log.IsEnabledFor(logging.DEBUG) {
|
||||
if !log.IsEnabledFor(slog.LevelDebug) {
|
||||
return ""
|
||||
}
|
||||
if node == nil {
|
||||
@@ -304,7 +303,7 @@ func NodeToString(node *CandidateNode) string {
|
||||
}
|
||||
|
||||
func NodeContentToString(node *CandidateNode, depth int) string {
|
||||
if !log.IsEnabledFor(logging.DEBUG) {
|
||||
if !log.IsEnabledFor(slog.LevelDebug) {
|
||||
return ""
|
||||
}
|
||||
var sb strings.Builder
|
||||
|
||||
+19
-1
@@ -24,7 +24,7 @@ type parseSnippetScenario struct {
|
||||
var parseSnippetScenarios = []parseSnippetScenario{
|
||||
{
|
||||
snippet: ":",
|
||||
expectedError: "yaml: did not find expected key",
|
||||
expectedError: "yaml: while parsing a block mapping at <unknown position>: did not find expected key",
|
||||
},
|
||||
{
|
||||
snippet: "",
|
||||
@@ -300,6 +300,24 @@ func TestRecurseNodeObjectEqual(t *testing.T) {
|
||||
test.AssertResult(t, true, recurseNodeObjectEqual(obj1, obj2))
|
||||
test.AssertResult(t, false, recurseNodeObjectEqual(obj1, obj3))
|
||||
test.AssertResult(t, false, recurseNodeObjectEqual(obj1, obj4))
|
||||
|
||||
// A null key must not match a null value in the other map.
|
||||
// Regression test for https://issues.oss-fuzz.com/issues/383860504
|
||||
nullKey := &CandidateNode{Kind: ScalarNode, Tag: "!!null"}
|
||||
nullVal := &CandidateNode{Kind: ScalarNode, Tag: "!!null"}
|
||||
intKey := createScalarNode(2, "2")
|
||||
intKey.Tag = "!!int"
|
||||
intVal := &CandidateNode{Kind: ScalarNode, Tag: "!!null"}
|
||||
|
||||
mapWithNullKey := &CandidateNode{
|
||||
Kind: MappingNode,
|
||||
Content: []*CandidateNode{nullKey, nullVal},
|
||||
}
|
||||
mapWithIntKey := &CandidateNode{
|
||||
Kind: MappingNode,
|
||||
Content: []*CandidateNode{intKey, intVal},
|
||||
}
|
||||
test.AssertResult(t, false, recurseNodeObjectEqual(mapWithNullKey, mapWithIntKey))
|
||||
}
|
||||
|
||||
func TestParseInt(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"os"
|
||||
)
|
||||
|
||||
// Logger wraps log/slog providing a printf-style interface used throughout yq.
|
||||
type Logger struct {
|
||||
levelVar slog.LevelVar
|
||||
slogger *slog.Logger
|
||||
}
|
||||
|
||||
func newLogger() *Logger {
|
||||
l := &Logger{}
|
||||
l.levelVar.Set(slog.LevelWarn)
|
||||
handler := slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: &l.levelVar})
|
||||
l.slogger = slog.New(handler)
|
||||
return l
|
||||
}
|
||||
|
||||
// SetLevel sets the minimum log level.
|
||||
func (l *Logger) SetLevel(level slog.Level) {
|
||||
l.levelVar.Set(level)
|
||||
}
|
||||
|
||||
// GetLevel returns the current log level.
|
||||
func (l *Logger) GetLevel() slog.Level {
|
||||
return l.levelVar.Level()
|
||||
}
|
||||
|
||||
// IsEnabledFor returns true if the given level is enabled.
|
||||
func (l *Logger) IsEnabledFor(level slog.Level) bool {
|
||||
return l.levelVar.Level() <= level
|
||||
}
|
||||
|
||||
// SetSlogger replaces the underlying slog.Logger (e.g. to configure output format).
|
||||
func (l *Logger) SetSlogger(logger *slog.Logger) {
|
||||
l.slogger = logger
|
||||
}
|
||||
|
||||
func (l *Logger) Debug(msg string) {
|
||||
if l.IsEnabledFor(slog.LevelDebug) {
|
||||
l.slogger.Debug(msg)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *Logger) Debugf(format string, args ...interface{}) {
|
||||
if l.IsEnabledFor(slog.LevelDebug) {
|
||||
l.slogger.Debug(fmt.Sprintf(format, args...))
|
||||
}
|
||||
}
|
||||
|
||||
func (l *Logger) Info(msg string) {
|
||||
l.slogger.Info(msg)
|
||||
}
|
||||
|
||||
func (l *Logger) Infof(format string, args ...interface{}) {
|
||||
l.slogger.Info(fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
func (l *Logger) Warning(msg string) {
|
||||
l.slogger.Warn(msg)
|
||||
}
|
||||
|
||||
func (l *Logger) Warningf(format string, args ...interface{}) {
|
||||
l.slogger.Warn(fmt.Sprintf(format, args...))
|
||||
}
|
||||
|
||||
func (l *Logger) Error(msg string) {
|
||||
l.slogger.Error(msg)
|
||||
}
|
||||
|
||||
func (l *Logger) Errorf(format string, args ...interface{}) {
|
||||
l.slogger.Error(fmt.Sprintf(format, args...))
|
||||
}
|
||||
@@ -25,7 +25,7 @@ var valueToStringFunc = func(p *Operation) string {
|
||||
}
|
||||
|
||||
func createValueOperation(value interface{}, stringValue string) *Operation {
|
||||
log.Debug("creating value op for string %v", stringValue)
|
||||
log.Debugf("creating value op for string %v", stringValue)
|
||||
var node = createScalarNode(value, stringValue)
|
||||
|
||||
return &Operation{
|
||||
@@ -164,6 +164,8 @@ var stringInterpolationOpType = &operationType{Type: "STRING_INT", NumArgs: 0, P
|
||||
var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 52, Handler: loadOperator, CheckForPostTraverse: true}
|
||||
var loadStringOpType = &operationType{Type: "LOAD_STRING", NumArgs: 1, Precedence: 52, Handler: loadStringOperator}
|
||||
|
||||
var systemOpType = &operationType{Type: "SYSTEM", NumArgs: 1, Precedence: 50, Handler: systemOperator}
|
||||
|
||||
var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 52, Handler: keysOperator, CheckForPostTraverse: true}
|
||||
|
||||
var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator}
|
||||
|
||||
@@ -195,9 +195,9 @@ func addMaps(target *CandidateNode, lhsC *CandidateNode, rhsC *CandidateNode) {
|
||||
for index := 0; index < len(rhs.Content); index = index + 2 {
|
||||
key := rhs.Content[index]
|
||||
value := rhs.Content[index+1]
|
||||
log.Debug("finding %v", key.Value)
|
||||
log.Debugf("finding %v", key.Value)
|
||||
indexInLHS := findKeyInMap(target, key)
|
||||
log.Debug("indexInLhs %v", indexInLHS)
|
||||
log.Debugf("indexInLhs %v", indexInLHS)
|
||||
if indexInLHS < 0 {
|
||||
// not in there, append it
|
||||
target.AddKeyValueChild(key, value)
|
||||
|
||||
@@ -527,6 +527,18 @@ var addOperatorScenarios = []expressionScenario{
|
||||
expression: `.a += [2]`,
|
||||
expectedError: "!!seq () cannot be added to a !!str (a)",
|
||||
},
|
||||
{
|
||||
// Regression test for https://issues.oss-fuzz.com/issues/383860504
|
||||
// Adding a map to itself must not panic when sequence keys contain
|
||||
// single-entry mappings with a null key in one and a non-null key
|
||||
// in the other.
|
||||
skipDoc: true,
|
||||
document: "? [{~: ~}]\n: v1\n? [{2: ~}]\n: v2",
|
||||
expression: `. += .`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::? [{~: ~}]\n: v1\n? [{2: ~}]\n: v2\n",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func TestAddOperatorScenarios(t *testing.T) {
|
||||
|
||||
@@ -256,7 +256,7 @@ func explodeNode(node *CandidateNode, context Context) error {
|
||||
node.Value = node.Alias.Value
|
||||
node.Alias = nil
|
||||
}
|
||||
log.Debug("now I'm %v", NodeToString(node))
|
||||
log.Debugf("now I'm %v", NodeToString(node))
|
||||
return nil
|
||||
case MappingNode:
|
||||
// //check the map has an alias in it
|
||||
@@ -304,7 +304,7 @@ func applyAlias(node *CandidateNode, alias *CandidateNode, aliasIndex int, newCo
|
||||
if alias == nil {
|
||||
return nil
|
||||
}
|
||||
log.Debug("alias: %v", NodeToString(alias))
|
||||
log.Debugf("alias: %v", NodeToString(alias))
|
||||
if alias.Kind != MappingNode {
|
||||
return fmt.Errorf("can only use merge anchors with maps (!!map) or sequences (!!seq) of maps, but got sequence containing %v", alias.Tag)
|
||||
}
|
||||
|
||||
@@ -37,7 +37,7 @@ func assignUpdateOperator(d *dataTreeNavigator, context Context, expressionNode
|
||||
|
||||
prefs := getAssignPreferences(expressionNode.Operation.Preferences)
|
||||
|
||||
log.Debug("assignUpdateOperator prefs: %v", prefs)
|
||||
log.Debugf("assignUpdateOperator prefs: %v", prefs)
|
||||
|
||||
if !expressionNode.Operation.UpdateAssign {
|
||||
// this works because we already ran against LHS with an editable context.
|
||||
|
||||
@@ -142,7 +142,7 @@ func notOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Cont
|
||||
|
||||
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
|
||||
candidate := el.Value.(*CandidateNode)
|
||||
log.Debug("notOperation checking %v", candidate)
|
||||
log.Debugf("notOperation checking %v", candidate)
|
||||
truthy := isTruthyNode(candidate)
|
||||
result := createBooleanCandidate(candidate, !truthy)
|
||||
results.PushBack(result)
|
||||
|
||||
@@ -46,9 +46,9 @@ func containsObject(lhs *CandidateNode, rhs *CandidateNode) (bool, error) {
|
||||
rhsKey := rhs.Content[index]
|
||||
rhsValue := rhs.Content[index+1]
|
||||
log.Debugf("Looking for %v in the lhs", rhsKey.Value)
|
||||
lhsKeyIndex := findInArray(lhs, rhsKey)
|
||||
lhsKeyIndex := findKeyInMap(lhs, rhsKey)
|
||||
log.Debugf("index is %v", lhsKeyIndex)
|
||||
if lhsKeyIndex < 0 || lhsKeyIndex%2 != 0 {
|
||||
if lhsKeyIndex < 0 {
|
||||
return false, nil
|
||||
}
|
||||
lhsValue := lhs.Content[lhsKeyIndex+1]
|
||||
|
||||
@@ -65,6 +65,16 @@ var containsOperatorScenarios = []expressionScenario{
|
||||
"D0, P[], (!!bool)::false\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
// Regression: findInArray could match a null key against a null
|
||||
// value at an earlier odd index, producing a false negative.
|
||||
skipDoc: true,
|
||||
document: "? 1\n: ~\n? ~\n: x",
|
||||
expression: `contains({~: "x"})`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!bool)::true\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "String contains substring",
|
||||
document: `"foobar"`,
|
||||
|
||||
@@ -80,7 +80,7 @@ func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *Expre
|
||||
|
||||
node, errorReading := parseSnippet(formattedTimeStr)
|
||||
if errorReading != nil {
|
||||
log.Debugf("could not parse %v - lets just leave it as a string: %w", formattedTimeStr, errorReading)
|
||||
log.Debugf("could not parse %v - lets just leave it as a string: %v", formattedTimeStr, errorReading)
|
||||
node = &CandidateNode{
|
||||
Kind: ScalarNode,
|
||||
Tag: "!!str",
|
||||
|
||||
@@ -45,7 +45,7 @@ func removeFromContext(context Context, candidate *CandidateNode) (Context, erro
|
||||
if nodeInContext != candidate {
|
||||
newResults.PushBack(nodeInContext)
|
||||
} else {
|
||||
log.Info("Need to delete this %v", NodeToString(nodeInContext))
|
||||
log.Infof("Need to delete this %v", NodeToString(nodeInContext))
|
||||
}
|
||||
}
|
||||
return context.ChildContext(newResults), nil
|
||||
|
||||
@@ -33,7 +33,7 @@ func configureEncoder(format *Format, indent int) Encoder {
|
||||
|
||||
func encodeToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) {
|
||||
var output bytes.Buffer
|
||||
log.Debug("printing with indent: %v", prefs.indent)
|
||||
log.Debugf("printing with indent: %v", prefs.indent)
|
||||
|
||||
encoder := configureEncoder(prefs.format, prefs.indent)
|
||||
if encoder == nil {
|
||||
|
||||
@@ -157,10 +157,10 @@ func withEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
log.Debug("candidate %v", NodeToString(candidate))
|
||||
log.Debug("candidate leading content: %v", candidate.LeadingContent)
|
||||
log.Debugf("candidate %v", NodeToString(candidate))
|
||||
log.Debugf("candidate leading content: %v", candidate.LeadingContent)
|
||||
collected.LeadingContent = candidate.LeadingContent
|
||||
log.Debug("candidate FootComment: [%v]", candidate.FootComment)
|
||||
log.Debugf("candidate FootComment: [%v]", candidate.FootComment)
|
||||
|
||||
collected.HeadComment = candidate.HeadComment
|
||||
collected.FootComment = candidate.FootComment
|
||||
|
||||
@@ -21,7 +21,7 @@ func envOperator(_ *dataTreeNavigator, context Context, expressionNode *Expressi
|
||||
return Context{}, fmt.Errorf("env operations have been disabled")
|
||||
}
|
||||
envName := expressionNode.Operation.CandidateNode.Value
|
||||
log.Debug("EnvOperator, env name:", envName)
|
||||
log.Debugf("EnvOperator, env name: %v", envName)
|
||||
|
||||
rawValue := os.Getenv(envName)
|
||||
|
||||
@@ -49,9 +49,9 @@ func envOperator(_ *dataTreeNavigator, context Context, expressionNode *Expressi
|
||||
}
|
||||
|
||||
}
|
||||
log.Debug("ENV tag", node.Tag)
|
||||
log.Debug("ENV value", node.Value)
|
||||
log.Debug("ENV Kind", node.Kind)
|
||||
log.Debugf("ENV tag: %v", node.Tag)
|
||||
log.Debugf("ENV value: %v", node.Value)
|
||||
log.Debugf("ENV Kind: %v", node.Kind)
|
||||
|
||||
return context.SingleChildContext(node), nil
|
||||
}
|
||||
@@ -78,7 +78,7 @@ func envsubstOperator(_ *dataTreeNavigator, context Context, expressionNode *Exp
|
||||
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
|
||||
node := el.Value.(*CandidateNode)
|
||||
if node.Tag != "!!str" {
|
||||
log.Warning("EnvSubstOperator, env name:", node.Tag, node.Value)
|
||||
log.Warningf("EnvSubstOperator, env name: %v %v", node.Tag, node.Value)
|
||||
return Context{}, fmt.Errorf("cannot substitute with %v, can only substitute strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
|
||||
}
|
||||
|
||||
|
||||
@@ -47,8 +47,8 @@ func mapOperator(d *dataTreeNavigator, context Context, expressionNode *Expressi
|
||||
}
|
||||
|
||||
result, err := d.GetMatchingNodes(splatted, expressionNode.RHS)
|
||||
log.Debug("expressionNode.Rhs %v", expressionNode.RHS.Operation.OperationType)
|
||||
log.Debug("result %v", result)
|
||||
log.Debugf("expressionNode.Rhs %v", expressionNode.RHS.Operation.OperationType)
|
||||
log.Debugf("result %v", result)
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
|
||||
@@ -155,8 +155,10 @@ func repeatString(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error
|
||||
return nil, err
|
||||
} else if count < 0 {
|
||||
return nil, fmt.Errorf("cannot repeat string by a negative number (%v)", count)
|
||||
} else if count > 10000000 {
|
||||
return nil, fmt.Errorf("cannot repeat string by more than 100 million (%v)", count)
|
||||
}
|
||||
maxResultLen := 10 * 1024 * 1024 // 10 MiB
|
||||
if count > 0 && len(stringNode.Value) > maxResultLen/count {
|
||||
return nil, fmt.Errorf("result of repeating string (%v bytes) by %v would exceed %v bytes", len(stringNode.Value), count, maxResultLen)
|
||||
}
|
||||
target.Value = strings.Repeat(stringNode.Value, count)
|
||||
|
||||
|
||||
@@ -237,12 +237,11 @@ var multiplyOperatorScenarios = []expressionScenario{
|
||||
expectedError: "cannot repeat string by a negative number (-4)",
|
||||
},
|
||||
{
|
||||
description: "Multiply string X by more than 100 million",
|
||||
// very large string.repeats causes a panic
|
||||
description: "Multiply string by count that exceeds result size limit",
|
||||
skipDoc: true,
|
||||
document: `n: 100000001`,
|
||||
expression: `"banana" * .n`,
|
||||
expectedError: "cannot repeat string by more than 100 million (100000001)",
|
||||
expectedError: "result of repeating string (6 bytes) by 100000001 would exceed 10485760 bytes",
|
||||
},
|
||||
{
|
||||
description: "Multiply int node X string",
|
||||
@@ -554,7 +553,7 @@ var multiplyOperatorScenarios = []expressionScenario{
|
||||
document: document,
|
||||
expression: `.b * .c`,
|
||||
expected: []string{
|
||||
"D0, P[b], (!!map)::{name: dog, <<: *cat}\n",
|
||||
"D0, P[b], (!!map)::{name: dog, \"<<\": *cat}\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -693,6 +692,27 @@ var multiplyOperatorScenarios = []expressionScenario{
|
||||
"D0, P[], (!!null)::null\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
// Regression test for https://issues.oss-fuzz.com/issues/418818862
|
||||
// Large repeat count with a long string must not panic.
|
||||
skipDoc: true,
|
||||
expression: `"abc" * 99999999`,
|
||||
expectedError: "result of repeating string (3 bytes) by 99999999 would exceed 10485760 bytes",
|
||||
},
|
||||
{
|
||||
// Regression test for https://issues.oss-fuzz.com/issues/383195001
|
||||
// Product of string length * repeat count must be bounded.
|
||||
skipDoc: true,
|
||||
expression: `"x" * 99999999`,
|
||||
expectedError: "result of repeating string (1 bytes) by 99999999 would exceed 10485760 bytes",
|
||||
},
|
||||
{
|
||||
// The size guard must not overflow: len * count can wrap to
|
||||
// a negative or small value on 64-bit, bypassing the check.
|
||||
skipDoc: true,
|
||||
expression: `"ab" * 4611686018427387904`,
|
||||
expectedError: "result of repeating string (2 bytes) by 4611686018427387904 would exceed 10485760 bytes",
|
||||
},
|
||||
}
|
||||
|
||||
func TestMultiplyOperatorScenarios(t *testing.T) {
|
||||
|
||||
+42
-13
@@ -16,11 +16,44 @@ func getSliceNumber(d *dataTreeNavigator, context Context, node *CandidateNode,
|
||||
return parseInt(result.MatchingNodes.Front().Value.(*CandidateNode).Value)
|
||||
}
|
||||
|
||||
// clampSliceIndex resolves a possibly-negative slice index against
|
||||
// length and clamps the result to [0, length].
|
||||
func clampSliceIndex(index, length int) int {
|
||||
if index < 0 {
|
||||
index += length
|
||||
}
|
||||
if index < 0 {
|
||||
return 0
|
||||
}
|
||||
if index > length {
|
||||
return length
|
||||
}
|
||||
return index
|
||||
}
|
||||
|
||||
func sliceStringNode(lhsNode *CandidateNode, firstNumber int, secondNumber int) *CandidateNode {
|
||||
runes := []rune(lhsNode.Value)
|
||||
length := len(runes)
|
||||
|
||||
relativeFirstNumber := clampSliceIndex(firstNumber, length)
|
||||
relativeSecondNumber := clampSliceIndex(secondNumber, length)
|
||||
if relativeSecondNumber < relativeFirstNumber {
|
||||
relativeSecondNumber = relativeFirstNumber
|
||||
}
|
||||
|
||||
log.Debugf("sliceStringNode: slice from %v to %v", relativeFirstNumber, relativeSecondNumber)
|
||||
|
||||
slicedString := string(runes[relativeFirstNumber:relativeSecondNumber])
|
||||
replacement := lhsNode.CreateReplacement(ScalarNode, lhsNode.Tag, slicedString)
|
||||
replacement.Style = lhsNode.Style
|
||||
return replacement
|
||||
}
|
||||
|
||||
func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
|
||||
|
||||
log.Debug("slice array operator!")
|
||||
log.Debug("lhs: %v", expressionNode.LHS.Operation.toString())
|
||||
log.Debug("rhs: %v", expressionNode.RHS.Operation.toString())
|
||||
log.Debugf("lhs: %v", expressionNode.LHS.Operation.toString())
|
||||
log.Debugf("rhs: %v", expressionNode.RHS.Operation.toString())
|
||||
|
||||
results := list.New()
|
||||
|
||||
@@ -28,28 +61,24 @@ func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *E
|
||||
lhsNode := el.Value.(*CandidateNode)
|
||||
|
||||
firstNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.LHS)
|
||||
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
relativeFirstNumber := firstNumber
|
||||
if relativeFirstNumber < 0 {
|
||||
relativeFirstNumber = len(lhsNode.Content) + firstNumber
|
||||
}
|
||||
|
||||
secondNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.RHS)
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
|
||||
relativeSecondNumber := secondNumber
|
||||
if relativeSecondNumber < 0 {
|
||||
relativeSecondNumber = len(lhsNode.Content) + secondNumber
|
||||
} else if relativeSecondNumber > len(lhsNode.Content) {
|
||||
relativeSecondNumber = len(lhsNode.Content)
|
||||
if lhsNode.Kind == ScalarNode && lhsNode.guessTagFromCustomType() == "!!str" {
|
||||
results.PushBack(sliceStringNode(lhsNode, firstNumber, secondNumber))
|
||||
continue
|
||||
}
|
||||
|
||||
log.Debug("calculateIndicesToTraverse: slice from %v to %v", relativeFirstNumber, relativeSecondNumber)
|
||||
relativeFirstNumber := clampSliceIndex(firstNumber, len(lhsNode.Content))
|
||||
relativeSecondNumber := clampSliceIndex(secondNumber, len(lhsNode.Content))
|
||||
|
||||
log.Debugf("calculateIndicesToTraverse: slice from %v to %v", relativeFirstNumber, relativeSecondNumber)
|
||||
|
||||
var newResults []*CandidateNode
|
||||
for i := relativeFirstNumber; i < relativeSecondNumber; i++ {
|
||||
|
||||
@@ -98,6 +98,115 @@ var sliceArrayScenarios = []expressionScenario{
|
||||
"D0, P[], (!!seq)::- cat1\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
// Regression test for https://issues.oss-fuzz.com/issues/438776028
|
||||
// Negative second index that underflows after adjustment must
|
||||
// clamp to zero, yielding an empty sequence.
|
||||
skipDoc: true,
|
||||
document: `[a, b, c]`,
|
||||
expression: `.[0:-99999]`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!seq)::[]\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
// First-index underflow: without clamping, the loop starts at a
|
||||
// negative index and panics on Content access.
|
||||
skipDoc: true,
|
||||
document: `[a, b, c]`,
|
||||
expression: `.[-99999:3]`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!seq)::- a\n- b\n- c\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
// Both indices underflow: both clamp to zero, yielding an empty
|
||||
// sequence.
|
||||
skipDoc: true,
|
||||
document: `[a, b, c]`,
|
||||
expression: `.[-99999:-99998]`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!seq)::[]\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Slicing strings",
|
||||
document: `country: Australia`,
|
||||
expression: `.country[0:5]`,
|
||||
expected: []string{
|
||||
"D0, P[country], (!!str)::Austr\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Slicing strings - without the second number",
|
||||
subdescription: "Finishes at the end of the string",
|
||||
document: `country: Australia`,
|
||||
expression: `.country[5:]`,
|
||||
expected: []string{
|
||||
"D0, P[country], (!!str)::alia\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Slicing strings - without the first number",
|
||||
subdescription: "Starts from the start of the string",
|
||||
document: `country: Australia`,
|
||||
expression: `.country[:5]`,
|
||||
expected: []string{
|
||||
"D0, P[country], (!!str)::Austr\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Slicing strings - use negative numbers to count backwards from the end",
|
||||
subdescription: "Negative indices count from the end of the string",
|
||||
document: `country: Australia`,
|
||||
expression: `.country[-5:]`,
|
||||
expected: []string{
|
||||
"D0, P[country], (!!str)::ralia\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
document: `country: Australia`,
|
||||
expression: `.country[1:-1]`,
|
||||
expected: []string{
|
||||
"D0, P[country], (!!str)::ustrali\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
document: `country: Australia`,
|
||||
expression: `.country[:]`,
|
||||
expected: []string{
|
||||
"D0, P[country], (!!str)::Australia\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "second index beyond string length clamps",
|
||||
document: `country: Australia`,
|
||||
expression: `.country[:100]`,
|
||||
expected: []string{
|
||||
"D0, P[country], (!!str)::Australia\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "first index beyond string length returns empty string",
|
||||
document: `country: Australia`,
|
||||
expression: `.country[100:]`,
|
||||
expected: []string{
|
||||
"D0, P[country], (!!str)::\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Slicing strings - Unicode",
|
||||
subdescription: "Indices are rune-based, so multi-byte characters are handled correctly",
|
||||
document: `greeting: héllo`,
|
||||
expression: `.greeting[1:3]`,
|
||||
expected: []string{
|
||||
"D0, P[greeting], (!!str)::él\n",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func TestSliceOperatorScenarios(t *testing.T) {
|
||||
|
||||
@@ -142,12 +142,12 @@ func (a sortableNodeArray) compare(lhs *CandidateNode, rhs *CandidateNode, dateT
|
||||
} else if isDateTime {
|
||||
lhsTime, err := parseDateTime(layout, lhs.Value)
|
||||
if err != nil {
|
||||
log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %w", lhs.Value, layout, err)
|
||||
log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %v", lhs.Value, layout, err)
|
||||
return strings.Compare(lhs.Value, rhs.Value)
|
||||
}
|
||||
rhsTime, err := parseDateTime(layout, rhs.Value)
|
||||
if err != nil {
|
||||
log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %w", rhs.Value, layout, err)
|
||||
log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %v", rhs.Value, layout, err)
|
||||
return strings.Compare(lhs.Value, rhs.Value)
|
||||
}
|
||||
if lhsTime.Equal(rhsTime) {
|
||||
|
||||
@@ -220,7 +220,7 @@ func getSubstituteParameters(d *dataTreeNavigator, block *ExpressionNode, contex
|
||||
regEx = regExNodes.MatchingNodes.Front().Value.(*CandidateNode).Value
|
||||
}
|
||||
|
||||
log.Debug("regEx %v", regEx)
|
||||
log.Debugf("regEx %v", regEx)
|
||||
|
||||
replacementNodes, err := d.GetMatchingNodes(context, block.RHS)
|
||||
if err != nil {
|
||||
@@ -318,7 +318,7 @@ func getMatches(matchPrefs matchPreferences, regEx *regexp.Regexp, value string)
|
||||
allIndices = [][]int{regEx.FindStringSubmatchIndex(value)}
|
||||
}
|
||||
|
||||
log.Debug("allMatches, %v", allMatches)
|
||||
log.Debugf("allMatches, %v", allMatches)
|
||||
return allMatches, allIndices
|
||||
}
|
||||
|
||||
@@ -423,7 +423,7 @@ func extractMatchArguments(d *dataTreeNavigator, context Context, expressionNode
|
||||
if regExNodes.MatchingNodes.Front() != nil {
|
||||
regExStr = regExNodes.MatchingNodes.Front().Value.(*CandidateNode).Value
|
||||
}
|
||||
log.Debug("regEx %v", regExStr)
|
||||
log.Debugf("regEx %v", regExStr)
|
||||
regEx, err := regexp.Compile(regExStr)
|
||||
return regEx, matchPrefs, err
|
||||
}
|
||||
@@ -564,7 +564,7 @@ func split(value string, spltStr string) (Kind, string, []*CandidateNode) {
|
||||
var contents []*CandidateNode
|
||||
|
||||
if value != "" {
|
||||
log.Debug("going to spltStr[%v]", spltStr)
|
||||
log.Debugf("going to spltStr[%v]", spltStr)
|
||||
var newStrings = strings.Split(value, spltStr)
|
||||
contents = make([]*CandidateNode, len(newStrings))
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ func parseStyle(customStyle string) (Style, error) {
|
||||
|
||||
func assignStyleOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
|
||||
|
||||
log.Debugf("AssignStyleOperator: %v")
|
||||
log.Debugf("AssignStyleOperator")
|
||||
var style Style
|
||||
if !expressionNode.Operation.UpdateAssign {
|
||||
rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS)
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"container/list"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func resolveSystemArgs(argsNode *CandidateNode) ([]string, error) {
|
||||
if argsNode == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if argsNode.Kind == SequenceNode {
|
||||
args := make([]string, 0, len(argsNode.Content))
|
||||
for _, child := range argsNode.Content {
|
||||
// Only non-null scalar children are valid arguments.
|
||||
if child == nil {
|
||||
continue
|
||||
}
|
||||
if child.Kind != ScalarNode || child.Tag == "!!null" {
|
||||
return nil, fmt.Errorf("system operator: argument must be a non-null scalar; got kind=%v tag=%v", child.Kind, child.Tag)
|
||||
}
|
||||
args = append(args, child.Value)
|
||||
}
|
||||
if len(args) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return args, nil
|
||||
}
|
||||
|
||||
// Single-argument case: only accept a non-null scalar node.
|
||||
if argsNode.Tag == "!!null" {
|
||||
return nil, nil
|
||||
}
|
||||
if argsNode.Kind != ScalarNode {
|
||||
return nil, fmt.Errorf("system operator: args must be a non-null scalar or sequence of non-null scalars; got kind=%v tag=%v", argsNode.Kind, argsNode.Tag)
|
||||
}
|
||||
return []string{argsNode.Value}, nil
|
||||
}
|
||||
|
||||
func resolveCommandNode(commandNodes Context) (string, error) {
|
||||
if commandNodes.MatchingNodes.Front() == nil {
|
||||
return "", fmt.Errorf("system operator: command expression returned no results")
|
||||
}
|
||||
if commandNodes.MatchingNodes.Len() > 1 {
|
||||
log.Debugf("system operator: command expression returned %d results, using first", commandNodes.MatchingNodes.Len())
|
||||
}
|
||||
cmdNode := commandNodes.MatchingNodes.Front().Value.(*CandidateNode)
|
||||
if cmdNode.Kind != ScalarNode || cmdNode.guessTagFromCustomType() != "!!str" {
|
||||
return "", fmt.Errorf("system operator: command must be a string scalar")
|
||||
}
|
||||
if cmdNode.Value == "" {
|
||||
return "", fmt.Errorf("system operator: command must be a non-empty string")
|
||||
}
|
||||
return cmdNode.Value, nil
|
||||
}
|
||||
|
||||
func systemOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
|
||||
if !ConfiguredSecurityPreferences.EnableSystemOps {
|
||||
return Context{}, fmt.Errorf("system operations are disabled, use --security-enable-system-operator to enable")
|
||||
}
|
||||
|
||||
// determine at parse time whether we have (command; args) or just (command)
|
||||
hasArgs := expressionNode.RHS.Operation.OperationType == blockOpType
|
||||
|
||||
var results = list.New()
|
||||
|
||||
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
|
||||
candidate := el.Value.(*CandidateNode)
|
||||
nodeContext := context.SingleReadonlyChildContext(candidate)
|
||||
|
||||
var command string
|
||||
var args []string
|
||||
|
||||
if hasArgs {
|
||||
block := expressionNode.RHS
|
||||
commandNodes, err := d.GetMatchingNodes(nodeContext, block.LHS)
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
command, err = resolveCommandNode(commandNodes)
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
|
||||
argsNodes, err := d.GetMatchingNodes(nodeContext, block.RHS)
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
if argsNodes.MatchingNodes.Len() > 1 {
|
||||
log.Debugf("system operator: args expression returned %d results, using first", argsNodes.MatchingNodes.Len())
|
||||
}
|
||||
if argsNodes.MatchingNodes.Front() != nil {
|
||||
args, err = resolveSystemArgs(argsNodes.MatchingNodes.Front().Value.(*CandidateNode))
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
commandNodes, err := d.GetMatchingNodes(nodeContext, expressionNode.RHS)
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
command, err = resolveCommandNode(commandNodes)
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
}
|
||||
|
||||
var stdin bytes.Buffer
|
||||
encoded, err := encodeToYamlString(candidate)
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
stdin.WriteString(encoded)
|
||||
|
||||
// #nosec G204 - intentional: user must explicitly enable this operator
|
||||
cmd := exec.Command(command, args...)
|
||||
cmd.Stdin = &stdin
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
stderrStr := strings.TrimSpace(stderr.String())
|
||||
if stderrStr != "" {
|
||||
return Context{}, fmt.Errorf("system command '%v' failed: %w\nstderr: %v", command, err, stderrStr)
|
||||
}
|
||||
return Context{}, fmt.Errorf("system command '%v' failed: %w", command, err)
|
||||
}
|
||||
|
||||
result := string(output)
|
||||
if strings.HasSuffix(result, "\r\n") {
|
||||
result = result[:len(result)-2]
|
||||
} else if strings.HasSuffix(result, "\n") {
|
||||
result = result[:len(result)-1]
|
||||
}
|
||||
newNode := candidate.CreateReplacement(ScalarNode, "!!str", result)
|
||||
results.PushBack(newNode)
|
||||
}
|
||||
|
||||
return context.ChildContext(results), nil
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func findExec(t *testing.T, name string) string {
|
||||
t.Helper()
|
||||
path, err := exec.LookPath(name)
|
||||
if err != nil {
|
||||
t.Skipf("skipping: %v not found: %v", name, err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
var systemOperatorDisabledScenarios = []expressionScenario{
|
||||
{
|
||||
description: "system operator returns error when disabled",
|
||||
subdescription: "Use `--security-enable-system-operator` to enable the system operator.",
|
||||
document: "country: Australia",
|
||||
expression: `.country = system("/usr/bin/echo"; "test")`,
|
||||
expectedError: "system operations are disabled, use --security-enable-system-operator to enable",
|
||||
},
|
||||
}
|
||||
|
||||
func TestSystemOperatorDisabledScenarios(t *testing.T) {
|
||||
originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps
|
||||
defer func() {
|
||||
ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps
|
||||
}()
|
||||
|
||||
ConfiguredSecurityPreferences.EnableSystemOps = false
|
||||
|
||||
for _, tt := range systemOperatorDisabledScenarios {
|
||||
testScenario(t, &tt)
|
||||
}
|
||||
documentOperatorScenarios(t, "system-operators", systemOperatorDisabledScenarios)
|
||||
}
|
||||
|
||||
func TestSystemOperatorEnabledScenarios(t *testing.T) {
|
||||
echoPath := findExec(t, "echo")
|
||||
falsePath := findExec(t, "false")
|
||||
|
||||
originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps
|
||||
defer func() {
|
||||
ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps
|
||||
}()
|
||||
|
||||
ConfiguredSecurityPreferences.EnableSystemOps = true
|
||||
|
||||
scenarios := []expressionScenario{
|
||||
{
|
||||
description: "Run a command with an argument",
|
||||
subdescription: "Use `--security-enable-system-operator` to enable the system operator.",
|
||||
yqFlags: "--security-enable-system-operator",
|
||||
document: "country: Australia",
|
||||
expression: `.country = system("` + echoPath + `"; "test")`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::country: test\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Run a command without arguments",
|
||||
subdescription: "Omit the semicolon and args to run the command with no extra arguments.",
|
||||
yqFlags: "--security-enable-system-operator",
|
||||
document: "a: hello",
|
||||
expression: `.a = system("` + echoPath + `")`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::a: \"\"\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Run a command with multiple arguments",
|
||||
subdescription: "Pass an array of arguments.",
|
||||
skipDoc: true,
|
||||
document: "a: hello",
|
||||
expression: `.a = system("` + echoPath + `"; ["foo", "bar"])`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::a: foo bar\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Command and args are evaluated per matched node",
|
||||
skipDoc: true,
|
||||
document: "cmd: " + echoPath + "\narg: hello",
|
||||
expression: `.result = system(.cmd; .arg)`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::cmd: " + echoPath + "\narg: hello\nresult: hello\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Command failure returns error",
|
||||
skipDoc: true,
|
||||
document: "a: hello",
|
||||
expression: `.a = system("` + falsePath + `")`,
|
||||
expectedError: "system command '" + falsePath + "' failed: exit status 1",
|
||||
},
|
||||
{
|
||||
description: "Null command returns error",
|
||||
skipDoc: true,
|
||||
document: "a: hello",
|
||||
expression: `.a = system(null)`,
|
||||
expectedError: "system operator: command must be a string scalar",
|
||||
},
|
||||
{
|
||||
description: "System operator processes multiple matched nodes",
|
||||
skipDoc: true,
|
||||
document: "a: first",
|
||||
document2: "a: second",
|
||||
expression: `.a = system("` + echoPath + `"; "replaced")`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::a: replaced\n",
|
||||
"D0, P[], (!!map)::a: replaced\n",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range scenarios {
|
||||
testScenario(t, &tt)
|
||||
}
|
||||
appendOperatorDocumentScenario(t, "system-operators", scenarios)
|
||||
}
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
|
||||
func assignTagOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
|
||||
|
||||
log.Debugf("AssignTagOperator: %v")
|
||||
log.Debugf("AssignTagOperator")
|
||||
tag := ""
|
||||
|
||||
if !expressionNode.Operation.UpdateAssign {
|
||||
|
||||
@@ -36,8 +36,32 @@ func traversePathOperator(_ *dataTreeNavigator, context Context, expressionNode
|
||||
return context.ChildContext(matches), nil
|
||||
}
|
||||
|
||||
// resolveAliasChain follows an alias chain iteratively, returning the
|
||||
// first non-alias node. Returns an error if a cycle is detected.
|
||||
func resolveAliasChain(node *CandidateNode) (*CandidateNode, error) {
|
||||
if node.Kind != AliasNode {
|
||||
return node, nil
|
||||
}
|
||||
visited := map[*CandidateNode]bool{}
|
||||
for node.Kind == AliasNode {
|
||||
if visited[node] {
|
||||
return nil, fmt.Errorf("alias cycle detected")
|
||||
}
|
||||
visited[node] = true
|
||||
log.Debug("its an alias!")
|
||||
node = node.Alias
|
||||
}
|
||||
return node, nil
|
||||
}
|
||||
|
||||
func traverse(context Context, matchingNode *CandidateNode, operation *Operation) (*list.List, error) {
|
||||
log.Debug("Traversing %v", NodeToString(matchingNode))
|
||||
log.Debugf("Traversing %v", NodeToString(matchingNode))
|
||||
|
||||
var err error
|
||||
matchingNode, err = resolveAliasChain(matchingNode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if matchingNode.Tag == "!!null" && operation.Value != "[]" && !context.DontAutoCreate {
|
||||
log.Debugf("Guessing kind")
|
||||
@@ -55,17 +79,13 @@ func traverse(context Context, matchingNode *CandidateNode, operation *Operation
|
||||
|
||||
switch matchingNode.Kind {
|
||||
case MappingNode:
|
||||
log.Debug("its a map with %v entries", len(matchingNode.Content)/2)
|
||||
log.Debugf("its a map with %v entries", len(matchingNode.Content)/2)
|
||||
return traverseMap(context, matchingNode, createStringScalarNode(operation.StringValue), operation.Preferences.(traversePreferences), false)
|
||||
|
||||
case SequenceNode:
|
||||
log.Debug("its a sequence of %v things!", len(matchingNode.Content))
|
||||
log.Debugf("its a sequence of %v things!", len(matchingNode.Content))
|
||||
return traverseArray(matchingNode, operation, operation.Preferences.(traversePreferences))
|
||||
|
||||
case AliasNode:
|
||||
log.Debug("its an alias!")
|
||||
matchingNode = matchingNode.Alias
|
||||
return traverse(context, matchingNode, operation)
|
||||
default:
|
||||
return list.New(), nil
|
||||
}
|
||||
@@ -79,7 +99,11 @@ func traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode
|
||||
log.Debugf("--traverseArrayOperator")
|
||||
|
||||
if expressionNode.RHS != nil && expressionNode.RHS.RHS != nil && expressionNode.RHS.RHS.Operation.OperationType == createMapOpType {
|
||||
return sliceArrayOperator(d, context, expressionNode.RHS.RHS)
|
||||
lhsContext, err := d.GetMatchingNodes(context, expressionNode.LHS)
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
return sliceArrayOperator(d, lhsContext, expressionNode.RHS.RHS)
|
||||
}
|
||||
|
||||
lhs, err := d.GetMatchingNodes(context, expressionNode.LHS)
|
||||
@@ -125,7 +149,13 @@ func traverseNodesWithArrayIndices(context Context, indicesToTraverse []*Candida
|
||||
return context.ChildContext(matchingNodeMap), nil
|
||||
}
|
||||
|
||||
func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesToTraverse []*CandidateNode, prefs traversePreferences) (*list.List, error) { // call this if doc / alias like the other traverse
|
||||
func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesToTraverse []*CandidateNode, prefs traversePreferences) (*list.List, error) {
|
||||
var err error
|
||||
matchingNode, err = resolveAliasChain(matchingNode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if matchingNode.Tag == "!!null" {
|
||||
log.Debugf("OperatorArrayTraverse got a null - turning it into an empty array")
|
||||
// auto vivification
|
||||
@@ -138,9 +168,6 @@ func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesT
|
||||
}
|
||||
|
||||
switch matchingNode.Kind {
|
||||
case AliasNode:
|
||||
matchingNode = matchingNode.Alias
|
||||
return traverseArrayIndices(context, matchingNode, indicesToTraverse, prefs)
|
||||
case SequenceNode:
|
||||
return traverseArrayWithIndices(matchingNode, indicesToTraverse, prefs)
|
||||
case MappingNode:
|
||||
@@ -158,7 +185,7 @@ func traverseMapWithIndices(context Context, candidate *CandidateNode, indices [
|
||||
var matchingNodeMap = list.New()
|
||||
|
||||
for _, indexNode := range indices {
|
||||
log.Debug("traverseMapWithIndices: %v", indexNode.Value)
|
||||
log.Debugf("traverseMapWithIndices: %v", indexNode.Value)
|
||||
newNodes, err := traverseMap(context, candidate, indexNode, prefs, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -183,7 +210,7 @@ func traverseArrayWithIndices(node *CandidateNode, indices []*CandidateNode, pre
|
||||
}
|
||||
|
||||
for _, indexNode := range indices {
|
||||
log.Debug("traverseArrayWithIndices: '%v'", indexNode.Value)
|
||||
log.Debugf("traverseArrayWithIndices: '%v'", indexNode.Value)
|
||||
index, err := parseInt(indexNode.Value)
|
||||
if err != nil && prefs.OptionalTraverse {
|
||||
continue
|
||||
@@ -366,7 +393,7 @@ func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, merge *CandidateNode
|
||||
}
|
||||
|
||||
func traverseArray(candidate *CandidateNode, operation *Operation, prefs traversePreferences) (*list.List, error) {
|
||||
log.Debug("operation Value %v", operation.Value)
|
||||
log.Debugf("operation Value %v", operation.Value)
|
||||
indices := []*CandidateNode{{Value: operation.StringValue}}
|
||||
return traverseArrayWithIndices(candidate, indices, prefs)
|
||||
}
|
||||
|
||||
@@ -665,6 +665,16 @@ var traversePathOperatorScenarios = []expressionScenario{
|
||||
"D0, P[a], (!!null)::null\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
// Regression test for https://issues.oss-fuzz.com/issues/390467412
|
||||
// go-yaml accepts cross-document alias references (invalid per
|
||||
// YAML spec). A nested assignment on such an alias can create a
|
||||
// circular alias node, which must not cause a stack overflow.
|
||||
skipDoc: true,
|
||||
document: "&-- a\n---\n*--",
|
||||
expression: ". = (.x = 1)",
|
||||
expectedError: "alias cycle detected",
|
||||
},
|
||||
}
|
||||
|
||||
func TestTraversePathOperatorScenarios(t *testing.T) {
|
||||
@@ -682,3 +692,58 @@ func TestTraversePathOperatorAlignedToSpecScenarios(t *testing.T) {
|
||||
appendOperatorDocumentScenario(t, "traverse-read", fixedTraversePathOperatorScenarios)
|
||||
ConfiguredYamlPreferences.FixMergeAnchorToSpec = false
|
||||
}
|
||||
|
||||
// Regression test for https://issues.oss-fuzz.com/issues/390467412
|
||||
// A circular alias (alias pointing back to itself) must not cause a
|
||||
// stack overflow. resolveAliasChain should detect the cycle and return
|
||||
// an error; both traverse() and traverseArrayIndices() use it.
|
||||
func TestTraverseAliasCycle(t *testing.T) {
|
||||
aliasNode := &CandidateNode{
|
||||
Kind: AliasNode,
|
||||
}
|
||||
aliasNode.Alias = aliasNode // A -> A
|
||||
|
||||
op := &Operation{
|
||||
OperationType: traversePathOpType,
|
||||
Value: "key",
|
||||
StringValue: "key",
|
||||
Preferences: traversePreferences{},
|
||||
}
|
||||
_, err := traverse(Context{}, aliasNode, op)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for alias cycle, got nil")
|
||||
}
|
||||
if err.Error() != "alias cycle detected" {
|
||||
t.Fatalf("expected 'alias cycle detected', got %q", err.Error())
|
||||
}
|
||||
|
||||
// Same cycle must be caught through the array traversal path.
|
||||
_, err = traverseArrayIndices(Context{}, aliasNode, nil, traversePreferences{})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for alias cycle via traverseArrayIndices, got nil")
|
||||
}
|
||||
if err.Error() != "alias cycle detected" {
|
||||
t.Fatalf("expected 'alias cycle detected', got %q", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTraverseAliasCycleChain(t *testing.T) {
|
||||
nodeA := &CandidateNode{Kind: AliasNode}
|
||||
nodeB := &CandidateNode{Kind: AliasNode}
|
||||
nodeA.Alias = nodeB
|
||||
nodeB.Alias = nodeA // A -> B -> A
|
||||
|
||||
op := &Operation{
|
||||
OperationType: traversePathOpType,
|
||||
Value: "key",
|
||||
StringValue: "key",
|
||||
Preferences: traversePreferences{},
|
||||
}
|
||||
_, err := traverse(Context{}, nodeA, op)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for alias cycle chain, got nil")
|
||||
}
|
||||
if err.Error() != "alias cycle detected" {
|
||||
t.Fatalf("expected 'alias cycle detected', got %q", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,21 +4,21 @@ import "container/list"
|
||||
|
||||
func unionOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
|
||||
log.Debug("unionOperator--")
|
||||
log.Debug("unionOperator: context: %v", NodesToString(context.MatchingNodes))
|
||||
log.Debugf("unionOperator: context: %v", NodesToString(context.MatchingNodes))
|
||||
lhs, err := d.GetMatchingNodes(context, expressionNode.LHS)
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
log.Debug("unionOperator: lhs: %v", NodesToString(lhs.MatchingNodes))
|
||||
log.Debug("unionOperator: rhs input: %v", NodesToString(context.MatchingNodes))
|
||||
log.Debug("unionOperator: rhs: %v", expressionNode.RHS.Operation.toString())
|
||||
log.Debugf("unionOperator: lhs: %v", NodesToString(lhs.MatchingNodes))
|
||||
log.Debugf("unionOperator: rhs input: %v", NodesToString(context.MatchingNodes))
|
||||
log.Debugf("unionOperator: rhs: %v", expressionNode.RHS.Operation.toString())
|
||||
rhs, err := d.GetMatchingNodes(context, expressionNode.RHS)
|
||||
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
log.Debug("unionOperator: lhs: %v", lhs.ToString())
|
||||
log.Debug("unionOperator: rhs: %v", rhs.ToString())
|
||||
log.Debugf("unionOperator: lhs: %v", lhs.ToString())
|
||||
log.Debugf("unionOperator: rhs: %v", rhs.ToString())
|
||||
|
||||
results := lhs.ChildContext(list.New())
|
||||
for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
|
||||
@@ -33,11 +33,11 @@ func unionOperator(d *dataTreeNavigator, context Context, expressionNode *Expres
|
||||
|
||||
for el := rhs.MatchingNodes.Front(); el != nil; el = el.Next() {
|
||||
node := el.Value.(*CandidateNode)
|
||||
log.Debug("union operator rhs: processing %v", NodeToString(node))
|
||||
log.Debugf("union operator rhs: processing %v", NodeToString(node))
|
||||
|
||||
results.MatchingNodes.PushBack(node)
|
||||
}
|
||||
}
|
||||
log.Debug("union operator: all together: %v", results.ToString())
|
||||
log.Debugf("union operator: all together: %v", results.ToString())
|
||||
return results, nil
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@ func referenceOperator(_ *dataTreeNavigator, context Context, expressionNode *Ex
|
||||
}
|
||||
|
||||
func valueOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
|
||||
log.Debug("value = %v", expressionNode.Operation.CandidateNode.Value)
|
||||
log.Debugf("value = %v", expressionNode.Operation.CandidateNode.Value)
|
||||
if context.MatchingNodes.Len() == 0 {
|
||||
clone := expressionNode.Operation.CandidateNode.Copy()
|
||||
return context.SingleChildContext(clone), nil
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
|
||||
func getVariableOperator(_ *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
|
||||
variableName := expressionNode.Operation.StringValue
|
||||
log.Debug("getVariableOperator %v", variableName)
|
||||
log.Debugf("getVariableOperator %v", variableName)
|
||||
result := context.GetVariable(variableName)
|
||||
if result == nil {
|
||||
result = list.New()
|
||||
@@ -67,7 +67,7 @@ func variableLoopSingleChild(d *dataTreeNavigator, context Context, originalExp
|
||||
|
||||
// now we loop over lhs, set variable to each result and calculate originalExp.Rhs
|
||||
for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
|
||||
log.Debug("PROCESSING VARIABLE: ", NodeToString(el.Value.(*CandidateNode)))
|
||||
log.Debugf("PROCESSING VARIABLE: %v", NodeToString(el.Value.(*CandidateNode)))
|
||||
var variableValue = list.New()
|
||||
if prefs.IsReference {
|
||||
variableValue.PushBack(el.Value)
|
||||
@@ -83,7 +83,7 @@ func variableLoopSingleChild(d *dataTreeNavigator, context Context, originalExp
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
log.Debug("PROCESSING VARIABLE DONE, got back: ", rhs.MatchingNodes.Len())
|
||||
log.Debugf("PROCESSING VARIABLE DONE, got back: %v", rhs.MatchingNodes.Len())
|
||||
results.PushBackList(rhs.MatchingNodes)
|
||||
}
|
||||
|
||||
|
||||
@@ -3,9 +3,9 @@ package yqlib
|
||||
import (
|
||||
"container/list"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"github.com/jinzhu/copier"
|
||||
logging "gopkg.in/op/go-logging.v1"
|
||||
)
|
||||
|
||||
type operatorHandler func(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error)
|
||||
@@ -88,7 +88,7 @@ func resultsForRHS(d *dataTreeNavigator, context Context, lhsCandidate *Candidat
|
||||
|
||||
for rightEl := rhs.MatchingNodes.Front(); rightEl != nil; rightEl = rightEl.Next() {
|
||||
rhsCandidate := rightEl.Value.(*CandidateNode)
|
||||
if !log.IsEnabledFor(logging.DEBUG) {
|
||||
if !log.IsEnabledFor(slog.LevelDebug) {
|
||||
log.Debugf("Applying lhs: %v, rhsCandidate, %v", NodeToString(lhsCandidate), NodeToString(rhsCandidate))
|
||||
}
|
||||
resultCandidate, err := prefs.Calculation(d, context, lhsCandidate, rhsCandidate)
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -14,7 +15,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/mikefarah/yq/v4/test"
|
||||
logging "gopkg.in/op/go-logging.v1"
|
||||
)
|
||||
|
||||
type expressionScenario struct {
|
||||
@@ -31,6 +31,7 @@ type expressionScenario struct {
|
||||
dontFormatInputForDoc bool // dont format input doc for documentation generation
|
||||
requiresFormat string
|
||||
skipForGoccy bool
|
||||
yqFlags string // extra yq flags to include in generated doc command snippets
|
||||
}
|
||||
|
||||
var goccyTesting = false
|
||||
@@ -38,9 +39,9 @@ var goccyTesting = false
|
||||
var testingDecoder = NewYamlDecoder(ConfiguredYamlPreferences)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
logging.SetLevel(logging.WARNING, "")
|
||||
GetLogger().SetLevel(slog.LevelWarn)
|
||||
if os.Getenv("DEBUG") == "true" {
|
||||
logging.SetLevel(logging.DEBUG, "")
|
||||
GetLogger().SetLevel(slog.LevelDebug)
|
||||
}
|
||||
ConfiguredYamlPreferences.ColorsEnabled = false
|
||||
ConfiguredJSONPreferences.ColorsEnabled = false
|
||||
@@ -356,14 +357,22 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
|
||||
|
||||
writeOrPanic(w, "then\n")
|
||||
|
||||
flagsPrefix := ""
|
||||
if s.yqFlags != "" {
|
||||
flagsPrefix = s.yqFlags + " "
|
||||
}
|
||||
if s.expression != "" {
|
||||
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v'%v' %v\n```\n", envCommand, command, strings.ReplaceAll(s.expression, "'", `'\''`), files))
|
||||
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v'%v' %v\n```\n", envCommand, flagsPrefix, command, strings.ReplaceAll(s.expression, "'", `'\''`), files))
|
||||
} else {
|
||||
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v\n```\n", envCommand, command, files))
|
||||
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v%v\n```\n", envCommand, flagsPrefix, command, files))
|
||||
}
|
||||
} else {
|
||||
writeOrPanic(w, "Running\n")
|
||||
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v--null-input '%v'\n```\n", envCommand, command, s.expression))
|
||||
flagsPrefix := ""
|
||||
if s.yqFlags != "" {
|
||||
flagsPrefix = s.yqFlags + " "
|
||||
}
|
||||
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v--null-input '%v'\n```\n", envCommand, flagsPrefix, command, s.expression))
|
||||
}
|
||||
return formattedDoc, formattedDoc2
|
||||
}
|
||||
|
||||
@@ -70,7 +70,7 @@ func removeLastEOL(b *bytes.Buffer) {
|
||||
}
|
||||
|
||||
func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
|
||||
log.Debug("PrintResults for %v matches", matchingNodes.Len())
|
||||
log.Debugf("PrintResults for %v matches", matchingNodes.Len())
|
||||
|
||||
if matchingNodes.Len() == 0 {
|
||||
log.Debug("no matching results, nothing to print")
|
||||
@@ -97,8 +97,8 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error {
|
||||
for el := matchingNodes.Front(); el != nil; el = el.Next() {
|
||||
|
||||
mappedDoc := el.Value.(*CandidateNode)
|
||||
log.Debug("print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v", p.firstTimePrinting, p.previousDocIndex)
|
||||
log.Debug("%v", NodeToString(mappedDoc))
|
||||
log.Debugf("print sep logic: p.firstTimePrinting: %v, previousDocIndex: %v", p.firstTimePrinting, p.previousDocIndex)
|
||||
log.Debugf("%v", NodeToString(mappedDoc))
|
||||
writer, errorWriting := p.printerWriter.GetWriter(mappedDoc)
|
||||
if errorWriting != nil {
|
||||
return errorWriting
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
package yqlib
|
||||
|
||||
type SecurityPreferences struct {
|
||||
DisableEnvOps bool
|
||||
DisableFileOps bool
|
||||
DisableEnvOps bool
|
||||
DisableFileOps bool
|
||||
EnableSystemOps bool
|
||||
}
|
||||
|
||||
var ConfiguredSecurityPreferences = SecurityPreferences{
|
||||
DisableEnvOps: false,
|
||||
DisableFileOps: false,
|
||||
DisableEnvOps: false,
|
||||
DisableFileOps: false,
|
||||
EnableSystemOps: false,
|
||||
}
|
||||
|
||||
@@ -76,6 +76,7 @@ func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, p
|
||||
}
|
||||
|
||||
func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *ExpressionNode, printer Printer, decoder Decoder) (uint, error) {
|
||||
filename = resolveFilename(filename)
|
||||
|
||||
var currentIndex uint
|
||||
err := decoder.Init(reader)
|
||||
|
||||
@@ -287,6 +287,18 @@ var expectedSubArrays = `array:
|
||||
- {}
|
||||
`
|
||||
|
||||
// Keys with special characters that require quoting in TOML
|
||||
var rtSpecialKeyInlineTable = `host = { "http://sealos.hub:5000" = { capabilities = ["pull", "resolve", "push"], skip_verify = true } }
|
||||
`
|
||||
|
||||
var rtSpecialKeyTableSection = `["/tmp/blah"]
|
||||
value = "hello"
|
||||
`
|
||||
|
||||
var rtSpecialKeyDottedTableSection = `[servers."http://localhost:8080"]
|
||||
ip = "127.0.0.1"
|
||||
`
|
||||
|
||||
var tomlScenarios = []formatScenario{
|
||||
{
|
||||
skipDoc: true,
|
||||
@@ -614,6 +626,58 @@ var tomlScenarios = []formatScenario{
|
||||
expected: tomlTableWithComments,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
// Encode (YAML → TOML) scenarios - verify readable table sections are produced
|
||||
{
|
||||
description: "Encode: Simple mapping produces table section",
|
||||
input: "arg:\n hello: foo\n",
|
||||
expected: "[arg]\nhello = \"foo\"\n",
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "Encode: Nested mappings produce nested table sections",
|
||||
input: "a:\n b:\n c: val\n",
|
||||
expected: "[a.b]\nc = \"val\"\n",
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "Encode: Mixed scalars and nested mapping",
|
||||
input: "a:\n hello: foo\n nested:\n key: val\n",
|
||||
expected: "[a]\nhello = \"foo\"\n\n[a.nested]\nkey = \"val\"\n",
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "Encode: YAML flow mapping stays inline",
|
||||
input: "arg: {hello: foo}\n",
|
||||
expected: "arg = { hello = \"foo\" }\n",
|
||||
scenarioType: "encode",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "Roundtrip: key with special characters in inline table",
|
||||
input: rtSpecialKeyInlineTable,
|
||||
expression: ".",
|
||||
expected: rtSpecialKeyInlineTable,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "Roundtrip: key with special characters in table section",
|
||||
input: rtSpecialKeyTableSection,
|
||||
expression: ".",
|
||||
expected: rtSpecialKeyTableSection,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "Roundtrip: special character key in dotted table section header",
|
||||
input: rtSpecialKeyDottedTableSection,
|
||||
expression: ".",
|
||||
expected: rtSpecialKeyDottedTableSection,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
}
|
||||
|
||||
func testTomlScenario(t *testing.T, s formatScenario) {
|
||||
@@ -629,6 +693,8 @@ func testTomlScenario(t *testing.T, s formatScenario) {
|
||||
}
|
||||
case "roundtrip":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewTomlDecoder(), NewTomlEncoder()), s.description)
|
||||
case "encode":
|
||||
test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewTomlEncoder()), s.description)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -654,6 +720,28 @@ func documentTomlDecodeScenario(w *bufio.Writer, s formatScenario) {
|
||||
writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewTomlDecoder(), NewYamlEncoder(ConfiguredYamlPreferences))))
|
||||
}
|
||||
|
||||
func documentTomlEncodeScenario(w *bufio.Writer, s formatScenario) {
|
||||
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
|
||||
|
||||
if s.subdescription != "" {
|
||||
writeOrPanic(w, s.subdescription)
|
||||
writeOrPanic(w, "\n\n")
|
||||
}
|
||||
|
||||
writeOrPanic(w, "Given a sample.yml file of:\n")
|
||||
writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input))
|
||||
|
||||
writeOrPanic(w, "then\n")
|
||||
expression := s.expression
|
||||
if expression == "" {
|
||||
expression = "."
|
||||
}
|
||||
writeOrPanic(w, fmt.Sprintf("```bash\nyq -o toml '%v' sample.yml\n```\n", expression))
|
||||
writeOrPanic(w, "will output\n")
|
||||
|
||||
writeOrPanic(w, fmt.Sprintf("```toml\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewTomlEncoder())))
|
||||
}
|
||||
|
||||
func documentTomlRoundtripScenario(w *bufio.Writer, s formatScenario) {
|
||||
writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
|
||||
|
||||
@@ -687,6 +775,8 @@ func documentTomlScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
|
||||
documentTomlDecodeScenario(w, s)
|
||||
case "roundtrip":
|
||||
documentTomlRoundtripScenario(w, s)
|
||||
case "encode":
|
||||
documentTomlEncodeScenario(w, s)
|
||||
|
||||
default:
|
||||
panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType))
|
||||
|
||||
@@ -9,6 +9,29 @@ import (
|
||||
"os"
|
||||
)
|
||||
|
||||
// filenameAliases maps real file paths to display names.
|
||||
// Used by front matter handling to preserve original filenames
|
||||
// when the actual content is read from temporary files.
|
||||
var filenameAliases = map[string]string{}
|
||||
|
||||
// SetFilenameAlias registers a display name for a file path so that
|
||||
// the filename operator returns the original name instead of a temp path.
|
||||
func SetFilenameAlias(realPath string, displayName string) {
|
||||
filenameAliases[realPath] = displayName
|
||||
}
|
||||
|
||||
// ClearFilenameAliases removes all filename aliases.
|
||||
func ClearFilenameAliases() {
|
||||
filenameAliases = map[string]string{}
|
||||
}
|
||||
|
||||
func resolveFilename(filename string) string {
|
||||
if alias, ok := filenameAliases[filename]; ok {
|
||||
return alias
|
||||
}
|
||||
return filename
|
||||
}
|
||||
|
||||
func readStream(filename string) (io.Reader, error) {
|
||||
var reader *bufio.Reader
|
||||
if filename == "-" {
|
||||
@@ -36,6 +59,7 @@ func ReadDocuments(reader io.Reader, decoder Decoder) (*list.List, error) {
|
||||
}
|
||||
|
||||
func readDocuments(reader io.Reader, filename string, fileIndex int, decoder Decoder) (*list.List, error) {
|
||||
filename = resolveFilename(filename)
|
||||
err := decoder.Init(reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -38,13 +38,13 @@ func (w *writeInPlaceHandlerImpl) CreateTempFile() (*os.File, error) {
|
||||
if err = changeOwner(info, file); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Debug("WriteInPlaceHandler: writing to tempfile: %v", file.Name())
|
||||
log.Debugf("WriteInPlaceHandler: writing to tempfile: %v", file.Name())
|
||||
w.tempFile = file
|
||||
return file, err
|
||||
}
|
||||
|
||||
func (w *writeInPlaceHandlerImpl) FinishWriteInPlace(evaluatedSuccessfully bool) error {
|
||||
log.Debug("Going to write in place, evaluatedSuccessfully=%v, target=%v", evaluatedSuccessfully, w.inputFilename)
|
||||
log.Debugf("Going to write in place, evaluatedSuccessfully=%v, target=%v", evaluatedSuccessfully, w.inputFilename)
|
||||
safelyCloseFile(w.tempFile)
|
||||
if evaluatedSuccessfully {
|
||||
log.Debug("Moving temp file to target")
|
||||
|
||||
+7
-1
@@ -109,6 +109,7 @@ hostpath
|
||||
hotdog
|
||||
howdy
|
||||
incase
|
||||
Infof
|
||||
inlinetables
|
||||
inplace
|
||||
ints
|
||||
@@ -297,4 +298,9 @@ subsubarray
|
||||
Ffile
|
||||
Fquery
|
||||
coverpkg
|
||||
gsub
|
||||
gsub
|
||||
ralia
|
||||
Austr
|
||||
ustrali
|
||||
héllo
|
||||
alia
|
||||
|
||||
@@ -8,6 +8,15 @@
|
||||
- git push --tags
|
||||
- use github actions to publish docker and make github release
|
||||
- check github updated yq action in marketplace
|
||||
- update github-action/Dockerfile to pin the newly published docker image digest:
|
||||
skopeo inspect docker://docker.io/mikefarah/yq:4 | python3 -c "import sys,json; d=json.load(sys.stdin); print(d['Digest'])"
|
||||
then update the FROM line in github-action/Dockerfile with the new digest:
|
||||
FROM mikefarah/yq:4@sha256:<new-digest>
|
||||
|
||||
// release artifacts are signed with cosign keyless signing (Sigstore)
|
||||
// users can verify with:
|
||||
// cosign verify-blob --bundle checksums.bundle checksums
|
||||
// install cosign: brew install cosign OR go install github.com/sigstore/cosign/v2/cmd/cosign@latest
|
||||
|
||||
|
||||
- snapcraft
|
||||
|
||||
@@ -1,3 +1,17 @@
|
||||
4.52.5:
|
||||
- Fix: reset TOML decoder state between files (#2634) thanks @terminalchai
|
||||
- Fix: preserve original filename when using --front-matter (#2613) thanks @cobyfrombrooklyn-bot
|
||||
- Fix typo in filename (#2611) thanks @alexandear
|
||||
- Bumped dependencies
|
||||
|
||||
4.52.4:
|
||||
- Dropping windows/arm - no longer supported in cross-compile
|
||||
|
||||
4.52.3:
|
||||
- Fixing comments in TOML arrays (#2592)
|
||||
- Bumped dependencies
|
||||
|
||||
|
||||
4.52.2:
|
||||
- Fixed bad instructions file breaking go-install (#2587) Thanks @theyoprst
|
||||
- Fixed TOML table scope after comments (#2588) Thanks @tomers
|
||||
|
||||
@@ -3,8 +3,6 @@
|
||||
set -o errexit
|
||||
set -o pipefail
|
||||
|
||||
# TODO: Check if the found golangci-lint version matches the expected version (e.g., v1.62.0), especially if falling back to PATH version.
|
||||
|
||||
GOPATH_LINT="$(go env GOPATH)/bin/golangci-lint"
|
||||
BIN_LINT="./bin/golangci-lint"
|
||||
LINT_CMD=""
|
||||
|
||||
+1
-1
@@ -1,5 +1,5 @@
|
||||
#!/bin/sh
|
||||
set -ex
|
||||
go mod download golang.org/x/tools@latest
|
||||
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.1.5
|
||||
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.11.3
|
||||
curl -sSfL https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.22.11
|
||||
+2
-2
@@ -1,5 +1,5 @@
|
||||
name: yq
|
||||
version: 'v4.52.2'
|
||||
version: 'v4.52.5'
|
||||
summary: A lightweight and portable command-line data file processor
|
||||
description: |
|
||||
`yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml, json, xml, csv, properties and TOML files.
|
||||
@@ -32,6 +32,6 @@ parts:
|
||||
build-environment:
|
||||
- CGO_ENABLED: 0
|
||||
source: https://github.com/mikefarah/yq.git
|
||||
source-tag: v4.52.2
|
||||
source-tag: v4.52.5
|
||||
build-snaps:
|
||||
- go/latest/stable
|
||||
|
||||
@@ -14,6 +14,7 @@ func TestMainFunction(t *testing.T) {
|
||||
cmd := command.New()
|
||||
if cmd == nil {
|
||||
t.Fatal("command.New() returned nil")
|
||||
return
|
||||
}
|
||||
|
||||
if cmd.Use != "yq" {
|
||||
@@ -99,6 +100,7 @@ func TestMainFunctionExecution(t *testing.T) {
|
||||
// the command structure is correct and can be configured
|
||||
if cmd == nil {
|
||||
t.Fatal("Command should not be nil")
|
||||
return
|
||||
}
|
||||
|
||||
if cmd.Use != "yq" {
|
||||
|
||||
Reference in New Issue
Block a user