Compare commits

...

7 Commits

Author SHA1 Message Date
Tony
fbf92beaf5
Merge 249efaee46 into 8e2c9b612d 2026-06-21 14:48:20 -04:00
dependabot[bot]
8e2c9b612d
Bump golang from 68cb6d6 to 11fd8f7 (#2738)
Bumps golang from `68cb6d6` to `11fd8f7`.

---
updated-dependencies:
- dependency-name: golang
  dependency-version: 1.26.4
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-06-16 16:46:10 +10:00
dependabot[bot]
0970cd4b05
Bump alpine from 5b10f43 to a2d49ea (#2739)
Bumps alpine from `5b10f43` to `a2d49ea`.

---
updated-dependencies:
- dependency-name: alpine
  dependency-version: '3'
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-06-16 16:46:03 +10:00
dependabot[bot]
bf3591a234
Bump golang.org/x/text from 0.37.0 to 0.38.0 (#2742)
Bumps [golang.org/x/text](https://github.com/golang/text) from 0.37.0 to 0.38.0.
- [Release notes](https://github.com/golang/text/releases)
- [Commits](https://github.com/golang/text/compare/v0.37.0...v0.38.0)

---
updated-dependencies:
- dependency-name: golang.org/x/text
  dependency-version: 0.38.0
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-06-16 16:45:37 +10:00
dependabot[bot]
09f1565d51
Bump github/codeql-action from 4.35.2 to 4.36.2 (#2743)
Bumps [github/codeql-action](https://github.com/github/codeql-action) from 4.35.2 to 4.36.2.
- [Release notes](https://github.com/github/codeql-action/releases)
- [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md)
- [Commits](95e58e9a2c...8aad20d150)

---
updated-dependencies:
- dependency-name: github/codeql-action
  dependency-version: 4.36.2
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-06-16 16:29:01 +10:00
dependabot[bot]
13d340ff51
Bump sigstore/cosign-installer from 3.10.1 to 4.1.2 (#2744)
Bumps [sigstore/cosign-installer](https://github.com/sigstore/cosign-installer) from 3.10.1 to 4.1.2.
- [Release notes](https://github.com/sigstore/cosign-installer/releases)
- [Commits](7e8b541eb2...6f9f177880)

---
updated-dependencies:
- dependency-name: sigstore/cosign-installer
  dependency-version: 4.1.2
  dependency-type: direct:production
  update-type: version-update:semver-major
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-06-16 16:28:52 +10:00
Tony
249efaee46 feat: add --split-exp-no-overwrite flag to refuse overwriting existing files
When using --split-exp to split documents into per-file outputs, yq
silently overwrites any pre-existing files at the target paths because
os.Create truncates. For workflows that generate filenames from input
data (e.g. '.metadata.name + ".yml"'), this can clobber unrelated files
when two documents map to the same name, or when a target path collides
with something already on disk.

This change adds an opt-in --split-exp-no-overwrite flag (and a
matching yqlib constructor NewMultiPrinterWriterWithOptions) that uses
O_WRONLY|O_CREATE|O_EXCL so existing files are left untouched and yq
exits with a clear error message instead.

The default behaviour (overwrite) is unchanged; the original
NewMultiPrinterWriter constructor still exists and now delegates to the
new options-aware constructor with noOverwrite=false.

Fixes #2028
2026-05-22 05:41:48 +08:00
13 changed files with 140 additions and 22 deletions

View File

@ -44,7 +44,7 @@ jobs:
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2
uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
@ -55,7 +55,7 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2
uses: github/codeql-action/autobuild@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
# Command-line programs to run using the OS shell.
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@ -69,4 +69,4 @@ jobs:
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2
uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2

View File

@ -43,7 +43,7 @@ jobs:
man.md
- name: Install cosign
uses: sigstore/cosign-installer@7e8b541eb2e61bf99390e1afd4be13a184e9ebc5 # v3.10.1
uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
- name: Cross compile
run: |

View File

@ -73,6 +73,6 @@ jobs:
# Upload the results to GitHub's code scanning dashboard (optional).
# Commenting out will disable upload of results to your repo's Code Scanning dashboard
- name: "Upload to code-scanning"
uses: github/codeql-action/upload-sarif@95e58e9a2cdfd71adc6e0353d5c52f41a045d225 # v4.35.2
uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
with:
sarif_file: results.sarif

View File

@ -1,4 +1,4 @@
FROM golang:1.26.4@sha256:68cb6d68bed024785b69195b89af7ac7a444f27791435f98647edff595aa0479 AS builder
FROM golang:1.26.4@sha256:11fd8f7f63db3b6fb198797042ba4c40a4a34dc83325d3328ca3bc4bb7726786 AS builder
WORKDIR /go/src/mikefarah/yq
@ -10,7 +10,7 @@ RUN ./scripts/acceptance.sh
# Choose alpine as a base image to make this useful for CI, as many
# CI tools expect an interactive shell inside the container
FROM alpine:3@sha256:5b10f432ef3da1b8d4c7eb6c487f2f5a8f096bc91145e68878dd4a5019afde11 AS production
FROM alpine:3@sha256:a2d49ea686c2adfe3c992e47dc3b5e7fa6e6b5055609400dc2acaeb241c829f4 AS production
LABEL maintainer="Mike Farah <mikefarah@users.noreply.github.com>"
COPY --from=builder /go/src/mikefarah/yq/yq /usr/bin/yq

View File

@ -1,4 +1,4 @@
FROM golang:1.26.4@sha256:68cb6d68bed024785b69195b89af7ac7a444f27791435f98647edff595aa0479
FROM golang:1.26.4@sha256:11fd8f7f63db3b6fb198797042ba4c40a4a34dc83325d3328ca3bc4bb7726786
COPY scripts/devtools.sh /opt/devtools.sh

View File

@ -433,6 +433,7 @@ Flags:
--shell-key-separator string separator for shell variable key paths (default "_")
-s, --split-exp string print each result (or doc) into a file named (exp). [exp] argument must return a string. You can use $index in the expression as the result counter. The necessary directories will be created.
--split-exp-file string Use a file to specify the split-exp expression.
--split-exp-no-overwrite When using --split-exp, fail if a target file already exists instead of overwriting it.
--string-interpolation Toggles strings interpolation of \(exp) (default true)
--tsv-auto-parse parse TSV YAML/JSON values (default true)
-r, --unwrapScalar unwrap scalar, print the value with no quotes, colors or comments. Defaults to true for yaml (default true)

View File

@ -31,6 +31,7 @@ var frontMatter = ""
var splitFileExp = ""
var splitFileExpFile = ""
var splitFileNoOverwrite = false
var completedSuccessfully = false

View File

@ -206,6 +206,7 @@ yq -P -oy sample.json
if err = rootCmd.MarkPersistentFlagFilename("split-exp-file"); err != nil {
panic(err)
}
rootCmd.PersistentFlags().BoolVarP(&splitFileNoOverwrite, "split-exp-no-overwrite", "", false, "When using --split-exp, fail if a target file already exists instead of overwriting it.")
rootCmd.PersistentFlags().StringVarP(&expressionFile, "from-file", "", "", "Load expression from specified file.")
if err = rootCmd.MarkPersistentFlagFilename("from-file"); err != nil {

View File

@ -186,7 +186,7 @@ func configurePrinterWriter(format *yqlib.Format, out io.Writer) (yqlib.PrinterW
if err != nil {
return nil, fmt.Errorf("bad split document expression: %w", err)
}
printerWriter = yqlib.NewMultiPrinterWriter(splitExp, format)
printerWriter = yqlib.NewMultiPrinterWriterWithOptions(splitExp, format, splitFileNoOverwrite)
} else {
printerWriter = yqlib.NewSinglePrinterWriter(out)
}

6
go.mod
View File

@ -22,7 +22,7 @@ require (
go.yaml.in/yaml/v4 v4.0.0-rc.5
golang.org/x/mod v0.36.0
golang.org/x/net v0.55.0
golang.org/x/text v0.37.0
golang.org/x/text v0.38.0
)
require (
@ -33,9 +33,9 @@ require (
github.com/mattn/go-colorable v0.1.14 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
golang.org/x/sync v0.20.0 // indirect
golang.org/x/sync v0.21.0 // indirect
golang.org/x/sys v0.45.0 // indirect
golang.org/x/tools v0.44.0 // indirect
golang.org/x/tools v0.45.0 // indirect
)
go 1.25.0

12
go.sum
View File

@ -74,15 +74,15 @@ golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4=
golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ=
golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8=
golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY=
golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc=
golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38=
golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c=
golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI=
golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE=
golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4=
golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@ -32,9 +32,17 @@ type multiPrintWriter struct {
nameExpression *ExpressionNode
extension string
index int
noOverwrite bool
}
func NewMultiPrinterWriter(expression *ExpressionNode, format *Format) PrinterWriter {
return NewMultiPrinterWriterWithOptions(expression, format, false)
}
// NewMultiPrinterWriterWithOptions creates a multi-file printer writer.
// When noOverwrite is true, attempting to write to a file that already
// exists will fail with an error instead of silently overwriting it.
func NewMultiPrinterWriterWithOptions(expression *ExpressionNode, format *Format, noOverwrite bool) PrinterWriter {
extension := "yml"
switch format {
@ -49,6 +57,7 @@ func NewMultiPrinterWriter(expression *ExpressionNode, format *Format) PrinterWr
extension: extension,
treeNavigator: NewDataTreeNavigator(),
index: 0,
noOverwrite: noOverwrite,
}
}
@ -75,10 +84,20 @@ func (sp *multiPrintWriter) GetWriter(node *CandidateNode) (*bufio.Writer, error
if err != nil {
return nil, err
}
f, err := os.Create(name)
if err != nil {
return nil, err
var f *os.File
if sp.noOverwrite {
f, err = os.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
if err != nil {
if os.IsExist(err) {
return nil, fmt.Errorf("refusing to overwrite existing file %q (--no-overwrite is set)", name)
}
return nil, err
}
} else {
f, err = os.Create(name)
if err != nil {
return nil, err
}
}
sp.index = sp.index + 1

View File

@ -0,0 +1,96 @@
package yqlib
import (
"os"
"path/filepath"
"strings"
"testing"
)
// helper to build an ExpressionNode that just yields a fixed string for the file name
func parseFilenameExp(t *testing.T, exp string) *ExpressionNode {
t.Helper()
InitExpressionParser()
node, err := ExpressionParser.ParseExpression(exp)
if err != nil {
t.Fatalf("failed to parse split-exp test expression %q: %v", exp, err)
}
return node
}
func TestMultiPrinterWriterOverwriteDefault(t *testing.T) {
dir := t.TempDir()
target := filepath.Join(dir, "out.yml")
if err := os.WriteFile(target, []byte("pre-existing\n"), 0600); err != nil {
t.Fatalf("setup: %v", err)
}
exp := parseFilenameExp(t, `"`+target+`"`)
pw := NewMultiPrinterWriter(exp, YamlFormat)
node := &CandidateNode{Kind: ScalarNode, Tag: "!!str", Value: "hello"}
w, err := pw.GetWriter(node)
if err != nil {
t.Fatalf("default behaviour should silently overwrite, got error: %v", err)
}
if w == nil {
t.Fatalf("expected a writer, got nil")
}
// confirm the file was truncated/recreated by os.Create
info, err := os.Stat(target)
if err != nil {
t.Fatalf("stat target: %v", err)
}
if info.Size() != 0 {
t.Fatalf("expected file to be truncated (size 0) before writes, got %d bytes", info.Size())
}
}
func TestMultiPrinterWriterNoOverwriteRefusesExisting(t *testing.T) {
dir := t.TempDir()
target := filepath.Join(dir, "out.yml")
if err := os.WriteFile(target, []byte("pre-existing\n"), 0600); err != nil {
t.Fatalf("setup: %v", err)
}
exp := parseFilenameExp(t, `"`+target+`"`)
pw := NewMultiPrinterWriterWithOptions(exp, YamlFormat, true)
node := &CandidateNode{Kind: ScalarNode, Tag: "!!str", Value: "hello"}
_, err := pw.GetWriter(node)
if err == nil {
t.Fatalf("expected error when --no-overwrite is set and target exists, got nil")
}
if !strings.Contains(err.Error(), "refusing to overwrite") {
t.Fatalf("expected refusing-to-overwrite error message, got: %v", err)
}
// file must be untouched
data, err := os.ReadFile(target)
if err != nil {
t.Fatalf("read target: %v", err)
}
if string(data) != "pre-existing\n" {
t.Fatalf("file should be untouched, contents = %q", string(data))
}
}
func TestMultiPrinterWriterNoOverwriteCreatesNew(t *testing.T) {
dir := t.TempDir()
target := filepath.Join(dir, "new.yml")
exp := parseFilenameExp(t, `"`+target+`"`)
pw := NewMultiPrinterWriterWithOptions(exp, YamlFormat, true)
node := &CandidateNode{Kind: ScalarNode, Tag: "!!str", Value: "hello"}
w, err := pw.GetWriter(node)
if err != nil {
t.Fatalf("no-overwrite should still create new files, got: %v", err)
}
if w == nil {
t.Fatalf("expected a writer, got nil")
}
if _, err := os.Stat(target); err != nil {
t.Fatalf("expected new file to exist, stat err: %v", err)
}
}