From 22e609b2d941421ab4a918c8811631a3a1050a6d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Apr 2026 18:57:30 +1000 Subject: [PATCH 1/4] Bump golang from 1.26.1 to 1.26.2 (#2654) Bumps golang from 1.26.1 to 1.26.2. --- updated-dependencies: - dependency-name: golang dependency-version: 1.26.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- Dockerfile.dev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2b53f48e..a598c449 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.26.1 AS builder +FROM golang:1.26.2 AS builder WORKDIR /go/src/mikefarah/yq diff --git a/Dockerfile.dev b/Dockerfile.dev index 990cffd2..51209dc6 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM golang:1.26.1 +FROM golang:1.26.2 RUN apt-get update && \ apt-get install -y npm && \ From 44c55c8a54d1fa207dac0bf6e9084d6dd0adff30 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Fri, 10 Apr 2026 20:06:46 +1000 Subject: [PATCH 2/4] Add system(command; args) operator (disabled by default) (#2640) * Initial plan * Add system(command; args) operator with --enable-system-operator flag Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/8a11e9a0-10d2-4f2a-ae29-4e9d0bfc266f Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> * Update pkg/yqlib/operator_system.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Evaluate system command/args per matched node using SingleReadonlyChildContext Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/dca841eb-3f63-4f23-adeb-556431560420 Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> * Add yqFlags to expressionScenario for doc command snippets; fix system op docs Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/3f8a5375-25fd-4428-a8e6-b630194c36b2 Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> * Update pkg/yqlib/doc/operators/headers/system-operators.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update pkg/yqlib/doc/operators/system-operators.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Validate command node type and handle multiple results with debug log Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/928aabc5-ad71-41d8-94ab-403942e3f92d Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> * Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Remove deprecated --enable-system-operator alias; use --security-enable-system-operator consistently Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/286b95e9-b6d7-4ab8-b401-2d7a03853922 Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> * Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Address deep review feedback: error on disabled, strict arg/cmd validation, debug logs, docs Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/fbfba2db-60ea-4c20-a4c2-0fd396b80c81 Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> Co-authored-by: Mike Farah Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- cmd/root.go | 1 + .../doc/operators/headers/system-operators.md | 27 ++++ pkg/yqlib/doc/operators/system-operators.md | 76 +++++++++ pkg/yqlib/lexer_participle.go | 2 + pkg/yqlib/operation.go | 2 + pkg/yqlib/operator_system.go | 146 ++++++++++++++++++ pkg/yqlib/operator_system_test.go | 123 +++++++++++++++ pkg/yqlib/operators_test.go | 15 +- pkg/yqlib/security_prefs.go | 10 +- 9 files changed, 395 insertions(+), 7 deletions(-) create mode 100644 pkg/yqlib/doc/operators/headers/system-operators.md create mode 100644 pkg/yqlib/doc/operators/system-operators.md create mode 100644 pkg/yqlib/operator_system.go create mode 100644 pkg/yqlib/operator_system_test.go diff --git a/cmd/root.go b/cmd/root.go index 449be025..d2f72bfa 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -212,6 +212,7 @@ yq -P -oy sample.json rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableEnvOps, "security-disable-env-ops", "", false, "Disable env related operations.") rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableFileOps, "security-disable-file-ops", "", false, "Disable file related operations (e.g. load)") + rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.EnableSystemOps, "security-enable-system-operator", "", false, "Enable system operator to allow execution of external commands.") rootCmd.AddCommand( createEvaluateSequenceCommand(), diff --git a/pkg/yqlib/doc/operators/headers/system-operators.md b/pkg/yqlib/doc/operators/headers/system-operators.md new file mode 100644 index 00000000..220a2fa3 --- /dev/null +++ b/pkg/yqlib/doc/operators/headers/system-operators.md @@ -0,0 +1,27 @@ +# System Operators + +The `system` operator allows you to run an external command and use its output as a value in your expression. + +**Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it. + +**Note:** When enabled, the system operator can replicate the functionality of `env` and `load` +operators via external commands. Enabling it effectively overrides `--security-disable-env-ops` +and `--security-disable-file-ops`. + +## Usage + +```bash +yq --security-enable-system-operator --null-input '.field = system("command"; "arg1")' +``` + +The operator takes: +- A command string (required) +- An argument (or an array of arguments), separated from the command by `;` (optional) + +The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string. + +## Disabling the system operator + +The system operator is disabled by default. When disabled, an error is returned instead of running the command, consistent with `--security-disable-env-ops` and `--security-disable-file-ops`. + +Use `--security-enable-system-operator` flag to enable it. diff --git a/pkg/yqlib/doc/operators/system-operators.md b/pkg/yqlib/doc/operators/system-operators.md new file mode 100644 index 00000000..df1a76cd --- /dev/null +++ b/pkg/yqlib/doc/operators/system-operators.md @@ -0,0 +1,76 @@ +# System Operators + +The `system` operator allows you to run an external command and use its output as a value in your expression. + +**Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it. + +**Note:** When enabled, the system operator can replicate the functionality of `env` and `load` +operators via external commands. Enabling it effectively overrides `--security-disable-env-ops` +and `--security-disable-file-ops`. + +## Usage + +```bash +yq --security-enable-system-operator --null-input '.field = system("command"; "arg1")' +``` + +The operator takes: +- A command string (required) +- An argument (or an array of arguments), separated from the command by `;` (optional) + +The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string. + +## Disabling the system operator + +The system operator is disabled by default. When disabled, an error is returned instead of running the command, consistent with `--security-disable-env-ops` and `--security-disable-file-ops`. + +Use `--security-enable-system-operator` flag to enable it. + +## system operator returns error when disabled +Use `--security-enable-system-operator` to enable the system operator. + +Given a sample.yml file of: +```yaml +country: Australia +``` +then +```bash +yq '.country = system("/usr/bin/echo"; "test")' sample.yml +``` +will output +```bash +Error: system operations are disabled, use --security-enable-system-operator to enable +``` + +## Run a command with an argument +Use `--security-enable-system-operator` to enable the system operator. + +Given a sample.yml file of: +```yaml +country: Australia +``` +then +```bash +yq --security-enable-system-operator '.country = system("/usr/bin/echo"; "test")' sample.yml +``` +will output +```yaml +country: test +``` + +## Run a command without arguments +Omit the semicolon and args to run the command with no extra arguments. + +Given a sample.yml file of: +```yaml +a: hello +``` +then +```bash +yq --security-enable-system-operator '.a = system("/usr/bin/echo")' sample.yml +``` +will output +```yaml +a: "" +``` + diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go index 866b736c..938b8a71 100644 --- a/pkg/yqlib/lexer_participle.go +++ b/pkg/yqlib/lexer_participle.go @@ -96,6 +96,8 @@ var participleYqRules = []*participleYqRule{ simpleOp("load_?str|str_?load", loadStringOpType), {"LoadYaml", `load`, loadOp(NewYamlDecoder(LoadYamlPreferences)), 0}, + simpleOp("system", systemOpType), + {"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0}, simpleOp("select", selectOpType), diff --git a/pkg/yqlib/operation.go b/pkg/yqlib/operation.go index f36054ee..cbea1d7b 100644 --- a/pkg/yqlib/operation.go +++ b/pkg/yqlib/operation.go @@ -164,6 +164,8 @@ var stringInterpolationOpType = &operationType{Type: "STRING_INT", NumArgs: 0, P var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 52, Handler: loadOperator, CheckForPostTraverse: true} var loadStringOpType = &operationType{Type: "LOAD_STRING", NumArgs: 1, Precedence: 52, Handler: loadStringOperator} +var systemOpType = &operationType{Type: "SYSTEM", NumArgs: 1, Precedence: 50, Handler: systemOperator} + var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 52, Handler: keysOperator, CheckForPostTraverse: true} var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator} diff --git a/pkg/yqlib/operator_system.go b/pkg/yqlib/operator_system.go new file mode 100644 index 00000000..c9937957 --- /dev/null +++ b/pkg/yqlib/operator_system.go @@ -0,0 +1,146 @@ +package yqlib + +import ( + "bytes" + "container/list" + "fmt" + "os/exec" + "strings" +) + +func resolveSystemArgs(argsNode *CandidateNode) ([]string, error) { + if argsNode == nil { + return nil, nil + } + + if argsNode.Kind == SequenceNode { + args := make([]string, 0, len(argsNode.Content)) + for _, child := range argsNode.Content { + // Only non-null scalar children are valid arguments. + if child == nil { + continue + } + if child.Kind != ScalarNode || child.Tag == "!!null" { + return nil, fmt.Errorf("system operator: argument must be a non-null scalar; got kind=%v tag=%v", child.Kind, child.Tag) + } + args = append(args, child.Value) + } + if len(args) == 0 { + return nil, nil + } + return args, nil + } + + // Single-argument case: only accept a non-null scalar node. + if argsNode.Tag == "!!null" { + return nil, nil + } + if argsNode.Kind != ScalarNode { + return nil, fmt.Errorf("system operator: args must be a non-null scalar or sequence of non-null scalars; got kind=%v tag=%v", argsNode.Kind, argsNode.Tag) + } + return []string{argsNode.Value}, nil +} + +func resolveCommandNode(commandNodes Context) (string, error) { + if commandNodes.MatchingNodes.Front() == nil { + return "", fmt.Errorf("system operator: command expression returned no results") + } + if commandNodes.MatchingNodes.Len() > 1 { + log.Debugf("system operator: command expression returned %d results, using first", commandNodes.MatchingNodes.Len()) + } + cmdNode := commandNodes.MatchingNodes.Front().Value.(*CandidateNode) + if cmdNode.Kind != ScalarNode || cmdNode.guessTagFromCustomType() != "!!str" { + return "", fmt.Errorf("system operator: command must be a string scalar") + } + if cmdNode.Value == "" { + return "", fmt.Errorf("system operator: command must be a non-empty string") + } + return cmdNode.Value, nil +} + +func systemOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { + if !ConfiguredSecurityPreferences.EnableSystemOps { + return Context{}, fmt.Errorf("system operations are disabled, use --security-enable-system-operator to enable") + } + + // determine at parse time whether we have (command; args) or just (command) + hasArgs := expressionNode.RHS.Operation.OperationType == blockOpType + + var results = list.New() + + for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + nodeContext := context.SingleReadonlyChildContext(candidate) + + var command string + var args []string + + if hasArgs { + block := expressionNode.RHS + commandNodes, err := d.GetMatchingNodes(nodeContext, block.LHS) + if err != nil { + return Context{}, err + } + command, err = resolveCommandNode(commandNodes) + if err != nil { + return Context{}, err + } + + argsNodes, err := d.GetMatchingNodes(nodeContext, block.RHS) + if err != nil { + return Context{}, err + } + if argsNodes.MatchingNodes.Len() > 1 { + log.Debugf("system operator: args expression returned %d results, using first", argsNodes.MatchingNodes.Len()) + } + if argsNodes.MatchingNodes.Front() != nil { + args, err = resolveSystemArgs(argsNodes.MatchingNodes.Front().Value.(*CandidateNode)) + if err != nil { + return Context{}, err + } + } + } else { + commandNodes, err := d.GetMatchingNodes(nodeContext, expressionNode.RHS) + if err != nil { + return Context{}, err + } + command, err = resolveCommandNode(commandNodes) + if err != nil { + return Context{}, err + } + } + + var stdin bytes.Buffer + encoded, err := encodeToYamlString(candidate) + if err != nil { + return Context{}, err + } + stdin.WriteString(encoded) + + // #nosec G204 - intentional: user must explicitly enable this operator + cmd := exec.Command(command, args...) + cmd.Stdin = &stdin + var stderr bytes.Buffer + cmd.Stderr = &stderr + + output, err := cmd.Output() + if err != nil { + stderrStr := strings.TrimSpace(stderr.String()) + if stderrStr != "" { + return Context{}, fmt.Errorf("system command '%v' failed: %w\nstderr: %v", command, err, stderrStr) + } + return Context{}, fmt.Errorf("system command '%v' failed: %w", command, err) + } + + result := string(output) + if strings.HasSuffix(result, "\r\n") { + result = result[:len(result)-2] + } else if strings.HasSuffix(result, "\n") { + result = result[:len(result)-1] + } + newNode := candidate.CreateReplacement(ScalarNode, "!!str", result) + results.PushBack(newNode) + } + + return context.ChildContext(results), nil +} diff --git a/pkg/yqlib/operator_system_test.go b/pkg/yqlib/operator_system_test.go new file mode 100644 index 00000000..46a0ecd4 --- /dev/null +++ b/pkg/yqlib/operator_system_test.go @@ -0,0 +1,123 @@ +package yqlib + +import ( + "os/exec" + "testing" +) + +func findExec(t *testing.T, name string) string { + t.Helper() + path, err := exec.LookPath(name) + if err != nil { + t.Skipf("skipping: %v not found: %v", name, err) + } + return path +} + +var systemOperatorDisabledScenarios = []expressionScenario{ + { + description: "system operator returns error when disabled", + subdescription: "Use `--security-enable-system-operator` to enable the system operator.", + document: "country: Australia", + expression: `.country = system("/usr/bin/echo"; "test")`, + expectedError: "system operations are disabled, use --security-enable-system-operator to enable", + }, +} + +func TestSystemOperatorDisabledScenarios(t *testing.T) { + originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps + defer func() { + ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps + }() + + ConfiguredSecurityPreferences.EnableSystemOps = false + + for _, tt := range systemOperatorDisabledScenarios { + testScenario(t, &tt) + } + documentOperatorScenarios(t, "system-operators", systemOperatorDisabledScenarios) +} + +func TestSystemOperatorEnabledScenarios(t *testing.T) { + echoPath := findExec(t, "echo") + falsePath := findExec(t, "false") + + originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps + defer func() { + ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps + }() + + ConfiguredSecurityPreferences.EnableSystemOps = true + + scenarios := []expressionScenario{ + { + description: "Run a command with an argument", + subdescription: "Use `--security-enable-system-operator` to enable the system operator.", + yqFlags: "--security-enable-system-operator", + document: "country: Australia", + expression: `.country = system("` + echoPath + `"; "test")`, + expected: []string{ + "D0, P[], (!!map)::country: test\n", + }, + }, + { + description: "Run a command without arguments", + subdescription: "Omit the semicolon and args to run the command with no extra arguments.", + yqFlags: "--security-enable-system-operator", + document: "a: hello", + expression: `.a = system("` + echoPath + `")`, + expected: []string{ + "D0, P[], (!!map)::a: \"\"\n", + }, + }, + { + description: "Run a command with multiple arguments", + subdescription: "Pass an array of arguments.", + skipDoc: true, + document: "a: hello", + expression: `.a = system("` + echoPath + `"; ["foo", "bar"])`, + expected: []string{ + "D0, P[], (!!map)::a: foo bar\n", + }, + }, + { + description: "Command and args are evaluated per matched node", + skipDoc: true, + document: "cmd: " + echoPath + "\narg: hello", + expression: `.result = system(.cmd; .arg)`, + expected: []string{ + "D0, P[], (!!map)::cmd: " + echoPath + "\narg: hello\nresult: hello\n", + }, + }, + { + description: "Command failure returns error", + skipDoc: true, + document: "a: hello", + expression: `.a = system("` + falsePath + `")`, + expectedError: "system command '" + falsePath + "' failed: exit status 1", + }, + { + description: "Null command returns error", + skipDoc: true, + document: "a: hello", + expression: `.a = system(null)`, + expectedError: "system operator: command must be a string scalar", + }, + { + description: "System operator processes multiple matched nodes", + skipDoc: true, + document: "a: first", + document2: "a: second", + expression: `.a = system("` + echoPath + `"; "replaced")`, + expected: []string{ + "D0, P[], (!!map)::a: replaced\n", + "D0, P[], (!!map)::a: replaced\n", + }, + }, + } + + for _, tt := range scenarios { + testScenario(t, &tt) + } + appendOperatorDocumentScenario(t, "system-operators", scenarios) +} diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index 1646fb9b..b4d4301e 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -31,6 +31,7 @@ type expressionScenario struct { dontFormatInputForDoc bool // dont format input doc for documentation generation requiresFormat string skipForGoccy bool + yqFlags string // extra yq flags to include in generated doc command snippets } var goccyTesting = false @@ -356,14 +357,22 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) { writeOrPanic(w, "then\n") + flagsPrefix := "" + if s.yqFlags != "" { + flagsPrefix = s.yqFlags + " " + } if s.expression != "" { - writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v'%v' %v\n```\n", envCommand, command, strings.ReplaceAll(s.expression, "'", `'\''`), files)) + writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v'%v' %v\n```\n", envCommand, flagsPrefix, command, strings.ReplaceAll(s.expression, "'", `'\''`), files)) } else { - writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v\n```\n", envCommand, command, files)) + writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v%v\n```\n", envCommand, flagsPrefix, command, files)) } } else { writeOrPanic(w, "Running\n") - writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v--null-input '%v'\n```\n", envCommand, command, s.expression)) + flagsPrefix := "" + if s.yqFlags != "" { + flagsPrefix = s.yqFlags + " " + } + writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v--null-input '%v'\n```\n", envCommand, flagsPrefix, command, s.expression)) } return formattedDoc, formattedDoc2 } diff --git a/pkg/yqlib/security_prefs.go b/pkg/yqlib/security_prefs.go index 3e2fe6b4..3c203014 100644 --- a/pkg/yqlib/security_prefs.go +++ b/pkg/yqlib/security_prefs.go @@ -1,11 +1,13 @@ package yqlib type SecurityPreferences struct { - DisableEnvOps bool - DisableFileOps bool + DisableEnvOps bool + DisableFileOps bool + EnableSystemOps bool } var ConfiguredSecurityPreferences = SecurityPreferences{ - DisableEnvOps: false, - DisableFileOps: false, + DisableEnvOps: false, + DisableFileOps: false, + EnableSystemOps: false, } From 8c018da9c94077c03e124208b4e8d70ad6324856 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sun, 12 Apr 2026 14:26:15 +1000 Subject: [PATCH 3/4] Bump go.yaml.in/yaml/v4 from 4.0.0-rc.3 to 4.0.0-rc.4 (#2579) * Bump go.yaml.in/yaml/v4 from 4.0.0-rc.3 to 4.0.0-rc.4 Bumps [go.yaml.in/yaml/v4](https://github.com/yaml/go-yaml) from 4.0.0-rc.3 to 4.0.0-rc.4. - [Commits](https://github.com/yaml/go-yaml/compare/v4.0.0-rc.3...v4.0.0-rc.4) --- updated-dependencies: - dependency-name: go.yaml.in/yaml/v4 dependency-version: 4.0.0-rc.4 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] * Fix test expectations for go.yaml.in/yaml/v4 rc.4 error message changes Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/e172bcc4-f547-4c9f-bcc5-ba61849d37e5 Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- pkg/yqlib/lib_test.go | 2 +- pkg/yqlib/operator_multiply_test.go | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 03e08b2f..8956e7ef 100644 --- a/go.mod +++ b/go.mod @@ -19,7 +19,7 @@ require ( github.com/spf13/pflag v1.0.10 github.com/yuin/gopher-lua v1.1.2 github.com/zclconf/go-cty v1.18.0 - go.yaml.in/yaml/v4 v4.0.0-rc.3 + go.yaml.in/yaml/v4 v4.0.0-rc.4 golang.org/x/mod v0.34.0 golang.org/x/net v0.52.0 golang.org/x/text v0.35.0 diff --git a/go.sum b/go.sum index 5ce25728..98f009a1 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ github.com/zclconf/go-cty v1.18.0/go.mod h1:qpnV6EDNgC1sns/AleL1fvatHw72j+S+nS+M github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo= github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go= -go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= +go.yaml.in/yaml/v4 v4.0.0-rc.4 h1:UP4+v6fFrBIb1l934bDl//mmnoIZEDK0idg1+AIvX5U= +go.yaml.in/yaml/v4 v4.0.0-rc.4/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= diff --git a/pkg/yqlib/lib_test.go b/pkg/yqlib/lib_test.go index 23ed2853..02ca6dbf 100644 --- a/pkg/yqlib/lib_test.go +++ b/pkg/yqlib/lib_test.go @@ -24,7 +24,7 @@ type parseSnippetScenario struct { var parseSnippetScenarios = []parseSnippetScenario{ { snippet: ":", - expectedError: "yaml: did not find expected key", + expectedError: "yaml: while parsing a block mapping at : did not find expected key", }, { snippet: "", diff --git a/pkg/yqlib/operator_multiply_test.go b/pkg/yqlib/operator_multiply_test.go index 2f233eaf..2c31b6f9 100644 --- a/pkg/yqlib/operator_multiply_test.go +++ b/pkg/yqlib/operator_multiply_test.go @@ -553,7 +553,7 @@ var multiplyOperatorScenarios = []expressionScenario{ document: document, expression: `.b * .c`, expected: []string{ - "D0, P[b], (!!map)::{name: dog, <<: *cat}\n", + "D0, P[b], (!!map)::{name: dog, \"<<\": *cat}\n", }, }, { From c47fe40a30ce47b1ac62da4e3b6f893212165722 Mon Sep 17 00:00:00 2001 From: Copilot <198982749+Copilot@users.noreply.github.com> Date: Sun, 12 Apr 2026 14:27:20 +1000 Subject: [PATCH 4/4] Fix TOML encoder to quote keys containing special characters (#2648) * Initial plan * Fix TOML encoder to quote keys with special characters Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/b2b52954-d13f-4e67-831a-16fdd3378de5 Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> * Add test for dotted table section header with special character key Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/12c783dd-8b7f-43bf-b71a-e7a0b5e55fea Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> * Apply De Morgan's law to tomlKey condition to fix staticcheck QF1001 Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/eeab0316-309f-418f-b357-11bbacffb471 Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> --- pkg/yqlib/encoder_toml.go | 46 +++++++++++++++++++++++++++++---------- pkg/yqlib/toml_test.go | 36 ++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 12 deletions(-) diff --git a/pkg/yqlib/encoder_toml.go b/pkg/yqlib/encoder_toml.go index b4ccc287..705154ab 100644 --- a/pkg/yqlib/encoder_toml.go +++ b/pkg/yqlib/encoder_toml.go @@ -69,6 +69,27 @@ func (te *tomlEncoder) CanHandleAliases() bool { // ---- helpers ---- +// tomlKey returns the key quoted if it contains characters that are not valid +// in a TOML bare key. TOML bare keys may only contain ASCII letters, ASCII +// digits, underscores, and dashes. +func tomlKey(key string) string { + for _, r := range key { + if (r < 'A' || r > 'Z') && (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '_' && r != '-' { + return fmt.Sprintf("%q", key) + } + } + return key +} + +// tomlDottedKey joins path components, quoting any that require it. +func tomlDottedKey(path []string) string { + parts := make([]string, len(path)) + for i, p := range path { + parts[i] = tomlKey(p) + } + return strings.Join(parts, ".") +} + func (te *tomlEncoder) writeComment(w io.Writer, comment string) error { if comment == "" { return nil @@ -148,9 +169,10 @@ func (te *tomlEncoder) encodeTopLevelEntry(w io.Writer, path []string, node *Can } if allMaps { key := path[len(path)-1] + quotedKey := tomlKey(key) for _, it := range node.Content { // [[key]] then body - if _, err := w.Write([]byte("[[" + key + "]]\n")); err != nil { + if _, err := w.Write([]byte("[[" + quotedKey + "]]\n")); err != nil { return err } if err := te.encodeMappingBodyWithPath(w, []string{key}, it); err != nil { @@ -185,7 +207,7 @@ func (te *tomlEncoder) writeAttribute(w io.Writer, key string, value *CandidateN } // Write the attribute - line := key + " = " + te.formatScalar(value) + line := tomlKey(key) + " = " + te.formatScalar(value) // Add line comment if present if value.LineComment != "" { @@ -210,7 +232,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida // Handle empty arrays if len(seq.Content) == 0 { - line := key + " = []" + line := tomlKey(key) + " = []" if seq.LineComment != "" { lineComment := strings.TrimSpace(seq.LineComment) if !strings.HasPrefix(lineComment, "#") { @@ -233,7 +255,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida if hasElementComments { // Write multiline array format with comments - if _, err := w.Write([]byte(key + " = [\n")); err != nil { + if _, err := w.Write([]byte(tomlKey(key) + " = [\n")); err != nil { return err } @@ -324,7 +346,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida } } - line := key + " = [" + strings.Join(items, ", ") + "]" + line := tomlKey(key) + " = [" + strings.Join(items, ", ") + "]" // Add line comment if present if seq.LineComment != "" { @@ -372,21 +394,21 @@ func (te *tomlEncoder) mappingToInlineTable(m *CandidateNode) (string, error) { v := m.Content[i+1] switch v.Kind { case ScalarNode: - parts = append(parts, fmt.Sprintf("%s = %s", k, te.formatScalar(v))) + parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), te.formatScalar(v))) case SequenceNode: // inline array in inline table arr, err := te.sequenceToInlineArray(v) if err != nil { return "", err } - parts = append(parts, fmt.Sprintf("%s = %s", k, arr)) + parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), arr)) case MappingNode: // nested inline table inline, err := te.mappingToInlineTable(v) if err != nil { return "", err } - parts = append(parts, fmt.Sprintf("%s = %s", k, inline)) + parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), inline)) default: return "", fmt.Errorf("unsupported inline table value kind: %v", v.Kind) } @@ -399,7 +421,7 @@ func (te *tomlEncoder) writeInlineTableAttribute(w io.Writer, key string, m *Can if err != nil { return err } - _, err = w.Write([]byte(key + " = " + inline + "\n")) + _, err = w.Write([]byte(tomlKey(key) + " = " + inline + "\n")) return err } @@ -421,7 +443,7 @@ func (te *tomlEncoder) writeTableHeader(w io.Writer, path []string, m *Candidate } // Write table header [a.b.c] - header := "[" + strings.Join(path, ".") + "]\n" + header := "[" + tomlDottedKey(path) + "]\n" _, err := w.Write([]byte(header)) return err } @@ -488,7 +510,7 @@ func (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *Cand } } if allMaps { - key := strings.Join(append(append([]string{}, path...), k), ".") + key := tomlDottedKey(append(append([]string{}, path...), k)) for _, it := range v.Content { if _, err := w.Write([]byte("[[" + key + "]]\n")); err != nil { return err @@ -586,7 +608,7 @@ func (te *tomlEncoder) encodeMappingBodyWithPath(w io.Writer, path []string, m * } } if allMaps { - dotted := strings.Join(append(append([]string{}, path...), k), ".") + dotted := tomlDottedKey(append(append([]string{}, path...), k)) for _, it := range v.Content { if _, err := w.Write([]byte("[[" + dotted + "]]\n")); err != nil { return err diff --git a/pkg/yqlib/toml_test.go b/pkg/yqlib/toml_test.go index fe542a01..18be6043 100644 --- a/pkg/yqlib/toml_test.go +++ b/pkg/yqlib/toml_test.go @@ -287,6 +287,18 @@ var expectedSubArrays = `array: - {} ` +// Keys with special characters that require quoting in TOML +var rtSpecialKeyInlineTable = `host = { "http://sealos.hub:5000" = { capabilities = ["pull", "resolve", "push"], skip_verify = true } } +` + +var rtSpecialKeyTableSection = `["/tmp/blah"] +value = "hello" +` + +var rtSpecialKeyDottedTableSection = `[servers."http://localhost:8080"] +ip = "127.0.0.1" +` + var tomlScenarios = []formatScenario{ { skipDoc: true, @@ -614,6 +626,30 @@ var tomlScenarios = []formatScenario{ expected: tomlTableWithComments, scenarioType: "roundtrip", }, + { + skipDoc: true, + description: "Roundtrip: key with special characters in inline table", + input: rtSpecialKeyInlineTable, + expression: ".", + expected: rtSpecialKeyInlineTable, + scenarioType: "roundtrip", + }, + { + skipDoc: true, + description: "Roundtrip: key with special characters in table section", + input: rtSpecialKeyTableSection, + expression: ".", + expected: rtSpecialKeyTableSection, + scenarioType: "roundtrip", + }, + { + skipDoc: true, + description: "Roundtrip: special character key in dotted table section header", + input: rtSpecialKeyDottedTableSection, + expression: ".", + expected: rtSpecialKeyDottedTableSection, + scenarioType: "roundtrip", + }, } func testTomlScenario(t *testing.T, s formatScenario) {