From d181cf87d15d421b04bc2aede661c50126c90acf Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 27 Mar 2026 10:03:56 +0000 Subject: [PATCH 1/9] Initial plan From 9a72c0f780025bc526e3402fca2d7e727d351629 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 27 Mar 2026 10:14:40 +0000 Subject: [PATCH 2/9] Add system(command; args) operator with --enable-system-operator flag Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/8a11e9a0-10d2-4f2a-ae29-4e9d0bfc266f Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> --- cmd/root.go | 1 + .../doc/operators/headers/system-operators.md | 23 ++++ pkg/yqlib/doc/operators/system-operators.md | 72 +++++++++++++ pkg/yqlib/lexer_participle.go | 2 + pkg/yqlib/operation.go | 2 + pkg/yqlib/operator_system.go | 102 ++++++++++++++++++ pkg/yqlib/operator_system_test.go | 84 +++++++++++++++ pkg/yqlib/security_prefs.go | 10 +- 8 files changed, 292 insertions(+), 4 deletions(-) create mode 100644 pkg/yqlib/doc/operators/headers/system-operators.md create mode 100644 pkg/yqlib/doc/operators/system-operators.md create mode 100644 pkg/yqlib/operator_system.go create mode 100644 pkg/yqlib/operator_system_test.go diff --git a/cmd/root.go b/cmd/root.go index 449be025..18960b51 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -212,6 +212,7 @@ yq -P -oy sample.json rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableEnvOps, "security-disable-env-ops", "", false, "Disable env related operations.") rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableFileOps, "security-disable-file-ops", "", false, "Disable file related operations (e.g. load)") + rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.EnableSystemOps, "enable-system-operator", "", false, "Enable system operator to allow execution of external commands.") rootCmd.AddCommand( createEvaluateSequenceCommand(), diff --git a/pkg/yqlib/doc/operators/headers/system-operators.md b/pkg/yqlib/doc/operators/headers/system-operators.md new file mode 100644 index 00000000..badb4c61 --- /dev/null +++ b/pkg/yqlib/doc/operators/headers/system-operators.md @@ -0,0 +1,23 @@ +# System Operators + +The `system` operator allows you to run an external command and use its output as a value in your expression. + +**Security warning**: The system operator is disabled by default. You must explicitly pass `--enable-system-operator` to use it. + +## Usage + +```bash +yq --enable-system-operator '.field = system("command"; "arg1")' +``` + +The operator takes: +- A command string (required) +- An argument or array of arguments separated by `;` (optional) + +The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string. + +## Disabling the system operator + +The system operator is disabled by default. When disabled, a warning is logged and `null` is returned instead of running the command. + +Use `--enable-system-operator` flag to enable it. diff --git a/pkg/yqlib/doc/operators/system-operators.md b/pkg/yqlib/doc/operators/system-operators.md new file mode 100644 index 00000000..a29acadc --- /dev/null +++ b/pkg/yqlib/doc/operators/system-operators.md @@ -0,0 +1,72 @@ +# System Operators + +The `system` operator allows you to run an external command and use its output as a value in your expression. + +**Security warning**: The system operator is disabled by default. You must explicitly pass `--enable-system-operator` to use it. + +## Usage + +```bash +yq --enable-system-operator '.field = system("command"; "arg1")' +``` + +The operator takes: +- A command string (required) +- An argument or array of arguments separated by `;` (optional) + +The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string. + +## Disabling the system operator + +The system operator is disabled by default. When disabled, a warning is logged and `null` is returned instead of running the command. + +Use `--enable-system-operator` flag to enable it. + +## system operator returns null when disabled +Use `--enable-system-operator` to enable the system operator. + +Given a sample.yml file of: +```yaml +country: Australia +``` +then +```bash +yq '.country = system("/usr/bin/echo"; "test")' sample.yml +``` +will output +```yaml +country: null +``` + +## Run a command with an argument +Use `--enable-system-operator` to enable the system operator. + +Given a sample.yml file of: +```yaml +country: Australia +``` +then +```bash +yq '.country = system("/usr/bin/echo"; "test")' sample.yml +``` +will output +```yaml +country: test +``` + +## Run a command without arguments +Omit the semicolon and args to run the command with no extra arguments. + +Given a sample.yml file of: +```yaml +a: hello +``` +then +```bash +yq '.a = system("/bin/echo")' sample.yml +``` +will output +```yaml +a: "" +``` + diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go index 866b736c..938b8a71 100644 --- a/pkg/yqlib/lexer_participle.go +++ b/pkg/yqlib/lexer_participle.go @@ -96,6 +96,8 @@ var participleYqRules = []*participleYqRule{ simpleOp("load_?str|str_?load", loadStringOpType), {"LoadYaml", `load`, loadOp(NewYamlDecoder(LoadYamlPreferences)), 0}, + simpleOp("system", systemOpType), + {"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0}, simpleOp("select", selectOpType), diff --git a/pkg/yqlib/operation.go b/pkg/yqlib/operation.go index f36054ee..cbea1d7b 100644 --- a/pkg/yqlib/operation.go +++ b/pkg/yqlib/operation.go @@ -164,6 +164,8 @@ var stringInterpolationOpType = &operationType{Type: "STRING_INT", NumArgs: 0, P var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 52, Handler: loadOperator, CheckForPostTraverse: true} var loadStringOpType = &operationType{Type: "LOAD_STRING", NumArgs: 1, Precedence: 52, Handler: loadStringOperator} +var systemOpType = &operationType{Type: "SYSTEM", NumArgs: 1, Precedence: 50, Handler: systemOperator} + var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 52, Handler: keysOperator, CheckForPostTraverse: true} var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator} diff --git a/pkg/yqlib/operator_system.go b/pkg/yqlib/operator_system.go new file mode 100644 index 00000000..e2d71203 --- /dev/null +++ b/pkg/yqlib/operator_system.go @@ -0,0 +1,102 @@ +package yqlib + +import ( + "bytes" + "container/list" + "fmt" + "os/exec" + "strings" +) + +func systemOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { + if !ConfiguredSecurityPreferences.EnableSystemOps { + log.Warning("system operator is disabled, use --enable-system-operator flag to enable") + results := list.New() + for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + results.PushBack(candidate.CreateReplacement(ScalarNode, "!!null", "null")) + } + return context.ChildContext(results), nil + } + + var command string + var argsExpression *ExpressionNode + + // check if it's a block operator (command; args) or just (command) + if expressionNode.RHS.Operation.OperationType == blockOpType { + block := expressionNode.RHS + commandNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), block.LHS) + if err != nil { + return Context{}, err + } + if commandNodes.MatchingNodes.Front() == nil { + return Context{}, fmt.Errorf("system operator: command expression returned no results") + } + command = commandNodes.MatchingNodes.Front().Value.(*CandidateNode).Value + argsExpression = block.RHS + } else { + commandNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) + if err != nil { + return Context{}, err + } + if commandNodes.MatchingNodes.Front() == nil { + return Context{}, fmt.Errorf("system operator: command expression returned no results") + } + command = commandNodes.MatchingNodes.Front().Value.(*CandidateNode).Value + } + + // evaluate args if present + var args []string + if argsExpression != nil { + argsNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), argsExpression) + if err != nil { + return Context{}, err + } + if argsNodes.MatchingNodes.Front() != nil { + argsNode := argsNodes.MatchingNodes.Front().Value.(*CandidateNode) + if argsNode.Kind == SequenceNode { + for _, child := range argsNode.Content { + args = append(args, child.Value) + } + } else if argsNode.Tag != "!!null" { + args = []string{argsNode.Value} + } + } + } + + var results = list.New() + + for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + + var stdin bytes.Buffer + if candidate.Tag != "!!null" { + encoded, err := encodeToYamlString(candidate) + if err != nil { + return Context{}, err + } + stdin.WriteString(encoded) + } + + // #nosec G204 - intentional: user must explicitly enable this operator + cmd := exec.Command(command, args...) + cmd.Stdin = &stdin + var stderr bytes.Buffer + cmd.Stderr = &stderr + + output, err := cmd.Output() + if err != nil { + stderrStr := strings.TrimSpace(stderr.String()) + if stderrStr != "" { + return Context{}, fmt.Errorf("system command '%v' failed: %w\nstderr: %v", command, err, stderrStr) + } + return Context{}, fmt.Errorf("system command '%v' failed: %w", command, err) + } + + result := strings.TrimRight(string(output), "\n") + newNode := candidate.CreateReplacement(ScalarNode, "!!str", result) + results.PushBack(newNode) + } + + return context.ChildContext(results), nil +} diff --git a/pkg/yqlib/operator_system_test.go b/pkg/yqlib/operator_system_test.go new file mode 100644 index 00000000..64f1a3f3 --- /dev/null +++ b/pkg/yqlib/operator_system_test.go @@ -0,0 +1,84 @@ +package yqlib + +import ( + "testing" +) + +var systemOperatorDisabledScenarios = []expressionScenario{ + { + description: "system operator returns null when disabled", + subdescription: "Use `--enable-system-operator` to enable the system operator.", + document: "country: Australia", + expression: `.country = system("/usr/bin/echo"; "test")`, + expected: []string{ + "D0, P[], (!!map)::country: null\n", + }, + }, +} + +var systemOperatorEnabledScenarios = []expressionScenario{ + { + description: "Run a command with an argument", + subdescription: "Use `--enable-system-operator` to enable the system operator.", + document: "country: Australia", + expression: `.country = system("/usr/bin/echo"; "test")`, + expected: []string{ + "D0, P[], (!!map)::country: test\n", + }, + }, + { + description: "Run a command without arguments", + subdescription: "Omit the semicolon and args to run the command with no extra arguments.", + document: "a: hello", + expression: `.a = system("/bin/echo")`, + expected: []string{ + "D0, P[], (!!map)::a: \"\"\n", + }, + }, + { + description: "Run a command with multiple arguments", + subdescription: "Pass an array of arguments.", + skipDoc: true, + document: "a: hello", + expression: `.a = system("/bin/echo"; ["foo", "bar"])`, + expected: []string{ + "D0, P[], (!!map)::a: foo bar\n", + }, + }, + { + description: "Command failure returns error", + skipDoc: true, + document: "a: hello", + expression: `.a = system("/bin/false")`, + expectedError: "system command '/bin/false' failed: exit status 1", + }, +} + +func TestSystemOperatorDisabledScenarios(t *testing.T) { + // ensure system operator is disabled + originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps + defer func() { + ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps + }() + + ConfiguredSecurityPreferences.EnableSystemOps = false + + for _, tt := range systemOperatorDisabledScenarios { + testScenario(t, &tt) + } + documentOperatorScenarios(t, "system-operators", systemOperatorDisabledScenarios) +} + +func TestSystemOperatorEnabledScenarios(t *testing.T) { + originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps + defer func() { + ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps + }() + + ConfiguredSecurityPreferences.EnableSystemOps = true + + for _, tt := range systemOperatorEnabledScenarios { + testScenario(t, &tt) + } + appendOperatorDocumentScenario(t, "system-operators", systemOperatorEnabledScenarios) +} diff --git a/pkg/yqlib/security_prefs.go b/pkg/yqlib/security_prefs.go index 3e2fe6b4..3c203014 100644 --- a/pkg/yqlib/security_prefs.go +++ b/pkg/yqlib/security_prefs.go @@ -1,11 +1,13 @@ package yqlib type SecurityPreferences struct { - DisableEnvOps bool - DisableFileOps bool + DisableEnvOps bool + DisableFileOps bool + EnableSystemOps bool } var ConfiguredSecurityPreferences = SecurityPreferences{ - DisableEnvOps: false, - DisableFileOps: false, + DisableEnvOps: false, + DisableFileOps: false, + EnableSystemOps: false, } From 6315cfe974c7e9390ccee2f85da98a55b51bce0c Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sat, 28 Mar 2026 12:59:37 +1100 Subject: [PATCH 3/9] Update pkg/yqlib/operator_system.go Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- pkg/yqlib/operator_system.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/yqlib/operator_system.go b/pkg/yqlib/operator_system.go index e2d71203..f7173939 100644 --- a/pkg/yqlib/operator_system.go +++ b/pkg/yqlib/operator_system.go @@ -93,7 +93,12 @@ func systemOperator(d *dataTreeNavigator, context Context, expressionNode *Expre return Context{}, fmt.Errorf("system command '%v' failed: %w", command, err) } - result := strings.TrimRight(string(output), "\n") + result := string(output) + if strings.HasSuffix(result, "\r\n") { + result = result[:len(result)-2] + } else if strings.HasSuffix(result, "\n") { + result = result[:len(result)-1] + } newNode := candidate.CreateReplacement(ScalarNode, "!!str", result) results.PushBack(newNode) } From da611f7a2b051ed2f25c3cd62e79d78909b3410b Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 28 Mar 2026 02:03:15 +0000 Subject: [PATCH 4/9] Evaluate system command/args per matched node using SingleReadonlyChildContext Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/dca841eb-3f63-4f23-adeb-556431560420 Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> --- pkg/yqlib/doc/operators/system-operators.md | 2 +- pkg/yqlib/operator_system.go | 93 +++++++++--------- pkg/yqlib/operator_system_test.go | 103 ++++++++++++-------- 3 files changed, 112 insertions(+), 86 deletions(-) diff --git a/pkg/yqlib/doc/operators/system-operators.md b/pkg/yqlib/doc/operators/system-operators.md index a29acadc..f9c0346c 100644 --- a/pkg/yqlib/doc/operators/system-operators.md +++ b/pkg/yqlib/doc/operators/system-operators.md @@ -63,7 +63,7 @@ a: hello ``` then ```bash -yq '.a = system("/bin/echo")' sample.yml +yq '.a = system("/usr/bin/echo")' sample.yml ``` will output ```yaml diff --git a/pkg/yqlib/operator_system.go b/pkg/yqlib/operator_system.go index f7173939..f2e8c9b7 100644 --- a/pkg/yqlib/operator_system.go +++ b/pkg/yqlib/operator_system.go @@ -8,6 +8,20 @@ import ( "strings" ) +func resolveSystemArgs(argsNode *CandidateNode) []string { + if argsNode.Kind == SequenceNode { + args := make([]string, 0, len(argsNode.Content)) + for _, child := range argsNode.Content { + args = append(args, child.Value) + } + return args + } + if argsNode.Tag != "!!null" { + return []string{argsNode.Value} + } + return nil +} + func systemOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { if !ConfiguredSecurityPreferences.EnableSystemOps { log.Warning("system operator is disabled, use --enable-system-operator flag to enable") @@ -19,55 +33,46 @@ func systemOperator(d *dataTreeNavigator, context Context, expressionNode *Expre return context.ChildContext(results), nil } - var command string - var argsExpression *ExpressionNode - - // check if it's a block operator (command; args) or just (command) - if expressionNode.RHS.Operation.OperationType == blockOpType { - block := expressionNode.RHS - commandNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), block.LHS) - if err != nil { - return Context{}, err - } - if commandNodes.MatchingNodes.Front() == nil { - return Context{}, fmt.Errorf("system operator: command expression returned no results") - } - command = commandNodes.MatchingNodes.Front().Value.(*CandidateNode).Value - argsExpression = block.RHS - } else { - commandNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) - if err != nil { - return Context{}, err - } - if commandNodes.MatchingNodes.Front() == nil { - return Context{}, fmt.Errorf("system operator: command expression returned no results") - } - command = commandNodes.MatchingNodes.Front().Value.(*CandidateNode).Value - } - - // evaluate args if present - var args []string - if argsExpression != nil { - argsNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), argsExpression) - if err != nil { - return Context{}, err - } - if argsNodes.MatchingNodes.Front() != nil { - argsNode := argsNodes.MatchingNodes.Front().Value.(*CandidateNode) - if argsNode.Kind == SequenceNode { - for _, child := range argsNode.Content { - args = append(args, child.Value) - } - } else if argsNode.Tag != "!!null" { - args = []string{argsNode.Value} - } - } - } + // determine at parse time whether we have (command; args) or just (command) + hasArgs := expressionNode.RHS.Operation.OperationType == blockOpType var results = list.New() for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) + nodeContext := context.SingleReadonlyChildContext(candidate) + + var command string + var args []string + + if hasArgs { + block := expressionNode.RHS + commandNodes, err := d.GetMatchingNodes(nodeContext, block.LHS) + if err != nil { + return Context{}, err + } + if commandNodes.MatchingNodes.Front() == nil { + return Context{}, fmt.Errorf("system operator: command expression returned no results") + } + command = commandNodes.MatchingNodes.Front().Value.(*CandidateNode).Value + + argsNodes, err := d.GetMatchingNodes(nodeContext, block.RHS) + if err != nil { + return Context{}, err + } + if argsNodes.MatchingNodes.Front() != nil { + args = resolveSystemArgs(argsNodes.MatchingNodes.Front().Value.(*CandidateNode)) + } + } else { + commandNodes, err := d.GetMatchingNodes(nodeContext, expressionNode.RHS) + if err != nil { + return Context{}, err + } + if commandNodes.MatchingNodes.Front() == nil { + return Context{}, fmt.Errorf("system operator: command expression returned no results") + } + command = commandNodes.MatchingNodes.Front().Value.(*CandidateNode).Value + } var stdin bytes.Buffer if candidate.Tag != "!!null" { diff --git a/pkg/yqlib/operator_system_test.go b/pkg/yqlib/operator_system_test.go index 64f1a3f3..388f4446 100644 --- a/pkg/yqlib/operator_system_test.go +++ b/pkg/yqlib/operator_system_test.go @@ -1,9 +1,19 @@ package yqlib import ( + "os/exec" "testing" ) +func findExec(t *testing.T, name string) string { + t.Helper() + path, err := exec.LookPath(name) + if err != nil { + t.Skipf("skipping: %v not found: %v", name, err) + } + return path +} + var systemOperatorDisabledScenarios = []expressionScenario{ { description: "system operator returns null when disabled", @@ -16,46 +26,7 @@ var systemOperatorDisabledScenarios = []expressionScenario{ }, } -var systemOperatorEnabledScenarios = []expressionScenario{ - { - description: "Run a command with an argument", - subdescription: "Use `--enable-system-operator` to enable the system operator.", - document: "country: Australia", - expression: `.country = system("/usr/bin/echo"; "test")`, - expected: []string{ - "D0, P[], (!!map)::country: test\n", - }, - }, - { - description: "Run a command without arguments", - subdescription: "Omit the semicolon and args to run the command with no extra arguments.", - document: "a: hello", - expression: `.a = system("/bin/echo")`, - expected: []string{ - "D0, P[], (!!map)::a: \"\"\n", - }, - }, - { - description: "Run a command with multiple arguments", - subdescription: "Pass an array of arguments.", - skipDoc: true, - document: "a: hello", - expression: `.a = system("/bin/echo"; ["foo", "bar"])`, - expected: []string{ - "D0, P[], (!!map)::a: foo bar\n", - }, - }, - { - description: "Command failure returns error", - skipDoc: true, - document: "a: hello", - expression: `.a = system("/bin/false")`, - expectedError: "system command '/bin/false' failed: exit status 1", - }, -} - func TestSystemOperatorDisabledScenarios(t *testing.T) { - // ensure system operator is disabled originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps defer func() { ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps @@ -70,6 +41,9 @@ func TestSystemOperatorDisabledScenarios(t *testing.T) { } func TestSystemOperatorEnabledScenarios(t *testing.T) { + echoPath := findExec(t, "echo") + falsePath := findExec(t, "false") + originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps defer func() { ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps @@ -77,8 +51,55 @@ func TestSystemOperatorEnabledScenarios(t *testing.T) { ConfiguredSecurityPreferences.EnableSystemOps = true - for _, tt := range systemOperatorEnabledScenarios { + scenarios := []expressionScenario{ + { + description: "Run a command with an argument", + subdescription: "Use `--enable-system-operator` to enable the system operator.", + document: "country: Australia", + expression: `.country = system("` + echoPath + `"; "test")`, + expected: []string{ + "D0, P[], (!!map)::country: test\n", + }, + }, + { + description: "Run a command without arguments", + subdescription: "Omit the semicolon and args to run the command with no extra arguments.", + document: "a: hello", + expression: `.a = system("` + echoPath + `")`, + expected: []string{ + "D0, P[], (!!map)::a: \"\"\n", + }, + }, + { + description: "Run a command with multiple arguments", + subdescription: "Pass an array of arguments.", + skipDoc: true, + document: "a: hello", + expression: `.a = system("` + echoPath + `"; ["foo", "bar"])`, + expected: []string{ + "D0, P[], (!!map)::a: foo bar\n", + }, + }, + { + description: "Command and args are evaluated per matched node", + skipDoc: true, + document: "cmd: " + echoPath + "\narg: hello", + expression: `.result = system(.cmd; .arg)`, + expected: []string{ + "D0, P[], (!!map)::cmd: " + echoPath + "\narg: hello\nresult: hello\n", + }, + }, + { + description: "Command failure returns error", + skipDoc: true, + document: "a: hello", + expression: `.a = system("` + falsePath + `")`, + expectedError: "system command '" + falsePath + "' failed: exit status 1", + }, + } + + for _, tt := range scenarios { testScenario(t, &tt) } - appendOperatorDocumentScenario(t, "system-operators", systemOperatorEnabledScenarios) + appendOperatorDocumentScenario(t, "system-operators", scenarios) } From 884c2d8b6b7dd8c8ff4b0ad1caf429ed1234540a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 28 Mar 2026 02:08:14 +0000 Subject: [PATCH 5/9] Add yqFlags to expressionScenario for doc command snippets; fix system op docs Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/3f8a5375-25fd-4428-a8e6-b630194c36b2 Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> --- pkg/yqlib/doc/operators/system-operators.md | 4 ++-- pkg/yqlib/operator_system_test.go | 2 ++ pkg/yqlib/operators_test.go | 15 ++++++++++++--- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/pkg/yqlib/doc/operators/system-operators.md b/pkg/yqlib/doc/operators/system-operators.md index f9c0346c..0223acaa 100644 --- a/pkg/yqlib/doc/operators/system-operators.md +++ b/pkg/yqlib/doc/operators/system-operators.md @@ -47,7 +47,7 @@ country: Australia ``` then ```bash -yq '.country = system("/usr/bin/echo"; "test")' sample.yml +yq --enable-system-operator '.country = system("/usr/bin/echo"; "test")' sample.yml ``` will output ```yaml @@ -63,7 +63,7 @@ a: hello ``` then ```bash -yq '.a = system("/usr/bin/echo")' sample.yml +yq --enable-system-operator '.a = system("/usr/bin/echo")' sample.yml ``` will output ```yaml diff --git a/pkg/yqlib/operator_system_test.go b/pkg/yqlib/operator_system_test.go index 388f4446..385e1561 100644 --- a/pkg/yqlib/operator_system_test.go +++ b/pkg/yqlib/operator_system_test.go @@ -55,6 +55,7 @@ func TestSystemOperatorEnabledScenarios(t *testing.T) { { description: "Run a command with an argument", subdescription: "Use `--enable-system-operator` to enable the system operator.", + yqFlags: "--enable-system-operator", document: "country: Australia", expression: `.country = system("` + echoPath + `"; "test")`, expected: []string{ @@ -64,6 +65,7 @@ func TestSystemOperatorEnabledScenarios(t *testing.T) { { description: "Run a command without arguments", subdescription: "Omit the semicolon and args to run the command with no extra arguments.", + yqFlags: "--enable-system-operator", document: "a: hello", expression: `.a = system("` + echoPath + `")`, expected: []string{ diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index 1646fb9b..b4d4301e 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -31,6 +31,7 @@ type expressionScenario struct { dontFormatInputForDoc bool // dont format input doc for documentation generation requiresFormat string skipForGoccy bool + yqFlags string // extra yq flags to include in generated doc command snippets } var goccyTesting = false @@ -356,14 +357,22 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) { writeOrPanic(w, "then\n") + flagsPrefix := "" + if s.yqFlags != "" { + flagsPrefix = s.yqFlags + " " + } if s.expression != "" { - writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v'%v' %v\n```\n", envCommand, command, strings.ReplaceAll(s.expression, "'", `'\''`), files)) + writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v'%v' %v\n```\n", envCommand, flagsPrefix, command, strings.ReplaceAll(s.expression, "'", `'\''`), files)) } else { - writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v\n```\n", envCommand, command, files)) + writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v%v\n```\n", envCommand, flagsPrefix, command, files)) } } else { writeOrPanic(w, "Running\n") - writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v--null-input '%v'\n```\n", envCommand, command, s.expression)) + flagsPrefix := "" + if s.yqFlags != "" { + flagsPrefix = s.yqFlags + " " + } + writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v--null-input '%v'\n```\n", envCommand, flagsPrefix, command, s.expression)) } return formattedDoc, formattedDoc2 } From 2c8605f63425230d39d1f868a07a01a73ea8dc83 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sat, 28 Mar 2026 14:21:28 +1100 Subject: [PATCH 6/9] Update pkg/yqlib/doc/operators/headers/system-operators.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- pkg/yqlib/doc/operators/headers/system-operators.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/yqlib/doc/operators/headers/system-operators.md b/pkg/yqlib/doc/operators/headers/system-operators.md index badb4c61..2df8fdaa 100644 --- a/pkg/yqlib/doc/operators/headers/system-operators.md +++ b/pkg/yqlib/doc/operators/headers/system-operators.md @@ -7,7 +7,7 @@ The `system` operator allows you to run an external command and use its output a ## Usage ```bash -yq --enable-system-operator '.field = system("command"; "arg1")' +yq --null-input --enable-system-operator '.field = system("command"; "arg1")' ``` The operator takes: From 5ea069a5ede7faf012fd71a6672171057e78b829 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sat, 28 Mar 2026 14:21:39 +1100 Subject: [PATCH 7/9] Update pkg/yqlib/doc/operators/system-operators.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- pkg/yqlib/doc/operators/system-operators.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/yqlib/doc/operators/system-operators.md b/pkg/yqlib/doc/operators/system-operators.md index 0223acaa..765ae940 100644 --- a/pkg/yqlib/doc/operators/system-operators.md +++ b/pkg/yqlib/doc/operators/system-operators.md @@ -7,7 +7,7 @@ The `system` operator allows you to run an external command and use its output a ## Usage ```bash -yq --enable-system-operator '.field = system("command"; "arg1")' +yq --enable-system-operator --null-input '.field = system("command"; "arg1")' ``` The operator takes: From 53abbbaee95f63330419411474ba8e8b6d558ccc Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Sat, 28 Mar 2026 03:27:46 +0000 Subject: [PATCH 8/9] Validate command node type and handle multiple results with debug log Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/928aabc5-ad71-41d8-94ab-403942e3f92d Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> --- .../doc/operators/headers/system-operators.md | 2 +- pkg/yqlib/operator_system.go | 26 ++++++++++++++----- pkg/yqlib/operator_system_test.go | 7 +++++ 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/pkg/yqlib/doc/operators/headers/system-operators.md b/pkg/yqlib/doc/operators/headers/system-operators.md index 2df8fdaa..9b84e8df 100644 --- a/pkg/yqlib/doc/operators/headers/system-operators.md +++ b/pkg/yqlib/doc/operators/headers/system-operators.md @@ -7,7 +7,7 @@ The `system` operator allows you to run an external command and use its output a ## Usage ```bash -yq --null-input --enable-system-operator '.field = system("command"; "arg1")' +yq --enable-system-operator --null-input '.field = system("command"; "arg1")' ``` The operator takes: diff --git a/pkg/yqlib/operator_system.go b/pkg/yqlib/operator_system.go index f2e8c9b7..2fb58869 100644 --- a/pkg/yqlib/operator_system.go +++ b/pkg/yqlib/operator_system.go @@ -22,6 +22,20 @@ func resolveSystemArgs(argsNode *CandidateNode) []string { return nil } +func resolveCommandNode(commandNodes Context) (string, error) { + if commandNodes.MatchingNodes.Front() == nil { + return "", fmt.Errorf("system operator: command expression returned no results") + } + if commandNodes.MatchingNodes.Len() > 1 { + log.Debugf("system operator: command expression returned %d results, using first", commandNodes.MatchingNodes.Len()) + } + cmdNode := commandNodes.MatchingNodes.Front().Value.(*CandidateNode) + if cmdNode.Kind != ScalarNode || cmdNode.Tag == "!!null" { + return "", fmt.Errorf("system operator: command must be a string scalar") + } + return cmdNode.Value, nil +} + func systemOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { if !ConfiguredSecurityPreferences.EnableSystemOps { log.Warning("system operator is disabled, use --enable-system-operator flag to enable") @@ -51,10 +65,10 @@ func systemOperator(d *dataTreeNavigator, context Context, expressionNode *Expre if err != nil { return Context{}, err } - if commandNodes.MatchingNodes.Front() == nil { - return Context{}, fmt.Errorf("system operator: command expression returned no results") + command, err = resolveCommandNode(commandNodes) + if err != nil { + return Context{}, err } - command = commandNodes.MatchingNodes.Front().Value.(*CandidateNode).Value argsNodes, err := d.GetMatchingNodes(nodeContext, block.RHS) if err != nil { @@ -68,10 +82,10 @@ func systemOperator(d *dataTreeNavigator, context Context, expressionNode *Expre if err != nil { return Context{}, err } - if commandNodes.MatchingNodes.Front() == nil { - return Context{}, fmt.Errorf("system operator: command expression returned no results") + command, err = resolveCommandNode(commandNodes) + if err != nil { + return Context{}, err } - command = commandNodes.MatchingNodes.Front().Value.(*CandidateNode).Value } var stdin bytes.Buffer diff --git a/pkg/yqlib/operator_system_test.go b/pkg/yqlib/operator_system_test.go index 385e1561..867e1921 100644 --- a/pkg/yqlib/operator_system_test.go +++ b/pkg/yqlib/operator_system_test.go @@ -98,6 +98,13 @@ func TestSystemOperatorEnabledScenarios(t *testing.T) { expression: `.a = system("` + falsePath + `")`, expectedError: "system command '" + falsePath + "' failed: exit status 1", }, + { + description: "Null command returns error", + skipDoc: true, + document: "a: hello", + expression: `.a = system(null)`, + expectedError: "system operator: command must be a string scalar", + }, } for _, tt := range scenarios { From e10e8127e1cb4986691c70aa7c7164e7cc75b81e Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sun, 29 Mar 2026 09:45:08 +1100 Subject: [PATCH 9/9] Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- cmd/root.go | 6 +++++- pkg/yqlib/operator_system.go | 13 +++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index 18960b51..3dfe18e6 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -212,7 +212,11 @@ yq -P -oy sample.json rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableEnvOps, "security-disable-env-ops", "", false, "Disable env related operations.") rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableFileOps, "security-disable-file-ops", "", false, "Disable file related operations (e.g. load)") - rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.EnableSystemOps, "enable-system-operator", "", false, "Enable system operator to allow execution of external commands.") + rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.EnableSystemOps, "security-enable-system-operator", "", false, "Enable system operator to allow execution of external commands.") + rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.EnableSystemOps, "enable-system-operator", "", false, "DEPRECATED: use --security-enable-system-operator instead. Enable system operator to allow execution of external commands.") + if err = rootCmd.MarkPersistentFlagDeprecated("enable-system-operator", "use --security-enable-system-operator instead."); err != nil { + panic(err) + } rootCmd.AddCommand( createEvaluateSequenceCommand(), diff --git a/pkg/yqlib/operator_system.go b/pkg/yqlib/operator_system.go index 2fb58869..8a37d274 100644 --- a/pkg/yqlib/operator_system.go +++ b/pkg/yqlib/operator_system.go @@ -33,6 +33,9 @@ func resolveCommandNode(commandNodes Context) (string, error) { if cmdNode.Kind != ScalarNode || cmdNode.Tag == "!!null" { return "", fmt.Errorf("system operator: command must be a string scalar") } + if cmdNode.Value == "" { + return "", fmt.Errorf("system operator: command must be a non-empty string") + } return cmdNode.Value, nil } @@ -89,13 +92,11 @@ func systemOperator(d *dataTreeNavigator, context Context, expressionNode *Expre } var stdin bytes.Buffer - if candidate.Tag != "!!null" { - encoded, err := encodeToYamlString(candidate) - if err != nil { - return Context{}, err - } - stdin.WriteString(encoded) + encoded, err := encodeToYamlString(candidate) + if err != nil { + return Context{}, err } + stdin.WriteString(encoded) // #nosec G204 - intentional: user must explicitly enable this operator cmd := exec.Command(command, args...)