From 9a72c0f780025bc526e3402fca2d7e727d351629 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Fri, 27 Mar 2026 10:14:40 +0000 Subject: [PATCH] Add system(command; args) operator with --enable-system-operator flag Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/8a11e9a0-10d2-4f2a-ae29-4e9d0bfc266f Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com> --- cmd/root.go | 1 + .../doc/operators/headers/system-operators.md | 23 ++++ pkg/yqlib/doc/operators/system-operators.md | 72 +++++++++++++ pkg/yqlib/lexer_participle.go | 2 + pkg/yqlib/operation.go | 2 + pkg/yqlib/operator_system.go | 102 ++++++++++++++++++ pkg/yqlib/operator_system_test.go | 84 +++++++++++++++ pkg/yqlib/security_prefs.go | 10 +- 8 files changed, 292 insertions(+), 4 deletions(-) create mode 100644 pkg/yqlib/doc/operators/headers/system-operators.md create mode 100644 pkg/yqlib/doc/operators/system-operators.md create mode 100644 pkg/yqlib/operator_system.go create mode 100644 pkg/yqlib/operator_system_test.go diff --git a/cmd/root.go b/cmd/root.go index 449be025..18960b51 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -212,6 +212,7 @@ yq -P -oy sample.json rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableEnvOps, "security-disable-env-ops", "", false, "Disable env related operations.") rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableFileOps, "security-disable-file-ops", "", false, "Disable file related operations (e.g. load)") + rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.EnableSystemOps, "enable-system-operator", "", false, "Enable system operator to allow execution of external commands.") rootCmd.AddCommand( createEvaluateSequenceCommand(), diff --git a/pkg/yqlib/doc/operators/headers/system-operators.md b/pkg/yqlib/doc/operators/headers/system-operators.md new file mode 100644 index 00000000..badb4c61 --- /dev/null +++ b/pkg/yqlib/doc/operators/headers/system-operators.md @@ -0,0 +1,23 @@ +# System Operators + +The `system` operator allows you to run an external command and use its output as a value in your expression. + +**Security warning**: The system operator is disabled by default. You must explicitly pass `--enable-system-operator` to use it. + +## Usage + +```bash +yq --enable-system-operator '.field = system("command"; "arg1")' +``` + +The operator takes: +- A command string (required) +- An argument or array of arguments separated by `;` (optional) + +The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string. + +## Disabling the system operator + +The system operator is disabled by default. When disabled, a warning is logged and `null` is returned instead of running the command. + +Use `--enable-system-operator` flag to enable it. diff --git a/pkg/yqlib/doc/operators/system-operators.md b/pkg/yqlib/doc/operators/system-operators.md new file mode 100644 index 00000000..a29acadc --- /dev/null +++ b/pkg/yqlib/doc/operators/system-operators.md @@ -0,0 +1,72 @@ +# System Operators + +The `system` operator allows you to run an external command and use its output as a value in your expression. + +**Security warning**: The system operator is disabled by default. You must explicitly pass `--enable-system-operator` to use it. + +## Usage + +```bash +yq --enable-system-operator '.field = system("command"; "arg1")' +``` + +The operator takes: +- A command string (required) +- An argument or array of arguments separated by `;` (optional) + +The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string. + +## Disabling the system operator + +The system operator is disabled by default. When disabled, a warning is logged and `null` is returned instead of running the command. + +Use `--enable-system-operator` flag to enable it. + +## system operator returns null when disabled +Use `--enable-system-operator` to enable the system operator. + +Given a sample.yml file of: +```yaml +country: Australia +``` +then +```bash +yq '.country = system("/usr/bin/echo"; "test")' sample.yml +``` +will output +```yaml +country: null +``` + +## Run a command with an argument +Use `--enable-system-operator` to enable the system operator. + +Given a sample.yml file of: +```yaml +country: Australia +``` +then +```bash +yq '.country = system("/usr/bin/echo"; "test")' sample.yml +``` +will output +```yaml +country: test +``` + +## Run a command without arguments +Omit the semicolon and args to run the command with no extra arguments. + +Given a sample.yml file of: +```yaml +a: hello +``` +then +```bash +yq '.a = system("/bin/echo")' sample.yml +``` +will output +```yaml +a: "" +``` + diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go index 866b736c..938b8a71 100644 --- a/pkg/yqlib/lexer_participle.go +++ b/pkg/yqlib/lexer_participle.go @@ -96,6 +96,8 @@ var participleYqRules = []*participleYqRule{ simpleOp("load_?str|str_?load", loadStringOpType), {"LoadYaml", `load`, loadOp(NewYamlDecoder(LoadYamlPreferences)), 0}, + simpleOp("system", systemOpType), + {"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0}, simpleOp("select", selectOpType), diff --git a/pkg/yqlib/operation.go b/pkg/yqlib/operation.go index f36054ee..cbea1d7b 100644 --- a/pkg/yqlib/operation.go +++ b/pkg/yqlib/operation.go @@ -164,6 +164,8 @@ var stringInterpolationOpType = &operationType{Type: "STRING_INT", NumArgs: 0, P var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 52, Handler: loadOperator, CheckForPostTraverse: true} var loadStringOpType = &operationType{Type: "LOAD_STRING", NumArgs: 1, Precedence: 52, Handler: loadStringOperator} +var systemOpType = &operationType{Type: "SYSTEM", NumArgs: 1, Precedence: 50, Handler: systemOperator} + var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 52, Handler: keysOperator, CheckForPostTraverse: true} var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator} diff --git a/pkg/yqlib/operator_system.go b/pkg/yqlib/operator_system.go new file mode 100644 index 00000000..e2d71203 --- /dev/null +++ b/pkg/yqlib/operator_system.go @@ -0,0 +1,102 @@ +package yqlib + +import ( + "bytes" + "container/list" + "fmt" + "os/exec" + "strings" +) + +func systemOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { + if !ConfiguredSecurityPreferences.EnableSystemOps { + log.Warning("system operator is disabled, use --enable-system-operator flag to enable") + results := list.New() + for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + results.PushBack(candidate.CreateReplacement(ScalarNode, "!!null", "null")) + } + return context.ChildContext(results), nil + } + + var command string + var argsExpression *ExpressionNode + + // check if it's a block operator (command; args) or just (command) + if expressionNode.RHS.Operation.OperationType == blockOpType { + block := expressionNode.RHS + commandNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), block.LHS) + if err != nil { + return Context{}, err + } + if commandNodes.MatchingNodes.Front() == nil { + return Context{}, fmt.Errorf("system operator: command expression returned no results") + } + command = commandNodes.MatchingNodes.Front().Value.(*CandidateNode).Value + argsExpression = block.RHS + } else { + commandNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.RHS) + if err != nil { + return Context{}, err + } + if commandNodes.MatchingNodes.Front() == nil { + return Context{}, fmt.Errorf("system operator: command expression returned no results") + } + command = commandNodes.MatchingNodes.Front().Value.(*CandidateNode).Value + } + + // evaluate args if present + var args []string + if argsExpression != nil { + argsNodes, err := d.GetMatchingNodes(context.ReadOnlyClone(), argsExpression) + if err != nil { + return Context{}, err + } + if argsNodes.MatchingNodes.Front() != nil { + argsNode := argsNodes.MatchingNodes.Front().Value.(*CandidateNode) + if argsNode.Kind == SequenceNode { + for _, child := range argsNode.Content { + args = append(args, child.Value) + } + } else if argsNode.Tag != "!!null" { + args = []string{argsNode.Value} + } + } + } + + var results = list.New() + + for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + + var stdin bytes.Buffer + if candidate.Tag != "!!null" { + encoded, err := encodeToYamlString(candidate) + if err != nil { + return Context{}, err + } + stdin.WriteString(encoded) + } + + // #nosec G204 - intentional: user must explicitly enable this operator + cmd := exec.Command(command, args...) + cmd.Stdin = &stdin + var stderr bytes.Buffer + cmd.Stderr = &stderr + + output, err := cmd.Output() + if err != nil { + stderrStr := strings.TrimSpace(stderr.String()) + if stderrStr != "" { + return Context{}, fmt.Errorf("system command '%v' failed: %w\nstderr: %v", command, err, stderrStr) + } + return Context{}, fmt.Errorf("system command '%v' failed: %w", command, err) + } + + result := strings.TrimRight(string(output), "\n") + newNode := candidate.CreateReplacement(ScalarNode, "!!str", result) + results.PushBack(newNode) + } + + return context.ChildContext(results), nil +} diff --git a/pkg/yqlib/operator_system_test.go b/pkg/yqlib/operator_system_test.go new file mode 100644 index 00000000..64f1a3f3 --- /dev/null +++ b/pkg/yqlib/operator_system_test.go @@ -0,0 +1,84 @@ +package yqlib + +import ( + "testing" +) + +var systemOperatorDisabledScenarios = []expressionScenario{ + { + description: "system operator returns null when disabled", + subdescription: "Use `--enable-system-operator` to enable the system operator.", + document: "country: Australia", + expression: `.country = system("/usr/bin/echo"; "test")`, + expected: []string{ + "D0, P[], (!!map)::country: null\n", + }, + }, +} + +var systemOperatorEnabledScenarios = []expressionScenario{ + { + description: "Run a command with an argument", + subdescription: "Use `--enable-system-operator` to enable the system operator.", + document: "country: Australia", + expression: `.country = system("/usr/bin/echo"; "test")`, + expected: []string{ + "D0, P[], (!!map)::country: test\n", + }, + }, + { + description: "Run a command without arguments", + subdescription: "Omit the semicolon and args to run the command with no extra arguments.", + document: "a: hello", + expression: `.a = system("/bin/echo")`, + expected: []string{ + "D0, P[], (!!map)::a: \"\"\n", + }, + }, + { + description: "Run a command with multiple arguments", + subdescription: "Pass an array of arguments.", + skipDoc: true, + document: "a: hello", + expression: `.a = system("/bin/echo"; ["foo", "bar"])`, + expected: []string{ + "D0, P[], (!!map)::a: foo bar\n", + }, + }, + { + description: "Command failure returns error", + skipDoc: true, + document: "a: hello", + expression: `.a = system("/bin/false")`, + expectedError: "system command '/bin/false' failed: exit status 1", + }, +} + +func TestSystemOperatorDisabledScenarios(t *testing.T) { + // ensure system operator is disabled + originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps + defer func() { + ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps + }() + + ConfiguredSecurityPreferences.EnableSystemOps = false + + for _, tt := range systemOperatorDisabledScenarios { + testScenario(t, &tt) + } + documentOperatorScenarios(t, "system-operators", systemOperatorDisabledScenarios) +} + +func TestSystemOperatorEnabledScenarios(t *testing.T) { + originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps + defer func() { + ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps + }() + + ConfiguredSecurityPreferences.EnableSystemOps = true + + for _, tt := range systemOperatorEnabledScenarios { + testScenario(t, &tt) + } + appendOperatorDocumentScenario(t, "system-operators", systemOperatorEnabledScenarios) +} diff --git a/pkg/yqlib/security_prefs.go b/pkg/yqlib/security_prefs.go index 3e2fe6b4..3c203014 100644 --- a/pkg/yqlib/security_prefs.go +++ b/pkg/yqlib/security_prefs.go @@ -1,11 +1,13 @@ package yqlib type SecurityPreferences struct { - DisableEnvOps bool - DisableFileOps bool + DisableEnvOps bool + DisableFileOps bool + EnableSystemOps bool } var ConfiguredSecurityPreferences = SecurityPreferences{ - DisableEnvOps: false, - DisableFileOps: false, + DisableEnvOps: false, + DisableFileOps: false, + EnableSystemOps: false, }