Add system(command; args) operator (disabled by default) (#2640)

* Initial plan

* Add system(command; args) operator with --enable-system-operator flag

Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/8a11e9a0-10d2-4f2a-ae29-4e9d0bfc266f

Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com>

* Update pkg/yqlib/operator_system.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Evaluate system command/args per matched node using SingleReadonlyChildContext

Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/dca841eb-3f63-4f23-adeb-556431560420

Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com>

* Add yqFlags to expressionScenario for doc command snippets; fix system op docs

Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/3f8a5375-25fd-4428-a8e6-b630194c36b2

Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com>

* Update pkg/yqlib/doc/operators/headers/system-operators.md

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update pkg/yqlib/doc/operators/system-operators.md

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Validate command node type and handle multiple results with debug log

Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/928aabc5-ad71-41d8-94ab-403942e3f92d

Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com>

* Apply suggestions from code review

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Remove deprecated --enable-system-operator alias; use --security-enable-system-operator consistently

Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/286b95e9-b6d7-4ab8-b401-2d7a03853922

Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com>

* Apply suggestions from code review

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Address deep review feedback: error on disabled, strict arg/cmd validation, debug logs, docs

Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/fbfba2db-60ea-4c20-a4c2-0fd396b80c81

Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com>
Co-authored-by: Mike Farah <mikefarah@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

cmd/root.go

@@ -212,6 +212,7 @@ yq -P -oy sample.json
 
 	rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableEnvOps, "security-disable-env-ops", "", false, "Disable env related operations.")
 	rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableFileOps, "security-disable-file-ops", "", false, "Disable file related operations (e.g. load)")
+	rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.EnableSystemOps, "security-enable-system-operator", "", false, "Enable system operator to allow execution of external commands.")
 
 	rootCmd.AddCommand(
 		createEvaluateSequenceCommand(),

pkg/yqlib/doc/operators/headers/system-operators.md

@@ -0,0 +1,27 @@
+# System Operators
+
+The `system` operator allows you to run an external command and use its output as a value in your expression.
+
+**Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it.
+
+**Note:** When enabled, the system operator can replicate the functionality of `env` and `load`
+operators via external commands. Enabling it effectively overrides `--security-disable-env-ops`
+and `--security-disable-file-ops`.
+
+## Usage
+
+```bash
+yq --security-enable-system-operator --null-input '.field = system("command"; "arg1")'
+```
+
+The operator takes:
+- A command string (required)
+- An argument (or an array of arguments), separated from the command by `;` (optional)
+
+The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string.
+
+## Disabling the system operator
+
+The system operator is disabled by default. When disabled, an error is returned instead of running the command, consistent with `--security-disable-env-ops` and `--security-disable-file-ops`.
+
+Use `--security-enable-system-operator` flag to enable it.

pkg/yqlib/doc/operators/system-operators.md

@@ -0,0 +1,76 @@
+# System Operators
+
+The `system` operator allows you to run an external command and use its output as a value in your expression.
+
+**Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it.
+
+**Note:** When enabled, the system operator can replicate the functionality of `env` and `load`
+operators via external commands. Enabling it effectively overrides `--security-disable-env-ops`
+and `--security-disable-file-ops`.
+
+## Usage
+
+```bash
+yq --security-enable-system-operator --null-input '.field = system("command"; "arg1")'
+```
+
+The operator takes:
+- A command string (required)
+- An argument (or an array of arguments), separated from the command by `;` (optional)
+
+The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string.
+
+## Disabling the system operator
+
+The system operator is disabled by default. When disabled, an error is returned instead of running the command, consistent with `--security-disable-env-ops` and `--security-disable-file-ops`.
+
+Use `--security-enable-system-operator` flag to enable it.
+
+## system operator returns error when disabled
+Use `--security-enable-system-operator` to enable the system operator.
+
+Given a sample.yml file of:
+```yaml
+country: Australia
+```
+then
+```bash
+yq '.country = system("/usr/bin/echo"; "test")' sample.yml
+```
+will output
+```bash
+Error: system operations are disabled, use --security-enable-system-operator to enable
+```
+
+## Run a command with an argument
+Use `--security-enable-system-operator` to enable the system operator.
+
+Given a sample.yml file of:
+```yaml
+country: Australia
+```
+then
+```bash
+yq --security-enable-system-operator '.country = system("/usr/bin/echo"; "test")' sample.yml
+```
+will output
+```yaml
+country: test
+```
+
+## Run a command without arguments
+Omit the semicolon and args to run the command with no extra arguments.
+
+Given a sample.yml file of:
+```yaml
+a: hello
+```
+then
+```bash
+yq --security-enable-system-operator '.a = system("/usr/bin/echo")' sample.yml
+```
+will output
+```yaml
+a: ""
+```
+

pkg/yqlib/lexer_participle.go

@@ -96,6 +96,8 @@ var participleYqRules = []*participleYqRule{
 	simpleOp("load_?str|str_?load", loadStringOpType),
 	{"LoadYaml", `load`, loadOp(NewYamlDecoder(LoadYamlPreferences)), 0},
 
+	simpleOp("system", systemOpType),
+
 	{"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0},
 
 	simpleOp("select", selectOpType),

pkg/yqlib/operation.go

@@ -164,6 +164,8 @@ var stringInterpolationOpType = &operationType{Type: "STRING_INT", NumArgs: 0, P
 var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 52, Handler: loadOperator, CheckForPostTraverse: true}
 var loadStringOpType = &operationType{Type: "LOAD_STRING", NumArgs: 1, Precedence: 52, Handler: loadStringOperator}
 
+var systemOpType = &operationType{Type: "SYSTEM", NumArgs: 1, Precedence: 50, Handler: systemOperator}
+
 var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 52, Handler: keysOperator, CheckForPostTraverse: true}
 
 var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator}

pkg/yqlib/operator_system.go

@@ -0,0 +1,146 @@
+package yqlib
+
+import (
+	"bytes"
+	"container/list"
+	"fmt"
+	"os/exec"
+	"strings"
+)
+
+func resolveSystemArgs(argsNode *CandidateNode) ([]string, error) {
+	if argsNode == nil {
+		return nil, nil
+	}
+
+	if argsNode.Kind == SequenceNode {
+		args := make([]string, 0, len(argsNode.Content))
+		for _, child := range argsNode.Content {
+			// Only non-null scalar children are valid arguments.
+			if child == nil {
+				continue
+			}
+			if child.Kind != ScalarNode || child.Tag == "!!null" {
+				return nil, fmt.Errorf("system operator: argument must be a non-null scalar; got kind=%v tag=%v", child.Kind, child.Tag)
+			}
+			args = append(args, child.Value)
+		}
+		if len(args) == 0 {
+			return nil, nil
+		}
+		return args, nil
+	}
+
+	// Single-argument case: only accept a non-null scalar node.
+	if argsNode.Tag == "!!null" {
+		return nil, nil
+	}
+	if argsNode.Kind != ScalarNode {
+		return nil, fmt.Errorf("system operator: args must be a non-null scalar or sequence of non-null scalars; got kind=%v tag=%v", argsNode.Kind, argsNode.Tag)
+	}
+	return []string{argsNode.Value}, nil
+}
+
+func resolveCommandNode(commandNodes Context) (string, error) {
+	if commandNodes.MatchingNodes.Front() == nil {
+		return "", fmt.Errorf("system operator: command expression returned no results")
+	}
+	if commandNodes.MatchingNodes.Len() > 1 {
+		log.Debugf("system operator: command expression returned %d results, using first", commandNodes.MatchingNodes.Len())
+	}
+	cmdNode := commandNodes.MatchingNodes.Front().Value.(*CandidateNode)
+	if cmdNode.Kind != ScalarNode || cmdNode.guessTagFromCustomType() != "!!str" {
+		return "", fmt.Errorf("system operator: command must be a string scalar")
+	}
+	if cmdNode.Value == "" {
+		return "", fmt.Errorf("system operator: command must be a non-empty string")
+	}
+	return cmdNode.Value, nil
+}
+
+func systemOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	if !ConfiguredSecurityPreferences.EnableSystemOps {
+		return Context{}, fmt.Errorf("system operations are disabled, use --security-enable-system-operator to enable")
+	}
+
+	// determine at parse time whether we have (command; args) or just (command)
+	hasArgs := expressionNode.RHS.Operation.OperationType == blockOpType
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		nodeContext := context.SingleReadonlyChildContext(candidate)
+
+		var command string
+		var args []string
+
+		if hasArgs {
+			block := expressionNode.RHS
+			commandNodes, err := d.GetMatchingNodes(nodeContext, block.LHS)
+			if err != nil {
+				return Context{}, err
+			}
+			command, err = resolveCommandNode(commandNodes)
+			if err != nil {
+				return Context{}, err
+			}
+
+			argsNodes, err := d.GetMatchingNodes(nodeContext, block.RHS)
+			if err != nil {
+				return Context{}, err
+			}
+			if argsNodes.MatchingNodes.Len() > 1 {
+				log.Debugf("system operator: args expression returned %d results, using first", argsNodes.MatchingNodes.Len())
+			}
+			if argsNodes.MatchingNodes.Front() != nil {
+				args, err = resolveSystemArgs(argsNodes.MatchingNodes.Front().Value.(*CandidateNode))
+				if err != nil {
+					return Context{}, err
+				}
+			}
+		} else {
+			commandNodes, err := d.GetMatchingNodes(nodeContext, expressionNode.RHS)
+			if err != nil {
+				return Context{}, err
+			}
+			command, err = resolveCommandNode(commandNodes)
+			if err != nil {
+				return Context{}, err
+			}
+		}
+
+		var stdin bytes.Buffer
+		encoded, err := encodeToYamlString(candidate)
+		if err != nil {
+			return Context{}, err
+		}
+		stdin.WriteString(encoded)
+
+		// #nosec G204 - intentional: user must explicitly enable this operator
+		cmd := exec.Command(command, args...)
+		cmd.Stdin = &stdin
+		var stderr bytes.Buffer
+		cmd.Stderr = &stderr
+
+		output, err := cmd.Output()
+		if err != nil {
+			stderrStr := strings.TrimSpace(stderr.String())
+			if stderrStr != "" {
+				return Context{}, fmt.Errorf("system command '%v' failed: %w\nstderr: %v", command, err, stderrStr)
+			}
+			return Context{}, fmt.Errorf("system command '%v' failed: %w", command, err)
+		}
+
+		result := string(output)
+		if strings.HasSuffix(result, "\r\n") {
+			result = result[:len(result)-2]
+		} else if strings.HasSuffix(result, "\n") {
+			result = result[:len(result)-1]
+		}
+		newNode := candidate.CreateReplacement(ScalarNode, "!!str", result)
+		results.PushBack(newNode)
+	}
+
+	return context.ChildContext(results), nil
+}

pkg/yqlib/operator_system_test.go

@@ -0,0 +1,123 @@
+package yqlib
+
+import (
+	"os/exec"
+	"testing"
+)
+
+func findExec(t *testing.T, name string) string {
+	t.Helper()
+	path, err := exec.LookPath(name)
+	if err != nil {
+		t.Skipf("skipping: %v not found: %v", name, err)
+	}
+	return path
+}
+
+var systemOperatorDisabledScenarios = []expressionScenario{
+	{
+		description:    "system operator returns error when disabled",
+		subdescription: "Use `--security-enable-system-operator` to enable the system operator.",
+		document:       "country: Australia",
+		expression:     `.country = system("/usr/bin/echo"; "test")`,
+		expectedError:  "system operations are disabled, use --security-enable-system-operator to enable",
+	},
+}
+
+func TestSystemOperatorDisabledScenarios(t *testing.T) {
+	originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps
+	defer func() {
+		ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps
+	}()
+
+	ConfiguredSecurityPreferences.EnableSystemOps = false
+
+	for _, tt := range systemOperatorDisabledScenarios {
+		testScenario(t, &tt)
+	}
+	documentOperatorScenarios(t, "system-operators", systemOperatorDisabledScenarios)
+}
+
+func TestSystemOperatorEnabledScenarios(t *testing.T) {
+	echoPath := findExec(t, "echo")
+	falsePath := findExec(t, "false")
+
+	originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps
+	defer func() {
+		ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps
+	}()
+
+	ConfiguredSecurityPreferences.EnableSystemOps = true
+
+	scenarios := []expressionScenario{
+		{
+			description:    "Run a command with an argument",
+			subdescription: "Use `--security-enable-system-operator` to enable the system operator.",
+			yqFlags:        "--security-enable-system-operator",
+			document:       "country: Australia",
+			expression:     `.country = system("` + echoPath + `"; "test")`,
+			expected: []string{
+				"D0, P[], (!!map)::country: test\n",
+			},
+		},
+		{
+			description:    "Run a command without arguments",
+			subdescription: "Omit the semicolon and args to run the command with no extra arguments.",
+			yqFlags:        "--security-enable-system-operator",
+			document:       "a: hello",
+			expression:     `.a = system("` + echoPath + `")`,
+			expected: []string{
+				"D0, P[], (!!map)::a: \"\"\n",
+			},
+		},
+		{
+			description:    "Run a command with multiple arguments",
+			subdescription: "Pass an array of arguments.",
+			skipDoc:        true,
+			document:       "a: hello",
+			expression:     `.a = system("` + echoPath + `"; ["foo", "bar"])`,
+			expected: []string{
+				"D0, P[], (!!map)::a: foo bar\n",
+			},
+		},
+		{
+			description: "Command and args are evaluated per matched node",
+			skipDoc:     true,
+			document:    "cmd: " + echoPath + "\narg: hello",
+			expression:  `.result = system(.cmd; .arg)`,
+			expected: []string{
+				"D0, P[], (!!map)::cmd: " + echoPath + "\narg: hello\nresult: hello\n",
+			},
+		},
+		{
+			description:   "Command failure returns error",
+			skipDoc:       true,
+			document:      "a: hello",
+			expression:    `.a = system("` + falsePath + `")`,
+			expectedError: "system command '" + falsePath + "' failed: exit status 1",
+		},
+		{
+			description:   "Null command returns error",
+			skipDoc:       true,
+			document:      "a: hello",
+			expression:    `.a = system(null)`,
+			expectedError: "system operator: command must be a string scalar",
+		},
+		{
+			description: "System operator processes multiple matched nodes",
+			skipDoc:     true,
+			document:    "a: first",
+			document2:   "a: second",
+			expression:  `.a = system("` + echoPath + `"; "replaced")`,
+			expected: []string{
+				"D0, P[], (!!map)::a: replaced\n",
+				"D0, P[], (!!map)::a: replaced\n",
+			},
+		},
+	}
+
+	for _, tt := range scenarios {
+		testScenario(t, &tt)
+	}
+	appendOperatorDocumentScenario(t, "system-operators", scenarios)
+}

pkg/yqlib/operators_test.go

@@ -31,6 +31,7 @@ type expressionScenario struct {
 	dontFormatInputForDoc bool // dont format input doc for documentation generation
 	requiresFormat        string
 	skipForGoccy          bool
+	yqFlags               string // extra yq flags to include in generated doc command snippets
 }
 
 var goccyTesting = false
@@ -356,14 +357,22 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
 
 		writeOrPanic(w, "then\n")
 
+		flagsPrefix := ""
+		if s.yqFlags != "" {
+			flagsPrefix = s.yqFlags + " "
+		}
 		if s.expression != "" {
-			writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v'%v' %v\n```\n", envCommand, command, strings.ReplaceAll(s.expression, "'", `'\''`), files))
+			writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v'%v' %v\n```\n", envCommand, flagsPrefix, command, strings.ReplaceAll(s.expression, "'", `'\''`), files))
 		} else {
-			writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v\n```\n", envCommand, command, files))
+			writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v%v\n```\n", envCommand, flagsPrefix, command, files))
 		}
 	} else {
 		writeOrPanic(w, "Running\n")
-		writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v--null-input '%v'\n```\n", envCommand, command, s.expression))
+		flagsPrefix := ""
+		if s.yqFlags != "" {
+			flagsPrefix = s.yqFlags + " "
+		}
+		writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v--null-input '%v'\n```\n", envCommand, flagsPrefix, command, s.expression))
 	}
 	return formattedDoc, formattedDoc2
 }

pkg/yqlib/security_prefs.go

@@ -1,11 +1,13 @@
 package yqlib
 
 type SecurityPreferences struct {
-	DisableEnvOps  bool
+	DisableEnvOps   bool
-	DisableFileOps bool
+	DisableFileOps  bool
+	EnableSystemOps bool
 }
 
 var ConfiguredSecurityPreferences = SecurityPreferences{
-	DisableEnvOps:  false,
+	DisableEnvOps:   false,
-	DisableFileOps: false,
+	DisableFileOps:  false,
+	EnableSystemOps: false,
 }