Merge ee21f7591f into c47fe40a30

* Initial plan

* Fix TOML encoder to quote keys with special characters

Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/b2b52954-d13f-4e67-831a-16fdd3378de5

Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com>

* Add test for dotted table section header with special character key

Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/12c783dd-8b7f-43bf-b71a-e7a0b5e55fea

Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com>

* Apply De Morgan's law to tomlKey condition to fix staticcheck QF1001

Agent-Logs-Url: https://github.com/mikefarah/yq/sessions/eeab0316-309f-418f-b357-11bbacffb471

Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com>

---------

Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com>
Co-authored-by: mikefarah <1151925+mikefarah@users.noreply.github.com>

.github/workflows/release.yml

@@ -44,7 +44,7 @@ jobs:
           ./scripts/xcompile.sh
 
       - name: Release
-        uses: softprops/action-gh-release@v1
+        uses: softprops/action-gh-release@v2
         with:
           files: build/*
           draft: true

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.26.1 AS builder
+FROM golang:1.26.2 AS builder
 
 WORKDIR /go/src/mikefarah/yq
 

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM golang:1.26.1
+FROM golang:1.26.2
 
 RUN apt-get update && \
     apt-get install -y npm && \

cmd/root.go

@@ -212,6 +212,7 @@ yq -P -oy sample.json
 
 	rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableEnvOps, "security-disable-env-ops", "", false, "Disable env related operations.")
 	rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableFileOps, "security-disable-file-ops", "", false, "Disable file related operations (e.g. load)")
+	rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.EnableSystemOps, "security-enable-system-operator", "", false, "Enable system operator to allow execution of external commands.")
 
 	rootCmd.AddCommand(
 		createEvaluateSequenceCommand(),

go.mod

@@ -17,9 +17,9 @@ require (
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
 	github.com/spf13/cobra v1.10.2
 	github.com/spf13/pflag v1.0.10
-	github.com/yuin/gopher-lua v1.1.1
+	github.com/yuin/gopher-lua v1.1.2
 	github.com/zclconf/go-cty v1.18.0
-	go.yaml.in/yaml/v4 v4.0.0-rc.3
+	go.yaml.in/yaml/v4 v4.0.0-rc.4
 	golang.org/x/mod v0.34.0
 	golang.org/x/net v0.52.0
 	golang.org/x/text v0.35.0

go.sum

@@ -61,15 +61,15 @@ github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3A
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
+github.com/yuin/gopher-lua v1.1.2 h1:yF/FjE3hD65tBbt0VXLE13HWS9h34fdzJmrWRXwobGA=
-github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
+github.com/yuin/gopher-lua v1.1.2/go.mod h1:7aRmXIWl37SqRf0koeyylBEzJ+aPt8A+mmkQ4f1ntR8=
 github.com/zclconf/go-cty v1.18.0 h1:pJ8+HNI4gFoyRNqVE37wWbJWVw43BZczFo7KUoRczaA=
 github.com/zclconf/go-cty v1.18.0/go.mod h1:qpnV6EDNgC1sns/AleL1fvatHw72j+S+nS+MJ+T2CSg=
 github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo=
 github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM=
 go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
-go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go=
+go.yaml.in/yaml/v4 v4.0.0-rc.4 h1:UP4+v6fFrBIb1l934bDl//mmnoIZEDK0idg1+AIvX5U=
-go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
+go.yaml.in/yaml/v4 v4.0.0-rc.4/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
 golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
 golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
 golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=

pkg/yqlib/candidate_node.go

@@ -100,6 +100,9 @@ type CandidateNode struct {
 	// For formats like HCL and TOML: indicates that child entries should be emitted as separate blocks/tables
 	// rather than consolidated into nested mappings (default behaviour)
 	EncodeSeparate bool
+	// For formats like HCL: indicates that a blank line preceded this node in the original source,
+	// so the encoder should emit a blank line before it to preserve formatting.
+	BlankLineBefore bool
 }
 
 func (n *CandidateNode) CreateChild() *CandidateNode {
@@ -411,7 +414,8 @@ func (n *CandidateNode) doCopy(cloneContent bool) *CandidateNode {
 		EvaluateTogether: n.EvaluateTogether,
 		IsMapKey:         n.IsMapKey,
 
-		EncodeSeparate: n.EncodeSeparate,
+		EncodeSeparate:  n.EncodeSeparate,
+		BlankLineBefore: n.BlankLineBefore,
 	}
 
 	if cloneContent {

pkg/yqlib/decoder_hcl.go

@@ -43,6 +43,36 @@ type attributeWithName struct {
 	Attr *hclsyntax.Attribute
 }
 
+// bodyItem represents either an attribute or a block at a given byte position in the source,
+// allowing attributes and blocks to be processed together in source order.
+type bodyItem struct {
+	startByte int
+	attr      *attributeWithName // non-nil for attributes
+	block     *hclsyntax.Block   // non-nil for blocks
+}
+
+// sortedBodyItems returns attributes and blocks interleaved in source declaration order.
+func sortedBodyItems(attrs hclsyntax.Attributes, blocks hclsyntax.Blocks) []bodyItem {
+	var items []bodyItem
+	for name, attr := range attrs {
+		items = append(items, bodyItem{
+			startByte: attr.Range().Start.Byte,
+			attr:      &attributeWithName{Name: name, Attr: attr},
+		})
+	}
+	for _, block := range blocks {
+		b := block
+		items = append(items, bodyItem{
+			startByte: b.TypeRange.Start.Byte,
+			block:     b,
+		})
+	}
+	sort.Slice(items, func(i, j int) bool {
+		return items[i].startByte < items[j].startByte
+	})
+	return items
+}
+
 // extractLineComment extracts any inline comment after the given position
 func extractLineComment(src []byte, endPos int) string {
 	// Look for # comment after the token
@@ -64,6 +94,59 @@ func extractLineComment(src []byte, endPos int) string {
 	return ""
 }
 
+// hasPrecedingBlankLine reports whether there is a blank line immediately before startPos,
+// skipping over any immediately preceding comment lines and whitespace.
+func hasPrecedingBlankLine(src []byte, startPos int) bool {
+	i := startPos - 1
+
+	// Skip trailing spaces/tabs on the current token's preceding content
+	for i >= 0 && (src[i] == ' ' || src[i] == '\t') {
+		i--
+	}
+
+	// We expect to be sitting just before a newline that ends the previous line.
+	// Walk backwards skipping comment lines until we find a blank line or a non-comment line.
+	for i >= 0 {
+		// We should be pointing at '\n' (end of previous line) or start of file.
+		if src[i] != '\n' {
+			return false
+		}
+		i-- // step past the '\n'
+
+		// Skip '\r' for Windows line endings
+		if i >= 0 && src[i] == '\r' {
+			i--
+		}
+
+		// If immediately another '\n', this is a blank line.
+		if i < 0 || src[i] == '\n' {
+			return true
+		}
+
+		// Read the previous line to see if it's a comment or blank.
+		lineEnd := i
+		for i >= 0 && src[i] != '\n' {
+			i--
+		}
+		lineStart := i + 1
+		line := strings.TrimSpace(string(src[lineStart : lineEnd+1]))
+
+		if line == "" {
+			return true
+		}
+
+		if strings.HasPrefix(line, "#") {
+			// This line is a comment belonging to the current element; keep scanning upward.
+			continue
+		}
+
+		// A non-blank, non-comment line: no blank line precedes this element.
+		return false
+	}
+
+	return false
+}
+
 // extractHeadComment extracts comments before a given start position
 func extractHeadComment(src []byte, startPos int) string {
 	var comments []string
@@ -136,39 +219,47 @@ func (dec *hclDecoder) Decode() (*CandidateNode, error) {
 
 	root := &CandidateNode{Kind: MappingNode}
 
-	// process attributes in declaration order
 	body := dec.file.Body.(*hclsyntax.Body)
-	firstAttr := true
-	for _, attrWithName := range sortedAttributes(body.Attributes) {
-		keyNode := createStringScalarNode(attrWithName.Name)
-		valNode := convertHclExprToNode(attrWithName.Attr.Expr, dec.fileBytes)
 
-		// Attach comments if any
+	// Count blocks by type at THIS level to detect multiple separate blocks of the same type.
-		attrRange := attrWithName.Attr.Range()
-		headComment := extractHeadComment(dec.fileBytes, attrRange.Start.Byte)
-		if firstAttr && headComment != "" {
-			// For the first attribute, apply its head comment to the root
-			root.HeadComment = headComment
-			firstAttr = false
-		} else if headComment != "" {
-			keyNode.HeadComment = headComment
-		}
-		if lineComment := extractLineComment(dec.fileBytes, attrRange.End.Byte); lineComment != "" {
-			valNode.LineComment = lineComment
-		}
-
-		root.AddKeyValueChild(keyNode, valNode)
-	}
-
-	// process blocks
-	// Count blocks by type at THIS level to detect multiple separate blocks
 	blocksByType := make(map[string]int)
 	for _, block := range body.Blocks {
 		blocksByType[block.Type]++
 	}
 
-	for _, block := range body.Blocks {
+	// Process attributes and blocks together in source declaration order.
-		addBlockToMapping(root, block, dec.fileBytes, blocksByType[block.Type] > 1)
+	isFirst := true
+	for _, item := range sortedBodyItems(body.Attributes, body.Blocks) {
+		if item.attr != nil {
+			aw := item.attr
+			keyNode := createStringScalarNode(aw.Name)
+			valNode := convertHclExprToNode(aw.Attr.Expr, dec.fileBytes)
+
+			attrRange := aw.Attr.Range()
+			headComment := extractHeadComment(dec.fileBytes, attrRange.Start.Byte)
+			if isFirst && headComment != "" {
+				// For the first element, apply its head comment to the root node
+				root.HeadComment = headComment
+			} else if headComment != "" {
+				keyNode.HeadComment = headComment
+			}
+			if lineComment := extractLineComment(dec.fileBytes, attrRange.End.Byte); lineComment != "" {
+				valNode.LineComment = lineComment
+			}
+			if !isFirst && hasPrecedingBlankLine(dec.fileBytes, attrRange.Start.Byte) {
+				keyNode.BlankLineBefore = true
+			}
+
+			root.AddKeyValueChild(keyNode, valNode)
+		} else {
+			block := item.block
+			headComment := extractHeadComment(dec.fileBytes, block.TypeRange.Start.Byte)
+			if isFirst && headComment != "" {
+				root.HeadComment = headComment
+			}
+			addBlockToMappingOrdered(root, block, dec.fileBytes, blocksByType[block.Type] > 1, isFirst, headComment)
+		}
+		isFirst = false
 	}
 
 	dec.documentIndex++
@@ -178,71 +269,105 @@ func (dec *hclDecoder) Decode() (*CandidateNode, error) {
 
 func hclBodyToNode(body *hclsyntax.Body, src []byte) *CandidateNode {
 	node := &CandidateNode{Kind: MappingNode}
-	for _, attrWithName := range sortedAttributes(body.Attributes) {
-		key := createStringScalarNode(attrWithName.Name)
-		val := convertHclExprToNode(attrWithName.Attr.Expr, src)
 
-		// Attach comments if any
-		attrRange := attrWithName.Attr.Range()
-		if headComment := extractHeadComment(src, attrRange.Start.Byte); headComment != "" {
-			key.HeadComment = headComment
-		}
-		if lineComment := extractLineComment(src, attrRange.End.Byte); lineComment != "" {
-			val.LineComment = lineComment
-		}
-
-		node.AddKeyValueChild(key, val)
-	}
-
-	// Process nested blocks, counting blocks by type at THIS level
-	// to detect which block types appear multiple times
 	blocksByType := make(map[string]int)
 	for _, block := range body.Blocks {
 		blocksByType[block.Type]++
 	}
 
-	for _, block := range body.Blocks {
+	isFirst := true
-		addBlockToMapping(node, block, src, blocksByType[block.Type] > 1)
+	for _, item := range sortedBodyItems(body.Attributes, body.Blocks) {
+		if item.attr != nil {
+			aw := item.attr
+			key := createStringScalarNode(aw.Name)
+			val := convertHclExprToNode(aw.Attr.Expr, src)
+
+			attrRange := aw.Attr.Range()
+			if headComment := extractHeadComment(src, attrRange.Start.Byte); headComment != "" {
+				key.HeadComment = headComment
+			}
+			if lineComment := extractLineComment(src, attrRange.End.Byte); lineComment != "" {
+				val.LineComment = lineComment
+			}
+			if !isFirst && hasPrecedingBlankLine(src, attrRange.Start.Byte) {
+				key.BlankLineBefore = true
+			}
+
+			node.AddKeyValueChild(key, val)
+		} else {
+			block := item.block
+			headComment := extractHeadComment(src, block.TypeRange.Start.Byte)
+			addBlockToMappingOrdered(node, block, src, blocksByType[block.Type] > 1, isFirst, headComment)
+		}
+		isFirst = false
 	}
 	return node
 }
 
-// addBlockToMapping nests block type and labels into the parent mapping, merging children.
+// addBlockToMappingOrdered nests a block's type and labels into the parent mapping, merging children.
-// isMultipleBlocksOfType indicates if there are multiple blocks of this type at THIS level
+// isMultipleBlocksOfType: there are multiple blocks of this type at this level.
-func addBlockToMapping(parent *CandidateNode, block *hclsyntax.Block, src []byte, isMultipleBlocksOfType bool) {
+// isFirstInParent: this block is the first element in the parent (no preceding sibling).
+// headComment: any comment extracted before this block's type keyword.
+func addBlockToMappingOrdered(parent *CandidateNode, block *hclsyntax.Block, src []byte, isMultipleBlocksOfType bool, isFirstInParent bool, headComment string) {
 	bodyNode := hclBodyToNode(block.Body, src)
 	current := parent
 
 	// ensure block type mapping exists
 	var typeNode *CandidateNode
+	var typeKeyNode *CandidateNode
 	for i := 0; i < len(current.Content); i += 2 {
 		if current.Content[i].Value == block.Type {
+			typeKeyNode = current.Content[i]
 			typeNode = current.Content[i+1]
 			break
 		}
 	}
 	if typeNode == nil {
-		_, typeNode = current.AddKeyValueChild(createStringScalarNode(block.Type), &CandidateNode{Kind: MappingNode})
+		var newTypeKey *CandidateNode
-		// Mark the type node if there are multiple blocks of this type at this level
+		newTypeKey, typeNode = current.AddKeyValueChild(createStringScalarNode(block.Type), &CandidateNode{Kind: MappingNode})
-		// This tells the encoder to emit them as separate blocks rather than consolidating them
+		typeKeyNode = newTypeKey
+		// Mark the type node if there are multiple blocks of this type at this level.
+		// This tells the encoder to emit them as separate blocks rather than consolidating them.
 		if isMultipleBlocksOfType {
 			typeNode.EncodeSeparate = true
 		}
+		// Store the head comment on the type key (non-first elements only; first element's
+		// comment is handled by the caller and applied to the root node).
+		if !isFirstInParent && headComment != "" {
+			typeKeyNode.HeadComment = headComment
+		}
+		// Detect blank line before this block in the source.
+		// Only set it when this is not the first element (i.e. something already precedes it).
+		if !isFirstInParent && hasPrecedingBlankLine(src, block.TypeRange.Start.Byte) {
+			typeKeyNode.BlankLineBefore = true
+		}
 	}
 	current = typeNode
 
 	// walk labels, creating/merging mappings
-	for _, label := range block.Labels {
+	for labelIdx, label := range block.Labels {
 		var next *CandidateNode
+		var labelKey *CandidateNode
 		for i := 0; i < len(current.Content); i += 2 {
 			if current.Content[i].Value == label {
+				labelKey = current.Content[i]
 				next = current.Content[i+1]
 				break
 			}
 		}
 		if next == nil {
-			_, next = current.AddKeyValueChild(createStringScalarNode(label), &CandidateNode{Kind: MappingNode})
+			var newLabelKey *CandidateNode
+			newLabelKey, next = current.AddKeyValueChild(createStringScalarNode(label), &CandidateNode{Kind: MappingNode})
+			labelKey = newLabelKey
+			// For same-type blocks: mark the first label key with BlankLineBefore when
+			// there is a blank line before this block in the source.
+			if labelIdx == 0 && len(current.Content) > 2 {
+				if hasPrecedingBlankLine(src, block.TypeRange.Start.Byte) {
+					labelKey.BlankLineBefore = true
+				}
+			}
 		}
+		_ = labelKey
 		current = next
 	}
 

pkg/yqlib/doc/operators/headers/slice-array.md

@@ -1,5 +1,5 @@
-# Slice/Splice Array
+# Slice Array or String
 
-The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.
+The slice operator works on both arrays and strings. Like the `jq` equivalent, `.[10:15]` will return a subarray (or substring) of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array or string.
 
-You may leave out the first or second number, which will refer to the start or end of the array respectively.
+You may leave out the first or second number, which will refer to the start or end of the array or string respectively.

pkg/yqlib/doc/operators/headers/system-operators.md

@@ -0,0 +1,27 @@
+# System Operators
+
+The `system` operator allows you to run an external command and use its output as a value in your expression.
+
+**Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it.
+
+**Note:** When enabled, the system operator can replicate the functionality of `env` and `load`
+operators via external commands. Enabling it effectively overrides `--security-disable-env-ops`
+and `--security-disable-file-ops`.
+
+## Usage
+
+```bash
+yq --security-enable-system-operator --null-input '.field = system("command"; "arg1")'
+```
+
+The operator takes:
+- A command string (required)
+- An argument (or an array of arguments), separated from the command by `;` (optional)
+
+The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string.
+
+## Disabling the system operator
+
+The system operator is disabled by default. When disabled, an error is returned instead of running the command, consistent with `--security-disable-env-ops` and `--security-disable-file-ops`.
+
+Use `--security-enable-system-operator` flag to enable it.

pkg/yqlib/doc/operators/slice-array.md

@@ -1,8 +1,8 @@
-# Slice/Splice Array
+# Slice Array or String
 
-The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.
+The slice operator works on both arrays and strings. Like the `jq` equivalent, `.[10:15]` will return a subarray (or substring) of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array or string.
 
-You may leave out the first or second number, which will refer to the start or end of the array respectively.
+You may leave out the first or second number, which will refer to the start or end of the array or string respectively.
 
 ## Slicing arrays
 Given a sample.yml file of:
@@ -103,3 +103,81 @@ will output
 - cow
 ```
 
+## Slicing strings
+Given a sample.yml file of:
+```yaml
+country: Australia
+```
+then
+```bash
+yq '.country[0:5]' sample.yml
+```
+will output
+```yaml
+Austr
+```
+
+## Slicing strings - without the second number
+Finishes at the end of the string
+
+Given a sample.yml file of:
+```yaml
+country: Australia
+```
+then
+```bash
+yq '.country[5:]' sample.yml
+```
+will output
+```yaml
+alia
+```
+
+## Slicing strings - without the first number
+Starts from the start of the string
+
+Given a sample.yml file of:
+```yaml
+country: Australia
+```
+then
+```bash
+yq '.country[:5]' sample.yml
+```
+will output
+```yaml
+Austr
+```
+
+## Slicing strings - use negative numbers to count backwards from the end
+Negative indices count from the end of the string
+
+Given a sample.yml file of:
+```yaml
+country: Australia
+```
+then
+```bash
+yq '.country[-5:]' sample.yml
+```
+will output
+```yaml
+ralia
+```
+
+## Slicing strings - Unicode
+Indices are rune-based, so multi-byte characters are handled correctly
+
+Given a sample.yml file of:
+```yaml
+greeting: héllo
+```
+then
+```bash
+yq '.greeting[1:3]' sample.yml
+```
+will output
+```yaml
+él
+```
+

pkg/yqlib/doc/operators/system-operators.md

@@ -0,0 +1,76 @@
+# System Operators
+
+The `system` operator allows you to run an external command and use its output as a value in your expression.
+
+**Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it.
+
+**Note:** When enabled, the system operator can replicate the functionality of `env` and `load`
+operators via external commands. Enabling it effectively overrides `--security-disable-env-ops`
+and `--security-disable-file-ops`.
+
+## Usage
+
+```bash
+yq --security-enable-system-operator --null-input '.field = system("command"; "arg1")'
+```
+
+The operator takes:
+- A command string (required)
+- An argument (or an array of arguments), separated from the command by `;` (optional)
+
+The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string.
+
+## Disabling the system operator
+
+The system operator is disabled by default. When disabled, an error is returned instead of running the command, consistent with `--security-disable-env-ops` and `--security-disable-file-ops`.
+
+Use `--security-enable-system-operator` flag to enable it.
+
+## system operator returns error when disabled
+Use `--security-enable-system-operator` to enable the system operator.
+
+Given a sample.yml file of:
+```yaml
+country: Australia
+```
+then
+```bash
+yq '.country = system("/usr/bin/echo"; "test")' sample.yml
+```
+will output
+```bash
+Error: system operations are disabled, use --security-enable-system-operator to enable
+```
+
+## Run a command with an argument
+Use `--security-enable-system-operator` to enable the system operator.
+
+Given a sample.yml file of:
+```yaml
+country: Australia
+```
+then
+```bash
+yq --security-enable-system-operator '.country = system("/usr/bin/echo"; "test")' sample.yml
+```
+will output
+```yaml
+country: test
+```
+
+## Run a command without arguments
+Omit the semicolon and args to run the command with no extra arguments.
+
+Given a sample.yml file of:
+```yaml
+a: hello
+```
+then
+```bash
+yq --security-enable-system-operator '.a = system("/usr/bin/echo")' sample.yml
+```
+will output
+```yaml
+a: ""
+```
+

pkg/yqlib/doc/usage/hcl.md

@@ -169,8 +169,10 @@ will output
 ```hcl
 # Arithmetic with literals and application-provided variables
 sum = 1 + addend
+
 # String interpolation and templates
 message = "Hello, ${name}!"
+
 # Application-provided functions
 shouty_message = upper(message)
 ```
@@ -199,3 +201,61 @@ resource "aws_instance" "db" {
 }
 ```
 
+## Roundtrip: blank lines between attributes are preserved
+Given a sample.hcl file of:
+```hcl
+name = "app"
+
+version = 1
+
+enabled = true
+
+```
+then
+```bash
+yq sample.hcl
+```
+will output
+```hcl
+name = "app"
+
+version = 1
+
+enabled = true
+```
+
+## Roundtrip: blank lines between blocks are preserved
+Given a sample.hcl file of:
+```hcl
+terraform {
+  source = "git::https://example.com/module.git"
+}
+
+include {
+  path = "../root.hcl"
+}
+
+dependency "base" {
+  config_path = "../base"
+}
+
+```
+then
+```bash
+yq sample.hcl
+```
+will output
+```hcl
+terraform {
+  source = "git::https://example.com/module.git"
+}
+
+include {
+  path = "../root.hcl"
+}
+
+dependency "base" {
+  config_path = "../base"
+}
+```
+

pkg/yqlib/encoder_hcl.go

@@ -50,9 +50,11 @@ func (he *hclEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 	f := hclwrite.NewEmptyFile()
 	body := f.Body()
 
-	// Collect comments as we encode
+	// Collect comments and blank-line markers as we encode
 	commentMap := make(map[string]string)
+	blankLineSet := make(map[string]bool)
 	he.collectComments(node, "", commentMap)
+	he.collectBlankLines(node, blankLineSet)
 
 	if err := he.encodeNode(body, node); err != nil {
 		return fmt.Errorf("failed to encode HCL: %w", err)
@@ -62,8 +64,12 @@ func (he *hclEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 	output := f.Bytes()
 	compactOutput := he.compactSpacing(output)
 
-	// Inject comments back into the output
+	// Inject comments first so that blank lines are inserted before comment+attribute groups.
-	finalOutput := he.injectComments(compactOutput, commentMap)
+	withComments := he.injectComments(compactOutput, commentMap)
+
+	// Inject blank lines before appropriate elements (after comments, so blanks appear before
+	// the comment that precedes an attribute, not between the comment and the attribute).
+	finalOutput := he.injectBlankLines(withComments, blankLineSet)
 
 	if he.prefs.ColorsEnabled {
 		colourized := he.colorizeHcl(finalOutput)
@@ -123,6 +129,79 @@ func (he *hclEncoder) collectComments(node *CandidateNode, prefix string, commen
 	}
 }
 
+// collectBlankLines recursively collects keys that have BlankLineBefore set, so
+// the encoder can re-insert blank lines into the output.
+func (he *hclEncoder) collectBlankLines(node *CandidateNode, blankLineSet map[string]bool) {
+	if node == nil || node.Kind != MappingNode {
+		return
+	}
+
+	for i := 0; i < len(node.Content); i += 2 {
+		keyNode := node.Content[i]
+		valueNode := node.Content[i+1]
+		key := keyNode.Value
+
+		if keyNode.BlankLineBefore {
+			blankLineSet[key] = true
+		}
+
+		if valueNode.Kind == MappingNode {
+			he.collectBlankLines(valueNode, blankLineSet)
+		}
+	}
+}
+
+// injectBlankLines inserts a blank line before each HCL element (or its preceding comment block)
+// whose key name is in blankLineSet. It scans lines in reverse: when it finds a key that needs a
+// blank line, it walks backward over any immediately preceding comment lines and inserts the blank
+// line before that comment block (so the blank line precedes the comment+key group, not between them).
+func (he *hclEncoder) injectBlankLines(output []byte, blankLineSet map[string]bool) []byte {
+	if len(blankLineSet) == 0 {
+		return output
+	}
+
+	// identRe captures the first identifier on a line (possibly indented).
+	identRe := regexp.MustCompile(`^\s*([A-Za-z_][A-Za-z0-9_-]*)(\s*(=|\{|"))`)
+	commentRe := regexp.MustCompile(`^\s*#`)
+
+	lines := strings.Split(string(output), "\n")
+
+	// Mark which line indices need a blank line inserted before them.
+	insertBefore := make(map[int]bool)
+
+	for i, line := range lines {
+		m := identRe.FindStringSubmatch(line)
+		if m == nil || !blankLineSet[m[1]] {
+			continue
+		}
+
+		// Walk backward over comment lines to find the insertion point.
+		insertAt := i
+		for insertAt > 0 && commentRe.MatchString(lines[insertAt-1]) {
+			insertAt--
+		}
+
+		// Only insert if the line before the insertion point is not already blank.
+		if insertAt > 0 && strings.TrimSpace(lines[insertAt-1]) != "" {
+			insertBefore[insertAt] = true
+		}
+	}
+
+	if len(insertBefore) == 0 {
+		return output
+	}
+
+	out := make([]string, 0, len(lines)+len(insertBefore))
+	for i, line := range lines {
+		if insertBefore[i] {
+			out = append(out, "")
+		}
+		out = append(out, line)
+	}
+
+	return []byte(strings.Join(out, "\n"))
+}
+
 // joinCommentPath concatenates path segments using commentPathSep, safely handling empty prefixes.
 func joinCommentPath(prefix, segment string) string {
 	if prefix == "" {
@@ -164,9 +243,16 @@ func (he *hclEncoder) injectComments(output []byte, commentMap map[string]string
 			continue
 		}
 
-		re := regexp.MustCompile(`(?m)^(\s*)` + regexp.QuoteMeta(key) + `\s*=`)
+		// Match both attribute assignments (key =) and block openers (key { or key "label").
-		if re.MatchString(result) {
+		// Use ( *) instead of (\s*) to only capture indentation spaces, not newlines.
-			result = re.ReplaceAllString(result, "$1"+trimmed+"\n$0")
+		// Replace only the first occurrence to avoid duplicating comments before same-named keys.
+		re := regexp.MustCompile(`(?m)^( *)` + regexp.QuoteMeta(key) + `( *(=|\{|"))`)
+		submatches := re.FindStringSubmatchIndex(result)
+		if submatches != nil {
+			// submatches[2] and submatches[3] are the bounds of the ( *) indentation group.
+			indent := result[submatches[2]:submatches[3]]
+			insertPos := submatches[0]
+			result = result[:insertPos] + indent + trimmed + "\n" + result[insertPos:]
 		}
 	}
 
@@ -345,6 +431,27 @@ func tokensForRawHCLExpr(expr string) (hclwrite.Tokens, error) {
 			tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenStar, Bytes: []byte{'*'}})
 		case ch == '/':
 			tokens = append(tokens, &hclwrite.Token{Type: hclsyntax.TokenSlash, Bytes: []byte{'/'}})
+		case ch == '"':
+			// Quoted string literal: consume until closing quote, respecting backslash escapes.
+			start := i
+			i++ // skip opening quote
+			for i < len(expr) && expr[i] != '"' {
+				if expr[i] == '\\' {
+					i++ // skip escape character
+				}
+				i++
+			}
+			if i < len(expr) {
+				i++ // skip closing quote
+			}
+			// Emit as a sequence of OQuote + QuotedLit + CQuote tokens so hclwrite renders it correctly.
+			raw := expr[start:i]
+			tokens = append(tokens,
+				&hclwrite.Token{Type: hclsyntax.TokenOQuote, Bytes: []byte{'"'}},
+				&hclwrite.Token{Type: hclsyntax.TokenQuotedLit, Bytes: []byte(raw[1 : len(raw)-1])},
+				&hclwrite.Token{Type: hclsyntax.TokenCQuote, Bytes: []byte{'"'}},
+			)
+			continue
 		default:
 			return nil, fmt.Errorf("unsupported character %q in raw HCL expression", ch)
 		}
@@ -353,6 +460,125 @@ func tokensForRawHCLExpr(expr string) (hclwrite.Tokens, error) {
 	return tokens, nil
 }
 
+// tokensForValue builds an hclwrite token stream for any node value, preserving raw HCL
+// expressions (Style==0 !!str scalars) and recursing into mappings and sequences.
+func tokensForValue(node *CandidateNode) (hclwrite.Tokens, error) {
+	switch node.Kind {
+	case ScalarNode:
+		if node.Tag == "!!str" {
+			if node.Style == 0 || node.Style&LiteralStyle != 0 {
+				// Raw HCL expression — emit without quotes.
+				return tokensForRawHCLExpr(node.Value)
+			}
+			if node.Style&DoubleQuotedStyle != 0 {
+				// Template or quoted string.
+				inner := node.Value
+				var toks hclwrite.Tokens
+				toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenOQuote, Bytes: []byte{'"'}})
+				for i := 0; i < len(inner); {
+					if i < len(inner)-1 && inner[i] == '$' && inner[i+1] == '{' {
+						toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenTemplateInterp, Bytes: []byte("${")})
+						i += 2
+						start := i
+						depth := 1
+						for i < len(inner) && depth > 0 {
+							switch inner[i] {
+							case '{':
+								depth++
+							case '}':
+								depth--
+							}
+							i++
+						}
+						expr := inner[start : i-1]
+						toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenIdent, Bytes: []byte(expr)})
+						toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenTemplateSeqEnd, Bytes: []byte("}")})
+					} else {
+						toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenQuotedLit, Bytes: []byte{inner[i]}})
+						i++
+					}
+				}
+				toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenCQuote, Bytes: []byte{'"'}})
+				return toks, nil
+			}
+			// Any other string style: emit as quoted string.
+			var toks hclwrite.Tokens
+			toks = append(toks,
+				&hclwrite.Token{Type: hclsyntax.TokenOQuote, Bytes: []byte{'"'}},
+				&hclwrite.Token{Type: hclsyntax.TokenQuotedLit, Bytes: []byte(node.Value)},
+				&hclwrite.Token{Type: hclsyntax.TokenCQuote, Bytes: []byte{'"'}},
+			)
+			return toks, nil
+		}
+		// Non-string scalar: use cty conversion and let hclwrite render it.
+		ctyVal, err := nodeToCtyValue(node)
+		if err != nil {
+			return nil, err
+		}
+		return hclwrite.TokensForValue(ctyVal), nil
+	case MappingNode:
+		// Build {\n  key = value\n  ...\n} as properly typed tokens so
+		// hclwrite's formatter counts brackets correctly and indents properly.
+		// Empty mappings are rendered as {} on a single line.
+		if len(node.Content) == 0 {
+			return hclwrite.Tokens{
+				{Type: hclsyntax.TokenOBrace, Bytes: []byte("{")},
+				{Type: hclsyntax.TokenCBrace, Bytes: []byte("}")},
+			}, nil
+		}
+		var toks hclwrite.Tokens
+		toks = append(toks,
+			&hclwrite.Token{Type: hclsyntax.TokenOBrace, Bytes: []byte("{")},
+			&hclwrite.Token{Type: hclsyntax.TokenNewline, Bytes: []byte("\n")},
+		)
+		for i := 0; i < len(node.Content); i += 2 {
+			keyNode := node.Content[i]
+			valNode := node.Content[i+1]
+			// Key token
+			if isValidHCLIdentifier(keyNode.Value) {
+				toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenIdent, Bytes: []byte(keyNode.Value)})
+			} else {
+				toks = append(toks,
+					&hclwrite.Token{Type: hclsyntax.TokenOQuote, Bytes: []byte{'"'}},
+					&hclwrite.Token{Type: hclsyntax.TokenQuotedLit, Bytes: []byte(keyNode.Value)},
+					&hclwrite.Token{Type: hclsyntax.TokenCQuote, Bytes: []byte{'"'}},
+				)
+			}
+			// Equals token
+			toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenEqual, Bytes: []byte("=")})
+			// Value tokens
+			valToks, err := tokensForValue(valNode)
+			if err != nil {
+				return nil, err
+			}
+			toks = append(toks, valToks...)
+			// Newline after each entry
+			toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenNewline, Bytes: []byte("\n")})
+		}
+		toks = append(toks,
+			&hclwrite.Token{Type: hclsyntax.TokenCBrace, Bytes: []byte("}")},
+		)
+		return toks, nil
+	case SequenceNode:
+		var toks hclwrite.Tokens
+		toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenOBrack, Bytes: []byte("[")})
+		for i, child := range node.Content {
+			if i > 0 {
+				toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenComma, Bytes: []byte(",")})
+			}
+			childToks, err := tokensForValue(child)
+			if err != nil {
+				return nil, err
+			}
+			toks = append(toks, childToks...)
+		}
+		toks = append(toks, &hclwrite.Token{Type: hclsyntax.TokenCBrack, Bytes: []byte("]")})
+		return toks, nil
+	default:
+		return nil, fmt.Errorf("unsupported node kind for HCL value: %v", kindToString(node.Kind))
+	}
+}
+
 // encodeAttribute encodes a value as an HCL attribute
 func (he *hclEncoder) encodeAttribute(body *hclwrite.Body, key string, valueNode *CandidateNode) error {
 	if valueNode.Kind == ScalarNode && valueNode.Tag == "!!str" {
@@ -386,6 +612,15 @@ func (he *hclEncoder) encodeAttribute(body *hclwrite.Body, key string, valueNode
 			return nil
 		}
 	}
+	// Flow-style mapping or sequence: use tokensForValue to preserve raw expressions inside.
+	if valueNode.Kind == MappingNode || valueNode.Kind == SequenceNode {
+		tokens, err := tokensForValue(valueNode)
+		if err != nil {
+			return err
+		}
+		body.SetAttributeRaw(key, tokens)
+		return nil
+	}
 	// Default: use cty.Value for quoted strings and all other types
 	ctyValue, err := nodeToCtyValue(valueNode)
 	if err != nil {
@@ -465,14 +700,12 @@ func (he *hclEncoder) encodeBlockIfMapping(body *hclwrite.Body, key string, valu
 			if handled, err := he.encodeMappingChildrenAsBlocks(block.Body(), nestedType, bodyNode); err == nil && handled {
 				return true
 			}
-			if err := he.encodeNodeAttributes(block.Body(), bodyNode); err == nil {
+			_ = he.encodeNodeAttributes(block.Body(), bodyNode)
-				return true
-			}
-		}
-		block := body.AppendNewBlock(key, labels)
-		if err := he.encodeNodeAttributes(block.Body(), bodyNode); err == nil {
 			return true
 		}
+		block := body.AppendNewBlock(key, labels)
+		_ = he.encodeNodeAttributes(block.Body(), bodyNode)
+		return true
 	}
 
 	// If all child values are mappings, treat each child key as a labelled instance of this block type
@@ -480,13 +713,12 @@ func (he *hclEncoder) encodeBlockIfMapping(body *hclwrite.Body, key string, valu
 		return true
 	}
 
-	// No labels detected, render as unlabelled block
+	// No labels detected, render as unlabelled block.
+	// Note: AppendNewBlock writes to the underlying buffer immediately, so we must return true
+	// regardless of whether encodeNodeAttributes succeeds to avoid double-emitting the key.
 	block := body.AppendNewBlock(key, nil)
-	if err := he.encodeNodeAttributes(block.Body(), valueNode); err == nil {
+	_ = he.encodeNodeAttributes(block.Body(), valueNode)
-		return true
+	return true
-	}
-
-	return false
 }
 
 // encodeNode encodes a CandidateNode directly to HCL, preserving style information

pkg/yqlib/encoder_toml.go

@@ -69,6 +69,27 @@ func (te *tomlEncoder) CanHandleAliases() bool {
 
 // ---- helpers ----
 
+// tomlKey returns the key quoted if it contains characters that are not valid
+// in a TOML bare key. TOML bare keys may only contain ASCII letters, ASCII
+// digits, underscores, and dashes.
+func tomlKey(key string) string {
+	for _, r := range key {
+		if (r < 'A' || r > 'Z') && (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '_' && r != '-' {
+			return fmt.Sprintf("%q", key)
+		}
+	}
+	return key
+}
+
+// tomlDottedKey joins path components, quoting any that require it.
+func tomlDottedKey(path []string) string {
+	parts := make([]string, len(path))
+	for i, p := range path {
+		parts[i] = tomlKey(p)
+	}
+	return strings.Join(parts, ".")
+}
+
 func (te *tomlEncoder) writeComment(w io.Writer, comment string) error {
 	if comment == "" {
 		return nil
@@ -148,9 +169,10 @@ func (te *tomlEncoder) encodeTopLevelEntry(w io.Writer, path []string, node *Can
 		}
 		if allMaps {
 			key := path[len(path)-1]
+			quotedKey := tomlKey(key)
 			for _, it := range node.Content {
 				// [[key]] then body
-				if _, err := w.Write([]byte("[[" + key + "]]\n")); err != nil {
+				if _, err := w.Write([]byte("[[" + quotedKey + "]]\n")); err != nil {
 					return err
 				}
 				if err := te.encodeMappingBodyWithPath(w, []string{key}, it); err != nil {
@@ -185,7 +207,7 @@ func (te *tomlEncoder) writeAttribute(w io.Writer, key string, value *CandidateN
 	}
 
 	// Write the attribute
-	line := key + " = " + te.formatScalar(value)
+	line := tomlKey(key) + " = " + te.formatScalar(value)
 
 	// Add line comment if present
 	if value.LineComment != "" {
@@ -210,7 +232,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida
 
 	// Handle empty arrays
 	if len(seq.Content) == 0 {
-		line := key + " = []"
+		line := tomlKey(key) + " = []"
 		if seq.LineComment != "" {
 			lineComment := strings.TrimSpace(seq.LineComment)
 			if !strings.HasPrefix(lineComment, "#") {
@@ -233,7 +255,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida
 
 	if hasElementComments {
 		// Write multiline array format with comments
-		if _, err := w.Write([]byte(key + " = [\n")); err != nil {
+		if _, err := w.Write([]byte(tomlKey(key) + " = [\n")); err != nil {
 			return err
 		}
 
@@ -324,7 +346,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida
 		}
 	}
 
-	line := key + " = [" + strings.Join(items, ", ") + "]"
+	line := tomlKey(key) + " = [" + strings.Join(items, ", ") + "]"
 
 	// Add line comment if present
 	if seq.LineComment != "" {
@@ -372,21 +394,21 @@ func (te *tomlEncoder) mappingToInlineTable(m *CandidateNode) (string, error) {
 		v := m.Content[i+1]
 		switch v.Kind {
 		case ScalarNode:
-			parts = append(parts, fmt.Sprintf("%s = %s", k, te.formatScalar(v)))
+			parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), te.formatScalar(v)))
 		case SequenceNode:
 			// inline array in inline table
 			arr, err := te.sequenceToInlineArray(v)
 			if err != nil {
 				return "", err
 			}
-			parts = append(parts, fmt.Sprintf("%s = %s", k, arr))
+			parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), arr))
 		case MappingNode:
 			// nested inline table
 			inline, err := te.mappingToInlineTable(v)
 			if err != nil {
 				return "", err
 			}
-			parts = append(parts, fmt.Sprintf("%s = %s", k, inline))
+			parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), inline))
 		default:
 			return "", fmt.Errorf("unsupported inline table value kind: %v", v.Kind)
 		}
@@ -399,7 +421,7 @@ func (te *tomlEncoder) writeInlineTableAttribute(w io.Writer, key string, m *Can
 	if err != nil {
 		return err
 	}
-	_, err = w.Write([]byte(key + " = " + inline + "\n"))
+	_, err = w.Write([]byte(tomlKey(key) + " = " + inline + "\n"))
 	return err
 }
 
@@ -421,7 +443,7 @@ func (te *tomlEncoder) writeTableHeader(w io.Writer, path []string, m *Candidate
 	}
 
 	// Write table header [a.b.c]
-	header := "[" + strings.Join(path, ".") + "]\n"
+	header := "[" + tomlDottedKey(path) + "]\n"
 	_, err := w.Write([]byte(header))
 	return err
 }
@@ -488,7 +510,7 @@ func (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *Cand
 				}
 			}
 			if allMaps {
-				key := strings.Join(append(append([]string{}, path...), k), ".")
+				key := tomlDottedKey(append(append([]string{}, path...), k))
 				for _, it := range v.Content {
 					if _, err := w.Write([]byte("[[" + key + "]]\n")); err != nil {
 						return err
@@ -586,7 +608,7 @@ func (te *tomlEncoder) encodeMappingBodyWithPath(w io.Writer, path []string, m *
 				}
 			}
 			if allMaps {
-				dotted := strings.Join(append(append([]string{}, path...), k), ".")
+				dotted := tomlDottedKey(append(append([]string{}, path...), k))
 				for _, it := range v.Content {
 					if _, err := w.Write([]byte("[[" + dotted + "]]\n")); err != nil {
 						return err

pkg/yqlib/hcl_test.go

@@ -71,8 +71,10 @@ shouty_message = upper(message)`
 
 var simpleSampleExpected = `# Arithmetic with literals and application-provided variables
 sum = 1 + addend
+
 # String interpolation and templates
 message = "Hello, ${name}!"
+
 # Application-provided functions
 shouty_message = upper(message)
 `
@@ -85,6 +87,51 @@ message: "Hello, ${name}!"
 shouty_message: upper(message)
 `
 
+var blankLinesBetweenAttributes = "name = \"app\"\n\nversion = 1\n\nenabled = true\n"
+
+var blankLinesBetweenBlocks = `terraform {
+  source = "git::https://example.com/module.git"
+}
+
+include {
+  path = "../root.hcl"
+}
+
+dependency "base" {
+  config_path = "../base"
+}
+`
+
+var blankLinesMixedAttributesAndBlocks = `# Root comment
+name = "app"
+
+version = 1
+
+terraform {
+  source = "git::https://example.com/module.git"
+}
+
+dependency "base" {
+  config_path = "../base"
+}
+`
+
+var blocksWithCommentsAndBlankLines = `# First block comment
+terraform {
+  source = "example"
+}
+
+# Second block comment
+include {
+  path = "../root.hcl"
+}
+
+# Third block comment
+dependencies {
+  paths = ["../base"]
+}
+`
+
 var hclFormatScenarios = []formatScenario{
 	{
 		description:  "Parse HCL",
@@ -472,6 +519,46 @@ var hclFormatScenarios = []formatScenario{
 		expected:     "service {\n  optional_field = null\n}\n",
 		scenarioType: "roundtrip",
 	},
+	{
+		description:  "block with function call containing quoted string argument",
+		skipDoc:      true,
+		input:        "include {\n  path = find_in_parent_folders(\"root.hcl\")\n}\n",
+		expected:     "include {\n  path = find_in_parent_folders(\"root.hcl\")\n}\n",
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "Roundtrip: object attribute with traversal expression values",
+		skipDoc:      true,
+		input:        "inputs = {\n  sub_id = dependency.base.outputs.subscription_id\n  rg_name = dependency.base.outputs.resource_group_name\n}\n",
+		expected:     "inputs = {\n  sub_id = dependency.base.outputs.subscription_id\n  rg_name = dependency.base.outputs.resource_group_name\n}\n",
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "Roundtrip: blank lines between attributes are preserved",
+		input:        blankLinesBetweenAttributes,
+		expected:     blankLinesBetweenAttributes,
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "Roundtrip: blank lines between blocks are preserved",
+		input:        blankLinesBetweenBlocks,
+		expected:     blankLinesBetweenBlocks,
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "Roundtrip: blank lines between mixed attributes and blocks are preserved",
+		skipDoc:      true,
+		input:        blankLinesMixedAttributesAndBlocks,
+		expected:     blankLinesMixedAttributesAndBlocks,
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "Roundtrip: blocks with comments and blank lines are preserved",
+		skipDoc:      true,
+		input:        blocksWithCommentsAndBlankLines,
+		expected:     blocksWithCommentsAndBlankLines,
+		scenarioType: "roundtrip",
+	},
 }
 
 func testHclScenario(t *testing.T, s formatScenario) {

pkg/yqlib/lexer.go

@@ -131,6 +131,11 @@ func handleToken(tokens []*token, index int, postProcessedTokens []*token) (toke
 			log.Debugf("previous token is : traverseArrayOpType")
 			// need to put the number 0 before this token, as that is implied
 			postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: createValueOperation(0, "0")})
+		} else if index >= 2 && tokens[index-1].TokenType == openCollect &&
+			(tokens[index-2].TokenType == operationToken || tokens[index-2].TokenType == closeCollect || tokens[index-2].TokenType == closeCollectObject) {
+			log.Debugf("previous token is : openCollect following a traversal, implying 0 start")
+			// need to put the number 0 before this token, as that is implied
+			postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: createValueOperation(0, "0")})
 		}
 	}
 

pkg/yqlib/lexer_participle.go

@@ -96,6 +96,8 @@ var participleYqRules = []*participleYqRule{
 	simpleOp("load_?str|str_?load", loadStringOpType),
 	{"LoadYaml", `load`, loadOp(NewYamlDecoder(LoadYamlPreferences)), 0},
 
+	simpleOp("system", systemOpType),
+
 	{"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0},
 
 	simpleOp("select", selectOpType),

pkg/yqlib/lib.go

@@ -80,7 +80,7 @@ func recurseNodeObjectEqual(lhs *CandidateNode, rhs *CandidateNode) bool {
 		key := lhs.Content[index]
 		value := lhs.Content[index+1]
 
-		indexInRHS := findInArray(rhs, key)
+		indexInRHS := findKeyInMap(rhs, key)
 
 		if indexInRHS == -1 || !recursiveNodeEqual(value, rhs.Content[indexInRHS+1]) {
 			return false

pkg/yqlib/lib_test.go

@@ -24,7 +24,7 @@ type parseSnippetScenario struct {
 var parseSnippetScenarios = []parseSnippetScenario{
 	{
 		snippet:       ":",
-		expectedError: "yaml: did not find expected key",
+		expectedError: "yaml: while parsing a block mapping at <unknown position>: did not find expected key",
 	},
 	{
 		snippet: "",
@@ -300,6 +300,24 @@ func TestRecurseNodeObjectEqual(t *testing.T) {
 	test.AssertResult(t, true, recurseNodeObjectEqual(obj1, obj2))
 	test.AssertResult(t, false, recurseNodeObjectEqual(obj1, obj3))
 	test.AssertResult(t, false, recurseNodeObjectEqual(obj1, obj4))
+
+	// A null key must not match a null value in the other map.
+	// Regression test for https://issues.oss-fuzz.com/issues/383860504
+	nullKey := &CandidateNode{Kind: ScalarNode, Tag: "!!null"}
+	nullVal := &CandidateNode{Kind: ScalarNode, Tag: "!!null"}
+	intKey := createScalarNode(2, "2")
+	intKey.Tag = "!!int"
+	intVal := &CandidateNode{Kind: ScalarNode, Tag: "!!null"}
+
+	mapWithNullKey := &CandidateNode{
+		Kind:    MappingNode,
+		Content: []*CandidateNode{nullKey, nullVal},
+	}
+	mapWithIntKey := &CandidateNode{
+		Kind:    MappingNode,
+		Content: []*CandidateNode{intKey, intVal},
+	}
+	test.AssertResult(t, false, recurseNodeObjectEqual(mapWithNullKey, mapWithIntKey))
 }
 
 func TestParseInt(t *testing.T) {

pkg/yqlib/operation.go

@@ -164,6 +164,8 @@ var stringInterpolationOpType = &operationType{Type: "STRING_INT", NumArgs: 0, P
 var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 52, Handler: loadOperator, CheckForPostTraverse: true}
 var loadStringOpType = &operationType{Type: "LOAD_STRING", NumArgs: 1, Precedence: 52, Handler: loadStringOperator}
 
+var systemOpType = &operationType{Type: "SYSTEM", NumArgs: 1, Precedence: 50, Handler: systemOperator}
+
 var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 52, Handler: keysOperator, CheckForPostTraverse: true}
 
 var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator}

pkg/yqlib/operator_add_test.go

@@ -527,6 +527,18 @@ var addOperatorScenarios = []expressionScenario{
 		expression:    `.a += [2]`,
 		expectedError: "!!seq () cannot be added to a !!str (a)",
 	},
+	{
+		// Regression test for https://issues.oss-fuzz.com/issues/383860504
+		// Adding a map to itself must not panic when sequence keys contain
+		// single-entry mappings with a null key in one and a non-null key
+		// in the other.
+		skipDoc:    true,
+		document:   "? [{~: ~}]\n: v1\n? [{2: ~}]\n: v2",
+		expression: `. += .`,
+		expected: []string{
+			"D0, P[], (!!map)::? [{~: ~}]\n: v1\n? [{2: ~}]\n: v2\n",
+		},
+	},
 }
 
 func TestAddOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_contains.go

@@ -46,9 +46,9 @@ func containsObject(lhs *CandidateNode, rhs *CandidateNode) (bool, error) {
 		rhsKey := rhs.Content[index]
 		rhsValue := rhs.Content[index+1]
 		log.Debugf("Looking for %v in the lhs", rhsKey.Value)
-		lhsKeyIndex := findInArray(lhs, rhsKey)
+		lhsKeyIndex := findKeyInMap(lhs, rhsKey)
 		log.Debugf("index is %v", lhsKeyIndex)
-		if lhsKeyIndex < 0 || lhsKeyIndex%2 != 0 {
+		if lhsKeyIndex < 0 {
 			return false, nil
 		}
 		lhsValue := lhs.Content[lhsKeyIndex+1]

pkg/yqlib/operator_contains_test.go

@@ -65,6 +65,16 @@ var containsOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!bool)::false\n",
 		},
 	},
+	{
+		// Regression: findInArray could match a null key against a null
+		// value at an earlier odd index, producing a false negative.
+		skipDoc:    true,
+		document:   "? 1\n: ~\n? ~\n: x",
+		expression: `contains({~: "x"})`,
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
 	{
 		description: "String contains substring",
 		document:    `"foobar"`,

pkg/yqlib/operator_multiply.go

@@ -155,8 +155,10 @@ func repeatString(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error
 		return nil, err
 	} else if count < 0 {
 		return nil, fmt.Errorf("cannot repeat string by a negative number (%v)", count)
-	} else if count > 10000000 {
+	}
-		return nil, fmt.Errorf("cannot repeat string by more than 100 million (%v)", count)
+	maxResultLen := 10 * 1024 * 1024 // 10 MiB
+	if count > 0 && len(stringNode.Value) > maxResultLen/count {
+		return nil, fmt.Errorf("result of repeating string (%v bytes) by %v would exceed %v bytes", len(stringNode.Value), count, maxResultLen)
 	}
 	target.Value = strings.Repeat(stringNode.Value, count)
 

pkg/yqlib/operator_multiply_test.go

@@ -237,12 +237,11 @@ var multiplyOperatorScenarios = []expressionScenario{
 		expectedError: "cannot repeat string by a negative number (-4)",
 	},
 	{
-		description: "Multiply string X by more than 100 million",
+		description:   "Multiply string by count that exceeds result size limit",
-		// very large string.repeats causes a panic
 		skipDoc:       true,
 		document:      `n: 100000001`,
 		expression:    `"banana" * .n`,
-		expectedError: "cannot repeat string by more than 100 million (100000001)",
+		expectedError: "result of repeating string (6 bytes) by 100000001 would exceed 10485760 bytes",
 	},
 	{
 		description: "Multiply int node X string",
@@ -554,7 +553,7 @@ var multiplyOperatorScenarios = []expressionScenario{
 		document:   document,
 		expression: `.b * .c`,
 		expected: []string{
-			"D0, P[b], (!!map)::{name: dog, <<: *cat}\n",
+			"D0, P[b], (!!map)::{name: dog, \"<<\": *cat}\n",
 		},
 	},
 	{
@@ -693,6 +692,27 @@ var multiplyOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!null)::null\n",
 		},
 	},
+	{
+		// Regression test for https://issues.oss-fuzz.com/issues/418818862
+		// Large repeat count with a long string must not panic.
+		skipDoc:       true,
+		expression:    `"abc" * 99999999`,
+		expectedError: "result of repeating string (3 bytes) by 99999999 would exceed 10485760 bytes",
+	},
+	{
+		// Regression test for https://issues.oss-fuzz.com/issues/383195001
+		// Product of string length * repeat count must be bounded.
+		skipDoc:       true,
+		expression:    `"x" * 99999999`,
+		expectedError: "result of repeating string (1 bytes) by 99999999 would exceed 10485760 bytes",
+	},
+	{
+		// The size guard must not overflow: len * count can wrap to
+		// a negative or small value on 64-bit, bypassing the check.
+		skipDoc:       true,
+		expression:    `"ab" * 4611686018427387904`,
+		expectedError: "result of repeating string (2 bytes) by 4611686018427387904 would exceed 10485760 bytes",
+	},
 }
 
 func TestMultiplyOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_slice.go

@@ -16,6 +16,39 @@ func getSliceNumber(d *dataTreeNavigator, context Context, node *CandidateNode,
 	return parseInt(result.MatchingNodes.Front().Value.(*CandidateNode).Value)
 }
 
+// clampSliceIndex resolves a possibly-negative slice index against
+// length and clamps the result to [0, length].
+func clampSliceIndex(index, length int) int {
+	if index < 0 {
+		index += length
+	}
+	if index < 0 {
+		return 0
+	}
+	if index > length {
+		return length
+	}
+	return index
+}
+
+func sliceStringNode(lhsNode *CandidateNode, firstNumber int, secondNumber int) *CandidateNode {
+	runes := []rune(lhsNode.Value)
+	length := len(runes)
+
+	relativeFirstNumber := clampSliceIndex(firstNumber, length)
+	relativeSecondNumber := clampSliceIndex(secondNumber, length)
+	if relativeSecondNumber < relativeFirstNumber {
+		relativeSecondNumber = relativeFirstNumber
+	}
+
+	log.Debugf("sliceStringNode: slice from %v to %v", relativeFirstNumber, relativeSecondNumber)
+
+	slicedString := string(runes[relativeFirstNumber:relativeSecondNumber])
+	replacement := lhsNode.CreateReplacement(ScalarNode, lhsNode.Tag, slicedString)
+	replacement.Style = lhsNode.Style
+	return replacement
+}
+
 func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 
 	log.Debug("slice array operator!")
@@ -28,27 +61,23 @@ func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *E
 		lhsNode := el.Value.(*CandidateNode)
 
 		firstNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.LHS)
-
 		if err != nil {
 			return Context{}, err
 		}
-		relativeFirstNumber := firstNumber
-		if relativeFirstNumber < 0 {
-			relativeFirstNumber = len(lhsNode.Content) + firstNumber
-		}
 
 		secondNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.RHS)
 		if err != nil {
 			return Context{}, err
 		}
 
-		relativeSecondNumber := secondNumber
+		if lhsNode.Kind == ScalarNode && lhsNode.guessTagFromCustomType() == "!!str" {
-		if relativeSecondNumber < 0 {
+			results.PushBack(sliceStringNode(lhsNode, firstNumber, secondNumber))
-			relativeSecondNumber = len(lhsNode.Content) + secondNumber
+			continue
-		} else if relativeSecondNumber > len(lhsNode.Content) {
-			relativeSecondNumber = len(lhsNode.Content)
 		}
 
+		relativeFirstNumber := clampSliceIndex(firstNumber, len(lhsNode.Content))
+		relativeSecondNumber := clampSliceIndex(secondNumber, len(lhsNode.Content))
+
 		log.Debugf("calculateIndicesToTraverse: slice from %v to %v", relativeFirstNumber, relativeSecondNumber)
 
 		var newResults []*CandidateNode

pkg/yqlib/operator_slice_test.go

@@ -98,6 +98,115 @@ var sliceArrayScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::- cat1\n",
 		},
 	},
+	{
+		// Regression test for https://issues.oss-fuzz.com/issues/438776028
+		// Negative second index that underflows after adjustment must
+		// clamp to zero, yielding an empty sequence.
+		skipDoc:    true,
+		document:   `[a, b, c]`,
+		expression: `.[0:-99999]`,
+		expected: []string{
+			"D0, P[], (!!seq)::[]\n",
+		},
+	},
+	{
+		// First-index underflow: without clamping, the loop starts at a
+		// negative index and panics on Content access.
+		skipDoc:    true,
+		document:   `[a, b, c]`,
+		expression: `.[-99999:3]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- a\n- b\n- c\n",
+		},
+	},
+	{
+		// Both indices underflow: both clamp to zero, yielding an empty
+		// sequence.
+		skipDoc:    true,
+		document:   `[a, b, c]`,
+		expression: `.[-99999:-99998]`,
+		expected: []string{
+			"D0, P[], (!!seq)::[]\n",
+		},
+	},
+	{
+		description: "Slicing strings",
+		document:    `country: Australia`,
+		expression:  `.country[0:5]`,
+		expected: []string{
+			"D0, P[country], (!!str)::Austr\n",
+		},
+	},
+	{
+		description:    "Slicing strings - without the second number",
+		subdescription: "Finishes at the end of the string",
+		document:       `country: Australia`,
+		expression:     `.country[5:]`,
+		expected: []string{
+			"D0, P[country], (!!str)::alia\n",
+		},
+	},
+	{
+		description:    "Slicing strings - without the first number",
+		subdescription: "Starts from the start of the string",
+		document:       `country: Australia`,
+		expression:     `.country[:5]`,
+		expected: []string{
+			"D0, P[country], (!!str)::Austr\n",
+		},
+	},
+	{
+		description:    "Slicing strings - use negative numbers to count backwards from the end",
+		subdescription: "Negative indices count from the end of the string",
+		document:       `country: Australia`,
+		expression:     `.country[-5:]`,
+		expected: []string{
+			"D0, P[country], (!!str)::ralia\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `country: Australia`,
+		expression: `.country[1:-1]`,
+		expected: []string{
+			"D0, P[country], (!!str)::ustrali\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `country: Australia`,
+		expression: `.country[:]`,
+		expected: []string{
+			"D0, P[country], (!!str)::Australia\n",
+		},
+	},
+	{
+		skipDoc:     true,
+		description: "second index beyond string length clamps",
+		document:    `country: Australia`,
+		expression:  `.country[:100]`,
+		expected: []string{
+			"D0, P[country], (!!str)::Australia\n",
+		},
+	},
+	{
+		skipDoc:     true,
+		description: "first index beyond string length returns empty string",
+		document:    `country: Australia`,
+		expression:  `.country[100:]`,
+		expected: []string{
+			"D0, P[country], (!!str)::\n",
+		},
+	},
+	{
+		description:    "Slicing strings - Unicode",
+		subdescription: "Indices are rune-based, so multi-byte characters are handled correctly",
+		document:       `greeting: héllo`,
+		expression:     `.greeting[1:3]`,
+		expected: []string{
+			"D0, P[greeting], (!!str)::él\n",
+		},
+	},
 }
 
 func TestSliceOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_system.go

@@ -0,0 +1,146 @@
+package yqlib
+
+import (
+	"bytes"
+	"container/list"
+	"fmt"
+	"os/exec"
+	"strings"
+)
+
+func resolveSystemArgs(argsNode *CandidateNode) ([]string, error) {
+	if argsNode == nil {
+		return nil, nil
+	}
+
+	if argsNode.Kind == SequenceNode {
+		args := make([]string, 0, len(argsNode.Content))
+		for _, child := range argsNode.Content {
+			// Only non-null scalar children are valid arguments.
+			if child == nil {
+				continue
+			}
+			if child.Kind != ScalarNode || child.Tag == "!!null" {
+				return nil, fmt.Errorf("system operator: argument must be a non-null scalar; got kind=%v tag=%v", child.Kind, child.Tag)
+			}
+			args = append(args, child.Value)
+		}
+		if len(args) == 0 {
+			return nil, nil
+		}
+		return args, nil
+	}
+
+	// Single-argument case: only accept a non-null scalar node.
+	if argsNode.Tag == "!!null" {
+		return nil, nil
+	}
+	if argsNode.Kind != ScalarNode {
+		return nil, fmt.Errorf("system operator: args must be a non-null scalar or sequence of non-null scalars; got kind=%v tag=%v", argsNode.Kind, argsNode.Tag)
+	}
+	return []string{argsNode.Value}, nil
+}
+
+func resolveCommandNode(commandNodes Context) (string, error) {
+	if commandNodes.MatchingNodes.Front() == nil {
+		return "", fmt.Errorf("system operator: command expression returned no results")
+	}
+	if commandNodes.MatchingNodes.Len() > 1 {
+		log.Debugf("system operator: command expression returned %d results, using first", commandNodes.MatchingNodes.Len())
+	}
+	cmdNode := commandNodes.MatchingNodes.Front().Value.(*CandidateNode)
+	if cmdNode.Kind != ScalarNode || cmdNode.guessTagFromCustomType() != "!!str" {
+		return "", fmt.Errorf("system operator: command must be a string scalar")
+	}
+	if cmdNode.Value == "" {
+		return "", fmt.Errorf("system operator: command must be a non-empty string")
+	}
+	return cmdNode.Value, nil
+}
+
+func systemOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	if !ConfiguredSecurityPreferences.EnableSystemOps {
+		return Context{}, fmt.Errorf("system operations are disabled, use --security-enable-system-operator to enable")
+	}
+
+	// determine at parse time whether we have (command; args) or just (command)
+	hasArgs := expressionNode.RHS.Operation.OperationType == blockOpType
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		nodeContext := context.SingleReadonlyChildContext(candidate)
+
+		var command string
+		var args []string
+
+		if hasArgs {
+			block := expressionNode.RHS
+			commandNodes, err := d.GetMatchingNodes(nodeContext, block.LHS)
+			if err != nil {
+				return Context{}, err
+			}
+			command, err = resolveCommandNode(commandNodes)
+			if err != nil {
+				return Context{}, err
+			}
+
+			argsNodes, err := d.GetMatchingNodes(nodeContext, block.RHS)
+			if err != nil {
+				return Context{}, err
+			}
+			if argsNodes.MatchingNodes.Len() > 1 {
+				log.Debugf("system operator: args expression returned %d results, using first", argsNodes.MatchingNodes.Len())
+			}
+			if argsNodes.MatchingNodes.Front() != nil {
+				args, err = resolveSystemArgs(argsNodes.MatchingNodes.Front().Value.(*CandidateNode))
+				if err != nil {
+					return Context{}, err
+				}
+			}
+		} else {
+			commandNodes, err := d.GetMatchingNodes(nodeContext, expressionNode.RHS)
+			if err != nil {
+				return Context{}, err
+			}
+			command, err = resolveCommandNode(commandNodes)
+			if err != nil {
+				return Context{}, err
+			}
+		}
+
+		var stdin bytes.Buffer
+		encoded, err := encodeToYamlString(candidate)
+		if err != nil {
+			return Context{}, err
+		}
+		stdin.WriteString(encoded)
+
+		// #nosec G204 - intentional: user must explicitly enable this operator
+		cmd := exec.Command(command, args...)
+		cmd.Stdin = &stdin
+		var stderr bytes.Buffer
+		cmd.Stderr = &stderr
+
+		output, err := cmd.Output()
+		if err != nil {
+			stderrStr := strings.TrimSpace(stderr.String())
+			if stderrStr != "" {
+				return Context{}, fmt.Errorf("system command '%v' failed: %w\nstderr: %v", command, err, stderrStr)
+			}
+			return Context{}, fmt.Errorf("system command '%v' failed: %w", command, err)
+		}
+
+		result := string(output)
+		if strings.HasSuffix(result, "\r\n") {
+			result = result[:len(result)-2]
+		} else if strings.HasSuffix(result, "\n") {
+			result = result[:len(result)-1]
+		}
+		newNode := candidate.CreateReplacement(ScalarNode, "!!str", result)
+		results.PushBack(newNode)
+	}
+
+	return context.ChildContext(results), nil
+}

pkg/yqlib/operator_system_test.go

@@ -0,0 +1,123 @@
+package yqlib
+
+import (
+	"os/exec"
+	"testing"
+)
+
+func findExec(t *testing.T, name string) string {
+	t.Helper()
+	path, err := exec.LookPath(name)
+	if err != nil {
+		t.Skipf("skipping: %v not found: %v", name, err)
+	}
+	return path
+}
+
+var systemOperatorDisabledScenarios = []expressionScenario{
+	{
+		description:    "system operator returns error when disabled",
+		subdescription: "Use `--security-enable-system-operator` to enable the system operator.",
+		document:       "country: Australia",
+		expression:     `.country = system("/usr/bin/echo"; "test")`,
+		expectedError:  "system operations are disabled, use --security-enable-system-operator to enable",
+	},
+}
+
+func TestSystemOperatorDisabledScenarios(t *testing.T) {
+	originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps
+	defer func() {
+		ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps
+	}()
+
+	ConfiguredSecurityPreferences.EnableSystemOps = false
+
+	for _, tt := range systemOperatorDisabledScenarios {
+		testScenario(t, &tt)
+	}
+	documentOperatorScenarios(t, "system-operators", systemOperatorDisabledScenarios)
+}
+
+func TestSystemOperatorEnabledScenarios(t *testing.T) {
+	echoPath := findExec(t, "echo")
+	falsePath := findExec(t, "false")
+
+	originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps
+	defer func() {
+		ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps
+	}()
+
+	ConfiguredSecurityPreferences.EnableSystemOps = true
+
+	scenarios := []expressionScenario{
+		{
+			description:    "Run a command with an argument",
+			subdescription: "Use `--security-enable-system-operator` to enable the system operator.",
+			yqFlags:        "--security-enable-system-operator",
+			document:       "country: Australia",
+			expression:     `.country = system("` + echoPath + `"; "test")`,
+			expected: []string{
+				"D0, P[], (!!map)::country: test\n",
+			},
+		},
+		{
+			description:    "Run a command without arguments",
+			subdescription: "Omit the semicolon and args to run the command with no extra arguments.",
+			yqFlags:        "--security-enable-system-operator",
+			document:       "a: hello",
+			expression:     `.a = system("` + echoPath + `")`,
+			expected: []string{
+				"D0, P[], (!!map)::a: \"\"\n",
+			},
+		},
+		{
+			description:    "Run a command with multiple arguments",
+			subdescription: "Pass an array of arguments.",
+			skipDoc:        true,
+			document:       "a: hello",
+			expression:     `.a = system("` + echoPath + `"; ["foo", "bar"])`,
+			expected: []string{
+				"D0, P[], (!!map)::a: foo bar\n",
+			},
+		},
+		{
+			description: "Command and args are evaluated per matched node",
+			skipDoc:     true,
+			document:    "cmd: " + echoPath + "\narg: hello",
+			expression:  `.result = system(.cmd; .arg)`,
+			expected: []string{
+				"D0, P[], (!!map)::cmd: " + echoPath + "\narg: hello\nresult: hello\n",
+			},
+		},
+		{
+			description:   "Command failure returns error",
+			skipDoc:       true,
+			document:      "a: hello",
+			expression:    `.a = system("` + falsePath + `")`,
+			expectedError: "system command '" + falsePath + "' failed: exit status 1",
+		},
+		{
+			description:   "Null command returns error",
+			skipDoc:       true,
+			document:      "a: hello",
+			expression:    `.a = system(null)`,
+			expectedError: "system operator: command must be a string scalar",
+		},
+		{
+			description: "System operator processes multiple matched nodes",
+			skipDoc:     true,
+			document:    "a: first",
+			document2:   "a: second",
+			expression:  `.a = system("` + echoPath + `"; "replaced")`,
+			expected: []string{
+				"D0, P[], (!!map)::a: replaced\n",
+				"D0, P[], (!!map)::a: replaced\n",
+			},
+		},
+	}
+
+	for _, tt := range scenarios {
+		testScenario(t, &tt)
+	}
+	appendOperatorDocumentScenario(t, "system-operators", scenarios)
+}

pkg/yqlib/operator_traverse_path.go

@@ -36,9 +36,33 @@ func traversePathOperator(_ *dataTreeNavigator, context Context, expressionNode
 	return context.ChildContext(matches), nil
 }
 
+// resolveAliasChain follows an alias chain iteratively, returning the
+// first non-alias node. Returns an error if a cycle is detected.
+func resolveAliasChain(node *CandidateNode) (*CandidateNode, error) {
+	if node.Kind != AliasNode {
+		return node, nil
+	}
+	visited := map[*CandidateNode]bool{}
+	for node.Kind == AliasNode {
+		if visited[node] {
+			return nil, fmt.Errorf("alias cycle detected")
+		}
+		visited[node] = true
+		log.Debug("its an alias!")
+		node = node.Alias
+	}
+	return node, nil
+}
+
 func traverse(context Context, matchingNode *CandidateNode, operation *Operation) (*list.List, error) {
 	log.Debugf("Traversing %v", NodeToString(matchingNode))
 
+	var err error
+	matchingNode, err = resolveAliasChain(matchingNode)
+	if err != nil {
+		return nil, err
+	}
+
 	if matchingNode.Tag == "!!null" && operation.Value != "[]" && !context.DontAutoCreate {
 		log.Debugf("Guessing kind")
 		// we must have added this automatically, lets guess what it should be now
@@ -62,10 +86,6 @@ func traverse(context Context, matchingNode *CandidateNode, operation *Operation
 		log.Debugf("its a sequence of %v things!", len(matchingNode.Content))
 		return traverseArray(matchingNode, operation, operation.Preferences.(traversePreferences))
 
-	case AliasNode:
-		log.Debug("its an alias!")
-		matchingNode = matchingNode.Alias
-		return traverse(context, matchingNode, operation)
 	default:
 		return list.New(), nil
 	}
@@ -79,7 +99,11 @@ func traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode
 	log.Debugf("--traverseArrayOperator")
 
 	if expressionNode.RHS != nil && expressionNode.RHS.RHS != nil && expressionNode.RHS.RHS.Operation.OperationType == createMapOpType {
-		return sliceArrayOperator(d, context, expressionNode.RHS.RHS)
+		lhsContext, err := d.GetMatchingNodes(context, expressionNode.LHS)
+		if err != nil {
+			return Context{}, err
+		}
+		return sliceArrayOperator(d, lhsContext, expressionNode.RHS.RHS)
 	}
 
 	lhs, err := d.GetMatchingNodes(context, expressionNode.LHS)
@@ -125,7 +149,13 @@ func traverseNodesWithArrayIndices(context Context, indicesToTraverse []*Candida
 	return context.ChildContext(matchingNodeMap), nil
 }
 
-func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesToTraverse []*CandidateNode, prefs traversePreferences) (*list.List, error) { // call this if doc / alias like the other traverse
+func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesToTraverse []*CandidateNode, prefs traversePreferences) (*list.List, error) {
+	var err error
+	matchingNode, err = resolveAliasChain(matchingNode)
+	if err != nil {
+		return nil, err
+	}
+
 	if matchingNode.Tag == "!!null" {
 		log.Debugf("OperatorArrayTraverse got a null - turning it into an empty array")
 		// auto vivification
@@ -138,9 +168,6 @@ func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesT
 	}
 
 	switch matchingNode.Kind {
-	case AliasNode:
-		matchingNode = matchingNode.Alias
-		return traverseArrayIndices(context, matchingNode, indicesToTraverse, prefs)
 	case SequenceNode:
 		return traverseArrayWithIndices(matchingNode, indicesToTraverse, prefs)
 	case MappingNode:

pkg/yqlib/operator_traverse_path_test.go

@@ -665,6 +665,16 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[a], (!!null)::null\n",
 		},
 	},
+	{
+		// Regression test for https://issues.oss-fuzz.com/issues/390467412
+		// go-yaml accepts cross-document alias references (invalid per
+		// YAML spec). A nested assignment on such an alias can create a
+		// circular alias node, which must not cause a stack overflow.
+		skipDoc:       true,
+		document:      "&-- a\n---\n*--",
+		expression:    ". = (.x = 1)",
+		expectedError: "alias cycle detected",
+	},
 }
 
 func TestTraversePathOperatorScenarios(t *testing.T) {
@@ -682,3 +692,58 @@ func TestTraversePathOperatorAlignedToSpecScenarios(t *testing.T) {
 	appendOperatorDocumentScenario(t, "traverse-read", fixedTraversePathOperatorScenarios)
 	ConfiguredYamlPreferences.FixMergeAnchorToSpec = false
 }
+
+// Regression test for https://issues.oss-fuzz.com/issues/390467412
+// A circular alias (alias pointing back to itself) must not cause a
+// stack overflow. resolveAliasChain should detect the cycle and return
+// an error; both traverse() and traverseArrayIndices() use it.
+func TestTraverseAliasCycle(t *testing.T) {
+	aliasNode := &CandidateNode{
+		Kind: AliasNode,
+	}
+	aliasNode.Alias = aliasNode // A -> A
+
+	op := &Operation{
+		OperationType: traversePathOpType,
+		Value:         "key",
+		StringValue:   "key",
+		Preferences:   traversePreferences{},
+	}
+	_, err := traverse(Context{}, aliasNode, op)
+	if err == nil {
+		t.Fatal("expected error for alias cycle, got nil")
+	}
+	if err.Error() != "alias cycle detected" {
+		t.Fatalf("expected 'alias cycle detected', got %q", err.Error())
+	}
+
+	// Same cycle must be caught through the array traversal path.
+	_, err = traverseArrayIndices(Context{}, aliasNode, nil, traversePreferences{})
+	if err == nil {
+		t.Fatal("expected error for alias cycle via traverseArrayIndices, got nil")
+	}
+	if err.Error() != "alias cycle detected" {
+		t.Fatalf("expected 'alias cycle detected', got %q", err.Error())
+	}
+}
+
+func TestTraverseAliasCycleChain(t *testing.T) {
+	nodeA := &CandidateNode{Kind: AliasNode}
+	nodeB := &CandidateNode{Kind: AliasNode}
+	nodeA.Alias = nodeB
+	nodeB.Alias = nodeA // A -> B -> A
+
+	op := &Operation{
+		OperationType: traversePathOpType,
+		Value:         "key",
+		StringValue:   "key",
+		Preferences:   traversePreferences{},
+	}
+	_, err := traverse(Context{}, nodeA, op)
+	if err == nil {
+		t.Fatal("expected error for alias cycle chain, got nil")
+	}
+	if err.Error() != "alias cycle detected" {
+		t.Fatalf("expected 'alias cycle detected', got %q", err.Error())
+	}
+}

pkg/yqlib/operators_test.go

@@ -31,6 +31,7 @@ type expressionScenario struct {
 	dontFormatInputForDoc bool // dont format input doc for documentation generation
 	requiresFormat        string
 	skipForGoccy          bool
+	yqFlags               string // extra yq flags to include in generated doc command snippets
 }
 
 var goccyTesting = false
@@ -356,14 +357,22 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
 
 		writeOrPanic(w, "then\n")
 
+		flagsPrefix := ""
+		if s.yqFlags != "" {
+			flagsPrefix = s.yqFlags + " "
+		}
 		if s.expression != "" {
-			writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v'%v' %v\n```\n", envCommand, command, strings.ReplaceAll(s.expression, "'", `'\''`), files))
+			writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v'%v' %v\n```\n", envCommand, flagsPrefix, command, strings.ReplaceAll(s.expression, "'", `'\''`), files))
 		} else {
-			writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v\n```\n", envCommand, command, files))
+			writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v%v\n```\n", envCommand, flagsPrefix, command, files))
 		}
 	} else {
 		writeOrPanic(w, "Running\n")
-		writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v--null-input '%v'\n```\n", envCommand, command, s.expression))
+		flagsPrefix := ""
+		if s.yqFlags != "" {
+			flagsPrefix = s.yqFlags + " "
+		}
+		writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v--null-input '%v'\n```\n", envCommand, flagsPrefix, command, s.expression))
 	}
 	return formattedDoc, formattedDoc2
 }

pkg/yqlib/security_prefs.go

@@ -1,11 +1,13 @@
 package yqlib
 
 type SecurityPreferences struct {
-	DisableEnvOps  bool
+	DisableEnvOps   bool
-	DisableFileOps bool
+	DisableFileOps  bool
+	EnableSystemOps bool
 }
 
 var ConfiguredSecurityPreferences = SecurityPreferences{
-	DisableEnvOps:  false,
+	DisableEnvOps:   false,
-	DisableFileOps: false,
+	DisableFileOps:  false,
+	EnableSystemOps: false,
 }

pkg/yqlib/toml_test.go

@@ -287,6 +287,18 @@ var expectedSubArrays = `array:
           - {}
 `
 
+// Keys with special characters that require quoting in TOML
+var rtSpecialKeyInlineTable = `host = { "http://sealos.hub:5000" = { capabilities = ["pull", "resolve", "push"], skip_verify = true } }
+`
+
+var rtSpecialKeyTableSection = `["/tmp/blah"]
+value = "hello"
+`
+
+var rtSpecialKeyDottedTableSection = `[servers."http://localhost:8080"]
+ip = "127.0.0.1"
+`
+
 var tomlScenarios = []formatScenario{
 	{
 		skipDoc:      true,
@@ -614,6 +626,30 @@ var tomlScenarios = []formatScenario{
 		expected:     tomlTableWithComments,
 		scenarioType: "roundtrip",
 	},
+	{
+		skipDoc:      true,
+		description:  "Roundtrip: key with special characters in inline table",
+		input:        rtSpecialKeyInlineTable,
+		expression:   ".",
+		expected:     rtSpecialKeyInlineTable,
+		scenarioType: "roundtrip",
+	},
+	{
+		skipDoc:      true,
+		description:  "Roundtrip: key with special characters in table section",
+		input:        rtSpecialKeyTableSection,
+		expression:   ".",
+		expected:     rtSpecialKeyTableSection,
+		scenarioType: "roundtrip",
+	},
+	{
+		skipDoc:      true,
+		description:  "Roundtrip: special character key in dotted table section header",
+		input:        rtSpecialKeyDottedTableSection,
+		expression:   ".",
+		expected:     rtSpecialKeyDottedTableSection,
+		scenarioType: "roundtrip",
+	},
 }
 
 func testTomlScenario(t *testing.T, s formatScenario) {

project-words.txt

@@ -299,3 +299,8 @@ Ffile
 Fquery
 coverpkg
 gsub
+ralia
+Austr
+ustrali
+héllo
+alia