Compare commits

..

1 Commits

Author SHA1 Message Date
Jiri Tyr
1ad3677589
Merge ee21f7591f into 17f66dc6c6 2026-03-26 11:35:10 +01:00
31 changed files with 64 additions and 890 deletions

View File

@ -44,7 +44,7 @@ jobs:
./scripts/xcompile.sh ./scripts/xcompile.sh
- name: Release - name: Release
uses: softprops/action-gh-release@v2 uses: softprops/action-gh-release@v1
with: with:
files: build/* files: build/*
draft: true draft: true

View File

@ -1,4 +1,4 @@
FROM golang:1.26.2 AS builder FROM golang:1.26.1 AS builder
WORKDIR /go/src/mikefarah/yq WORKDIR /go/src/mikefarah/yq

View File

@ -1,4 +1,4 @@
FROM golang:1.26.2 FROM golang:1.26.1
RUN apt-get update && \ RUN apt-get update && \
apt-get install -y npm && \ apt-get install -y npm && \

View File

@ -212,7 +212,6 @@ yq -P -oy sample.json
rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableEnvOps, "security-disable-env-ops", "", false, "Disable env related operations.") rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableEnvOps, "security-disable-env-ops", "", false, "Disable env related operations.")
rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableFileOps, "security-disable-file-ops", "", false, "Disable file related operations (e.g. load)") rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableFileOps, "security-disable-file-ops", "", false, "Disable file related operations (e.g. load)")
rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.EnableSystemOps, "security-enable-system-operator", "", false, "Enable system operator to allow execution of external commands.")
rootCmd.AddCommand( rootCmd.AddCommand(
createEvaluateSequenceCommand(), createEvaluateSequenceCommand(),

4
go.mod
View File

@ -17,9 +17,9 @@ require (
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
github.com/spf13/cobra v1.10.2 github.com/spf13/cobra v1.10.2
github.com/spf13/pflag v1.0.10 github.com/spf13/pflag v1.0.10
github.com/yuin/gopher-lua v1.1.2 github.com/yuin/gopher-lua v1.1.1
github.com/zclconf/go-cty v1.18.0 github.com/zclconf/go-cty v1.18.0
go.yaml.in/yaml/v4 v4.0.0-rc.4 go.yaml.in/yaml/v4 v4.0.0-rc.3
golang.org/x/mod v0.34.0 golang.org/x/mod v0.34.0
golang.org/x/net v0.52.0 golang.org/x/net v0.52.0
golang.org/x/text v0.35.0 golang.org/x/text v0.35.0

8
go.sum
View File

@ -61,15 +61,15 @@ github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3A
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/yuin/gopher-lua v1.1.2 h1:yF/FjE3hD65tBbt0VXLE13HWS9h34fdzJmrWRXwobGA= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
github.com/yuin/gopher-lua v1.1.2/go.mod h1:7aRmXIWl37SqRf0koeyylBEzJ+aPt8A+mmkQ4f1ntR8= github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
github.com/zclconf/go-cty v1.18.0 h1:pJ8+HNI4gFoyRNqVE37wWbJWVw43BZczFo7KUoRczaA= github.com/zclconf/go-cty v1.18.0 h1:pJ8+HNI4gFoyRNqVE37wWbJWVw43BZczFo7KUoRczaA=
github.com/zclconf/go-cty v1.18.0/go.mod h1:qpnV6EDNgC1sns/AleL1fvatHw72j+S+nS+MJ+T2CSg= github.com/zclconf/go-cty v1.18.0/go.mod h1:qpnV6EDNgC1sns/AleL1fvatHw72j+S+nS+MJ+T2CSg=
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo= github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo=
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM= github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
go.yaml.in/yaml/v4 v4.0.0-rc.4 h1:UP4+v6fFrBIb1l934bDl//mmnoIZEDK0idg1+AIvX5U= go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go=
go.yaml.in/yaml/v4 v4.0.0-rc.4/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI= golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY= golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0= golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=

View File

@ -1,5 +1,5 @@
# Slice Array or String # Slice/Splice Array
The slice operator works on both arrays and strings. Like the `jq` equivalent, `.[10:15]` will return a subarray (or substring) of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array or string. The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.
You may leave out the first or second number, which will refer to the start or end of the array or string respectively. You may leave out the first or second number, which will refer to the start or end of the array respectively.

View File

@ -1,27 +0,0 @@
# System Operators
The `system` operator allows you to run an external command and use its output as a value in your expression.
**Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it.
**Note:** When enabled, the system operator can replicate the functionality of `env` and `load`
operators via external commands. Enabling it effectively overrides `--security-disable-env-ops`
and `--security-disable-file-ops`.
## Usage
```bash
yq --security-enable-system-operator --null-input '.field = system("command"; "arg1")'
```
The operator takes:
- A command string (required)
- An argument (or an array of arguments), separated from the command by `;` (optional)
The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string.
## Disabling the system operator
The system operator is disabled by default. When disabled, an error is returned instead of running the command, consistent with `--security-disable-env-ops` and `--security-disable-file-ops`.
Use `--security-enable-system-operator` flag to enable it.

View File

@ -1,8 +1,8 @@
# Slice Array or String # Slice/Splice Array
The slice operator works on both arrays and strings. Like the `jq` equivalent, `.[10:15]` will return a subarray (or substring) of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array or string. The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.
You may leave out the first or second number, which will refer to the start or end of the array or string respectively. You may leave out the first or second number, which will refer to the start or end of the array respectively.
## Slicing arrays ## Slicing arrays
Given a sample.yml file of: Given a sample.yml file of:
@ -103,81 +103,3 @@ will output
- cow - cow
``` ```
## Slicing strings
Given a sample.yml file of:
```yaml
country: Australia
```
then
```bash
yq '.country[0:5]' sample.yml
```
will output
```yaml
Austr
```
## Slicing strings - without the second number
Finishes at the end of the string
Given a sample.yml file of:
```yaml
country: Australia
```
then
```bash
yq '.country[5:]' sample.yml
```
will output
```yaml
alia
```
## Slicing strings - without the first number
Starts from the start of the string
Given a sample.yml file of:
```yaml
country: Australia
```
then
```bash
yq '.country[:5]' sample.yml
```
will output
```yaml
Austr
```
## Slicing strings - use negative numbers to count backwards from the end
Negative indices count from the end of the string
Given a sample.yml file of:
```yaml
country: Australia
```
then
```bash
yq '.country[-5:]' sample.yml
```
will output
```yaml
ralia
```
## Slicing strings - Unicode
Indices are rune-based, so multi-byte characters are handled correctly
Given a sample.yml file of:
```yaml
greeting: héllo
```
then
```bash
yq '.greeting[1:3]' sample.yml
```
will output
```yaml
él
```

View File

@ -1,76 +0,0 @@
# System Operators
The `system` operator allows you to run an external command and use its output as a value in your expression.
**Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it.
**Note:** When enabled, the system operator can replicate the functionality of `env` and `load`
operators via external commands. Enabling it effectively overrides `--security-disable-env-ops`
and `--security-disable-file-ops`.
## Usage
```bash
yq --security-enable-system-operator --null-input '.field = system("command"; "arg1")'
```
The operator takes:
- A command string (required)
- An argument (or an array of arguments), separated from the command by `;` (optional)
The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string.
## Disabling the system operator
The system operator is disabled by default. When disabled, an error is returned instead of running the command, consistent with `--security-disable-env-ops` and `--security-disable-file-ops`.
Use `--security-enable-system-operator` flag to enable it.
## system operator returns error when disabled
Use `--security-enable-system-operator` to enable the system operator.
Given a sample.yml file of:
```yaml
country: Australia
```
then
```bash
yq '.country = system("/usr/bin/echo"; "test")' sample.yml
```
will output
```bash
Error: system operations are disabled, use --security-enable-system-operator to enable
```
## Run a command with an argument
Use `--security-enable-system-operator` to enable the system operator.
Given a sample.yml file of:
```yaml
country: Australia
```
then
```bash
yq --security-enable-system-operator '.country = system("/usr/bin/echo"; "test")' sample.yml
```
will output
```yaml
country: test
```
## Run a command without arguments
Omit the semicolon and args to run the command with no extra arguments.
Given a sample.yml file of:
```yaml
a: hello
```
then
```bash
yq --security-enable-system-operator '.a = system("/usr/bin/echo")' sample.yml
```
will output
```yaml
a: ""
```

View File

@ -69,27 +69,6 @@ func (te *tomlEncoder) CanHandleAliases() bool {
// ---- helpers ---- // ---- helpers ----
// tomlKey returns the key quoted if it contains characters that are not valid
// in a TOML bare key. TOML bare keys may only contain ASCII letters, ASCII
// digits, underscores, and dashes.
func tomlKey(key string) string {
for _, r := range key {
if (r < 'A' || r > 'Z') && (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '_' && r != '-' {
return fmt.Sprintf("%q", key)
}
}
return key
}
// tomlDottedKey joins path components, quoting any that require it.
func tomlDottedKey(path []string) string {
parts := make([]string, len(path))
for i, p := range path {
parts[i] = tomlKey(p)
}
return strings.Join(parts, ".")
}
func (te *tomlEncoder) writeComment(w io.Writer, comment string) error { func (te *tomlEncoder) writeComment(w io.Writer, comment string) error {
if comment == "" { if comment == "" {
return nil return nil
@ -169,10 +148,9 @@ func (te *tomlEncoder) encodeTopLevelEntry(w io.Writer, path []string, node *Can
} }
if allMaps { if allMaps {
key := path[len(path)-1] key := path[len(path)-1]
quotedKey := tomlKey(key)
for _, it := range node.Content { for _, it := range node.Content {
// [[key]] then body // [[key]] then body
if _, err := w.Write([]byte("[[" + quotedKey + "]]\n")); err != nil { if _, err := w.Write([]byte("[[" + key + "]]\n")); err != nil {
return err return err
} }
if err := te.encodeMappingBodyWithPath(w, []string{key}, it); err != nil { if err := te.encodeMappingBodyWithPath(w, []string{key}, it); err != nil {
@ -207,7 +185,7 @@ func (te *tomlEncoder) writeAttribute(w io.Writer, key string, value *CandidateN
} }
// Write the attribute // Write the attribute
line := tomlKey(key) + " = " + te.formatScalar(value) line := key + " = " + te.formatScalar(value)
// Add line comment if present // Add line comment if present
if value.LineComment != "" { if value.LineComment != "" {
@ -232,7 +210,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida
// Handle empty arrays // Handle empty arrays
if len(seq.Content) == 0 { if len(seq.Content) == 0 {
line := tomlKey(key) + " = []" line := key + " = []"
if seq.LineComment != "" { if seq.LineComment != "" {
lineComment := strings.TrimSpace(seq.LineComment) lineComment := strings.TrimSpace(seq.LineComment)
if !strings.HasPrefix(lineComment, "#") { if !strings.HasPrefix(lineComment, "#") {
@ -255,7 +233,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida
if hasElementComments { if hasElementComments {
// Write multiline array format with comments // Write multiline array format with comments
if _, err := w.Write([]byte(tomlKey(key) + " = [\n")); err != nil { if _, err := w.Write([]byte(key + " = [\n")); err != nil {
return err return err
} }
@ -346,7 +324,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida
} }
} }
line := tomlKey(key) + " = [" + strings.Join(items, ", ") + "]" line := key + " = [" + strings.Join(items, ", ") + "]"
// Add line comment if present // Add line comment if present
if seq.LineComment != "" { if seq.LineComment != "" {
@ -394,21 +372,21 @@ func (te *tomlEncoder) mappingToInlineTable(m *CandidateNode) (string, error) {
v := m.Content[i+1] v := m.Content[i+1]
switch v.Kind { switch v.Kind {
case ScalarNode: case ScalarNode:
parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), te.formatScalar(v))) parts = append(parts, fmt.Sprintf("%s = %s", k, te.formatScalar(v)))
case SequenceNode: case SequenceNode:
// inline array in inline table // inline array in inline table
arr, err := te.sequenceToInlineArray(v) arr, err := te.sequenceToInlineArray(v)
if err != nil { if err != nil {
return "", err return "", err
} }
parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), arr)) parts = append(parts, fmt.Sprintf("%s = %s", k, arr))
case MappingNode: case MappingNode:
// nested inline table // nested inline table
inline, err := te.mappingToInlineTable(v) inline, err := te.mappingToInlineTable(v)
if err != nil { if err != nil {
return "", err return "", err
} }
parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), inline)) parts = append(parts, fmt.Sprintf("%s = %s", k, inline))
default: default:
return "", fmt.Errorf("unsupported inline table value kind: %v", v.Kind) return "", fmt.Errorf("unsupported inline table value kind: %v", v.Kind)
} }
@ -421,7 +399,7 @@ func (te *tomlEncoder) writeInlineTableAttribute(w io.Writer, key string, m *Can
if err != nil { if err != nil {
return err return err
} }
_, err = w.Write([]byte(tomlKey(key) + " = " + inline + "\n")) _, err = w.Write([]byte(key + " = " + inline + "\n"))
return err return err
} }
@ -443,7 +421,7 @@ func (te *tomlEncoder) writeTableHeader(w io.Writer, path []string, m *Candidate
} }
// Write table header [a.b.c] // Write table header [a.b.c]
header := "[" + tomlDottedKey(path) + "]\n" header := "[" + strings.Join(path, ".") + "]\n"
_, err := w.Write([]byte(header)) _, err := w.Write([]byte(header))
return err return err
} }
@ -510,7 +488,7 @@ func (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *Cand
} }
} }
if allMaps { if allMaps {
key := tomlDottedKey(append(append([]string{}, path...), k)) key := strings.Join(append(append([]string{}, path...), k), ".")
for _, it := range v.Content { for _, it := range v.Content {
if _, err := w.Write([]byte("[[" + key + "]]\n")); err != nil { if _, err := w.Write([]byte("[[" + key + "]]\n")); err != nil {
return err return err
@ -608,7 +586,7 @@ func (te *tomlEncoder) encodeMappingBodyWithPath(w io.Writer, path []string, m *
} }
} }
if allMaps { if allMaps {
dotted := tomlDottedKey(append(append([]string{}, path...), k)) dotted := strings.Join(append(append([]string{}, path...), k), ".")
for _, it := range v.Content { for _, it := range v.Content {
if _, err := w.Write([]byte("[[" + dotted + "]]\n")); err != nil { if _, err := w.Write([]byte("[[" + dotted + "]]\n")); err != nil {
return err return err

View File

@ -131,11 +131,6 @@ func handleToken(tokens []*token, index int, postProcessedTokens []*token) (toke
log.Debugf("previous token is : traverseArrayOpType") log.Debugf("previous token is : traverseArrayOpType")
// need to put the number 0 before this token, as that is implied // need to put the number 0 before this token, as that is implied
postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: createValueOperation(0, "0")}) postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: createValueOperation(0, "0")})
} else if index >= 2 && tokens[index-1].TokenType == openCollect &&
(tokens[index-2].TokenType == operationToken || tokens[index-2].TokenType == closeCollect || tokens[index-2].TokenType == closeCollectObject) {
log.Debugf("previous token is : openCollect following a traversal, implying 0 start")
// need to put the number 0 before this token, as that is implied
postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: createValueOperation(0, "0")})
} }
} }

View File

@ -96,8 +96,6 @@ var participleYqRules = []*participleYqRule{
simpleOp("load_?str|str_?load", loadStringOpType), simpleOp("load_?str|str_?load", loadStringOpType),
{"LoadYaml", `load`, loadOp(NewYamlDecoder(LoadYamlPreferences)), 0}, {"LoadYaml", `load`, loadOp(NewYamlDecoder(LoadYamlPreferences)), 0},
simpleOp("system", systemOpType),
{"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0}, {"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0},
simpleOp("select", selectOpType), simpleOp("select", selectOpType),

View File

@ -80,7 +80,7 @@ func recurseNodeObjectEqual(lhs *CandidateNode, rhs *CandidateNode) bool {
key := lhs.Content[index] key := lhs.Content[index]
value := lhs.Content[index+1] value := lhs.Content[index+1]
indexInRHS := findKeyInMap(rhs, key) indexInRHS := findInArray(rhs, key)
if indexInRHS == -1 || !recursiveNodeEqual(value, rhs.Content[indexInRHS+1]) { if indexInRHS == -1 || !recursiveNodeEqual(value, rhs.Content[indexInRHS+1]) {
return false return false

View File

@ -24,7 +24,7 @@ type parseSnippetScenario struct {
var parseSnippetScenarios = []parseSnippetScenario{ var parseSnippetScenarios = []parseSnippetScenario{
{ {
snippet: ":", snippet: ":",
expectedError: "yaml: while parsing a block mapping at <unknown position>: did not find expected key", expectedError: "yaml: did not find expected key",
}, },
{ {
snippet: "", snippet: "",
@ -300,24 +300,6 @@ func TestRecurseNodeObjectEqual(t *testing.T) {
test.AssertResult(t, true, recurseNodeObjectEqual(obj1, obj2)) test.AssertResult(t, true, recurseNodeObjectEqual(obj1, obj2))
test.AssertResult(t, false, recurseNodeObjectEqual(obj1, obj3)) test.AssertResult(t, false, recurseNodeObjectEqual(obj1, obj3))
test.AssertResult(t, false, recurseNodeObjectEqual(obj1, obj4)) test.AssertResult(t, false, recurseNodeObjectEqual(obj1, obj4))
// A null key must not match a null value in the other map.
// Regression test for https://issues.oss-fuzz.com/issues/383860504
nullKey := &CandidateNode{Kind: ScalarNode, Tag: "!!null"}
nullVal := &CandidateNode{Kind: ScalarNode, Tag: "!!null"}
intKey := createScalarNode(2, "2")
intKey.Tag = "!!int"
intVal := &CandidateNode{Kind: ScalarNode, Tag: "!!null"}
mapWithNullKey := &CandidateNode{
Kind: MappingNode,
Content: []*CandidateNode{nullKey, nullVal},
}
mapWithIntKey := &CandidateNode{
Kind: MappingNode,
Content: []*CandidateNode{intKey, intVal},
}
test.AssertResult(t, false, recurseNodeObjectEqual(mapWithNullKey, mapWithIntKey))
} }
func TestParseInt(t *testing.T) { func TestParseInt(t *testing.T) {

View File

@ -164,8 +164,6 @@ var stringInterpolationOpType = &operationType{Type: "STRING_INT", NumArgs: 0, P
var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 52, Handler: loadOperator, CheckForPostTraverse: true} var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 52, Handler: loadOperator, CheckForPostTraverse: true}
var loadStringOpType = &operationType{Type: "LOAD_STRING", NumArgs: 1, Precedence: 52, Handler: loadStringOperator} var loadStringOpType = &operationType{Type: "LOAD_STRING", NumArgs: 1, Precedence: 52, Handler: loadStringOperator}
var systemOpType = &operationType{Type: "SYSTEM", NumArgs: 1, Precedence: 50, Handler: systemOperator}
var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 52, Handler: keysOperator, CheckForPostTraverse: true} var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 52, Handler: keysOperator, CheckForPostTraverse: true}
var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator} var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator}

View File

@ -527,18 +527,6 @@ var addOperatorScenarios = []expressionScenario{
expression: `.a += [2]`, expression: `.a += [2]`,
expectedError: "!!seq () cannot be added to a !!str (a)", expectedError: "!!seq () cannot be added to a !!str (a)",
}, },
{
// Regression test for https://issues.oss-fuzz.com/issues/383860504
// Adding a map to itself must not panic when sequence keys contain
// single-entry mappings with a null key in one and a non-null key
// in the other.
skipDoc: true,
document: "? [{~: ~}]\n: v1\n? [{2: ~}]\n: v2",
expression: `. += .`,
expected: []string{
"D0, P[], (!!map)::? [{~: ~}]\n: v1\n? [{2: ~}]\n: v2\n",
},
},
} }
func TestAddOperatorScenarios(t *testing.T) { func TestAddOperatorScenarios(t *testing.T) {

View File

@ -46,9 +46,9 @@ func containsObject(lhs *CandidateNode, rhs *CandidateNode) (bool, error) {
rhsKey := rhs.Content[index] rhsKey := rhs.Content[index]
rhsValue := rhs.Content[index+1] rhsValue := rhs.Content[index+1]
log.Debugf("Looking for %v in the lhs", rhsKey.Value) log.Debugf("Looking for %v in the lhs", rhsKey.Value)
lhsKeyIndex := findKeyInMap(lhs, rhsKey) lhsKeyIndex := findInArray(lhs, rhsKey)
log.Debugf("index is %v", lhsKeyIndex) log.Debugf("index is %v", lhsKeyIndex)
if lhsKeyIndex < 0 { if lhsKeyIndex < 0 || lhsKeyIndex%2 != 0 {
return false, nil return false, nil
} }
lhsValue := lhs.Content[lhsKeyIndex+1] lhsValue := lhs.Content[lhsKeyIndex+1]

View File

@ -65,16 +65,6 @@ var containsOperatorScenarios = []expressionScenario{
"D0, P[], (!!bool)::false\n", "D0, P[], (!!bool)::false\n",
}, },
}, },
{
// Regression: findInArray could match a null key against a null
// value at an earlier odd index, producing a false negative.
skipDoc: true,
document: "? 1\n: ~\n? ~\n: x",
expression: `contains({~: "x"})`,
expected: []string{
"D0, P[], (!!bool)::true\n",
},
},
{ {
description: "String contains substring", description: "String contains substring",
document: `"foobar"`, document: `"foobar"`,

View File

@ -155,10 +155,8 @@ func repeatString(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error
return nil, err return nil, err
} else if count < 0 { } else if count < 0 {
return nil, fmt.Errorf("cannot repeat string by a negative number (%v)", count) return nil, fmt.Errorf("cannot repeat string by a negative number (%v)", count)
} } else if count > 10000000 {
maxResultLen := 10 * 1024 * 1024 // 10 MiB return nil, fmt.Errorf("cannot repeat string by more than 100 million (%v)", count)
if count > 0 && len(stringNode.Value) > maxResultLen/count {
return nil, fmt.Errorf("result of repeating string (%v bytes) by %v would exceed %v bytes", len(stringNode.Value), count, maxResultLen)
} }
target.Value = strings.Repeat(stringNode.Value, count) target.Value = strings.Repeat(stringNode.Value, count)

View File

@ -237,11 +237,12 @@ var multiplyOperatorScenarios = []expressionScenario{
expectedError: "cannot repeat string by a negative number (-4)", expectedError: "cannot repeat string by a negative number (-4)",
}, },
{ {
description: "Multiply string by count that exceeds result size limit", description: "Multiply string X by more than 100 million",
// very large string.repeats causes a panic
skipDoc: true, skipDoc: true,
document: `n: 100000001`, document: `n: 100000001`,
expression: `"banana" * .n`, expression: `"banana" * .n`,
expectedError: "result of repeating string (6 bytes) by 100000001 would exceed 10485760 bytes", expectedError: "cannot repeat string by more than 100 million (100000001)",
}, },
{ {
description: "Multiply int node X string", description: "Multiply int node X string",
@ -553,7 +554,7 @@ var multiplyOperatorScenarios = []expressionScenario{
document: document, document: document,
expression: `.b * .c`, expression: `.b * .c`,
expected: []string{ expected: []string{
"D0, P[b], (!!map)::{name: dog, \"<<\": *cat}\n", "D0, P[b], (!!map)::{name: dog, <<: *cat}\n",
}, },
}, },
{ {
@ -692,27 +693,6 @@ var multiplyOperatorScenarios = []expressionScenario{
"D0, P[], (!!null)::null\n", "D0, P[], (!!null)::null\n",
}, },
}, },
{
// Regression test for https://issues.oss-fuzz.com/issues/418818862
// Large repeat count with a long string must not panic.
skipDoc: true,
expression: `"abc" * 99999999`,
expectedError: "result of repeating string (3 bytes) by 99999999 would exceed 10485760 bytes",
},
{
// Regression test for https://issues.oss-fuzz.com/issues/383195001
// Product of string length * repeat count must be bounded.
skipDoc: true,
expression: `"x" * 99999999`,
expectedError: "result of repeating string (1 bytes) by 99999999 would exceed 10485760 bytes",
},
{
// The size guard must not overflow: len * count can wrap to
// a negative or small value on 64-bit, bypassing the check.
skipDoc: true,
expression: `"ab" * 4611686018427387904`,
expectedError: "result of repeating string (2 bytes) by 4611686018427387904 would exceed 10485760 bytes",
},
} }
func TestMultiplyOperatorScenarios(t *testing.T) { func TestMultiplyOperatorScenarios(t *testing.T) {

View File

@ -16,39 +16,6 @@ func getSliceNumber(d *dataTreeNavigator, context Context, node *CandidateNode,
return parseInt(result.MatchingNodes.Front().Value.(*CandidateNode).Value) return parseInt(result.MatchingNodes.Front().Value.(*CandidateNode).Value)
} }
// clampSliceIndex resolves a possibly-negative slice index against
// length and clamps the result to [0, length].
func clampSliceIndex(index, length int) int {
if index < 0 {
index += length
}
if index < 0 {
return 0
}
if index > length {
return length
}
return index
}
func sliceStringNode(lhsNode *CandidateNode, firstNumber int, secondNumber int) *CandidateNode {
runes := []rune(lhsNode.Value)
length := len(runes)
relativeFirstNumber := clampSliceIndex(firstNumber, length)
relativeSecondNumber := clampSliceIndex(secondNumber, length)
if relativeSecondNumber < relativeFirstNumber {
relativeSecondNumber = relativeFirstNumber
}
log.Debugf("sliceStringNode: slice from %v to %v", relativeFirstNumber, relativeSecondNumber)
slicedString := string(runes[relativeFirstNumber:relativeSecondNumber])
replacement := lhsNode.CreateReplacement(ScalarNode, lhsNode.Tag, slicedString)
replacement.Style = lhsNode.Style
return replacement
}
func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debug("slice array operator!") log.Debug("slice array operator!")
@ -61,23 +28,27 @@ func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *E
lhsNode := el.Value.(*CandidateNode) lhsNode := el.Value.(*CandidateNode)
firstNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.LHS) firstNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.LHS)
if err != nil { if err != nil {
return Context{}, err return Context{}, err
} }
relativeFirstNumber := firstNumber
if relativeFirstNumber < 0 {
relativeFirstNumber = len(lhsNode.Content) + firstNumber
}
secondNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.RHS) secondNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.RHS)
if err != nil { if err != nil {
return Context{}, err return Context{}, err
} }
if lhsNode.Kind == ScalarNode && lhsNode.guessTagFromCustomType() == "!!str" { relativeSecondNumber := secondNumber
results.PushBack(sliceStringNode(lhsNode, firstNumber, secondNumber)) if relativeSecondNumber < 0 {
continue relativeSecondNumber = len(lhsNode.Content) + secondNumber
} else if relativeSecondNumber > len(lhsNode.Content) {
relativeSecondNumber = len(lhsNode.Content)
} }
relativeFirstNumber := clampSliceIndex(firstNumber, len(lhsNode.Content))
relativeSecondNumber := clampSliceIndex(secondNumber, len(lhsNode.Content))
log.Debugf("calculateIndicesToTraverse: slice from %v to %v", relativeFirstNumber, relativeSecondNumber) log.Debugf("calculateIndicesToTraverse: slice from %v to %v", relativeFirstNumber, relativeSecondNumber)
var newResults []*CandidateNode var newResults []*CandidateNode

View File

@ -98,115 +98,6 @@ var sliceArrayScenarios = []expressionScenario{
"D0, P[], (!!seq)::- cat1\n", "D0, P[], (!!seq)::- cat1\n",
}, },
}, },
{
// Regression test for https://issues.oss-fuzz.com/issues/438776028
// Negative second index that underflows after adjustment must
// clamp to zero, yielding an empty sequence.
skipDoc: true,
document: `[a, b, c]`,
expression: `.[0:-99999]`,
expected: []string{
"D0, P[], (!!seq)::[]\n",
},
},
{
// First-index underflow: without clamping, the loop starts at a
// negative index and panics on Content access.
skipDoc: true,
document: `[a, b, c]`,
expression: `.[-99999:3]`,
expected: []string{
"D0, P[], (!!seq)::- a\n- b\n- c\n",
},
},
{
// Both indices underflow: both clamp to zero, yielding an empty
// sequence.
skipDoc: true,
document: `[a, b, c]`,
expression: `.[-99999:-99998]`,
expected: []string{
"D0, P[], (!!seq)::[]\n",
},
},
{
description: "Slicing strings",
document: `country: Australia`,
expression: `.country[0:5]`,
expected: []string{
"D0, P[country], (!!str)::Austr\n",
},
},
{
description: "Slicing strings - without the second number",
subdescription: "Finishes at the end of the string",
document: `country: Australia`,
expression: `.country[5:]`,
expected: []string{
"D0, P[country], (!!str)::alia\n",
},
},
{
description: "Slicing strings - without the first number",
subdescription: "Starts from the start of the string",
document: `country: Australia`,
expression: `.country[:5]`,
expected: []string{
"D0, P[country], (!!str)::Austr\n",
},
},
{
description: "Slicing strings - use negative numbers to count backwards from the end",
subdescription: "Negative indices count from the end of the string",
document: `country: Australia`,
expression: `.country[-5:]`,
expected: []string{
"D0, P[country], (!!str)::ralia\n",
},
},
{
skipDoc: true,
document: `country: Australia`,
expression: `.country[1:-1]`,
expected: []string{
"D0, P[country], (!!str)::ustrali\n",
},
},
{
skipDoc: true,
document: `country: Australia`,
expression: `.country[:]`,
expected: []string{
"D0, P[country], (!!str)::Australia\n",
},
},
{
skipDoc: true,
description: "second index beyond string length clamps",
document: `country: Australia`,
expression: `.country[:100]`,
expected: []string{
"D0, P[country], (!!str)::Australia\n",
},
},
{
skipDoc: true,
description: "first index beyond string length returns empty string",
document: `country: Australia`,
expression: `.country[100:]`,
expected: []string{
"D0, P[country], (!!str)::\n",
},
},
{
description: "Slicing strings - Unicode",
subdescription: "Indices are rune-based, so multi-byte characters are handled correctly",
document: `greeting: héllo`,
expression: `.greeting[1:3]`,
expected: []string{
"D0, P[greeting], (!!str)::él\n",
},
},
} }
func TestSliceOperatorScenarios(t *testing.T) { func TestSliceOperatorScenarios(t *testing.T) {

View File

@ -1,146 +0,0 @@
package yqlib
import (
"bytes"
"container/list"
"fmt"
"os/exec"
"strings"
)
func resolveSystemArgs(argsNode *CandidateNode) ([]string, error) {
if argsNode == nil {
return nil, nil
}
if argsNode.Kind == SequenceNode {
args := make([]string, 0, len(argsNode.Content))
for _, child := range argsNode.Content {
// Only non-null scalar children are valid arguments.
if child == nil {
continue
}
if child.Kind != ScalarNode || child.Tag == "!!null" {
return nil, fmt.Errorf("system operator: argument must be a non-null scalar; got kind=%v tag=%v", child.Kind, child.Tag)
}
args = append(args, child.Value)
}
if len(args) == 0 {
return nil, nil
}
return args, nil
}
// Single-argument case: only accept a non-null scalar node.
if argsNode.Tag == "!!null" {
return nil, nil
}
if argsNode.Kind != ScalarNode {
return nil, fmt.Errorf("system operator: args must be a non-null scalar or sequence of non-null scalars; got kind=%v tag=%v", argsNode.Kind, argsNode.Tag)
}
return []string{argsNode.Value}, nil
}
func resolveCommandNode(commandNodes Context) (string, error) {
if commandNodes.MatchingNodes.Front() == nil {
return "", fmt.Errorf("system operator: command expression returned no results")
}
if commandNodes.MatchingNodes.Len() > 1 {
log.Debugf("system operator: command expression returned %d results, using first", commandNodes.MatchingNodes.Len())
}
cmdNode := commandNodes.MatchingNodes.Front().Value.(*CandidateNode)
if cmdNode.Kind != ScalarNode || cmdNode.guessTagFromCustomType() != "!!str" {
return "", fmt.Errorf("system operator: command must be a string scalar")
}
if cmdNode.Value == "" {
return "", fmt.Errorf("system operator: command must be a non-empty string")
}
return cmdNode.Value, nil
}
func systemOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
if !ConfiguredSecurityPreferences.EnableSystemOps {
return Context{}, fmt.Errorf("system operations are disabled, use --security-enable-system-operator to enable")
}
// determine at parse time whether we have (command; args) or just (command)
hasArgs := expressionNode.RHS.Operation.OperationType == blockOpType
var results = list.New()
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
nodeContext := context.SingleReadonlyChildContext(candidate)
var command string
var args []string
if hasArgs {
block := expressionNode.RHS
commandNodes, err := d.GetMatchingNodes(nodeContext, block.LHS)
if err != nil {
return Context{}, err
}
command, err = resolveCommandNode(commandNodes)
if err != nil {
return Context{}, err
}
argsNodes, err := d.GetMatchingNodes(nodeContext, block.RHS)
if err != nil {
return Context{}, err
}
if argsNodes.MatchingNodes.Len() > 1 {
log.Debugf("system operator: args expression returned %d results, using first", argsNodes.MatchingNodes.Len())
}
if argsNodes.MatchingNodes.Front() != nil {
args, err = resolveSystemArgs(argsNodes.MatchingNodes.Front().Value.(*CandidateNode))
if err != nil {
return Context{}, err
}
}
} else {
commandNodes, err := d.GetMatchingNodes(nodeContext, expressionNode.RHS)
if err != nil {
return Context{}, err
}
command, err = resolveCommandNode(commandNodes)
if err != nil {
return Context{}, err
}
}
var stdin bytes.Buffer
encoded, err := encodeToYamlString(candidate)
if err != nil {
return Context{}, err
}
stdin.WriteString(encoded)
// #nosec G204 - intentional: user must explicitly enable this operator
cmd := exec.Command(command, args...)
cmd.Stdin = &stdin
var stderr bytes.Buffer
cmd.Stderr = &stderr
output, err := cmd.Output()
if err != nil {
stderrStr := strings.TrimSpace(stderr.String())
if stderrStr != "" {
return Context{}, fmt.Errorf("system command '%v' failed: %w\nstderr: %v", command, err, stderrStr)
}
return Context{}, fmt.Errorf("system command '%v' failed: %w", command, err)
}
result := string(output)
if strings.HasSuffix(result, "\r\n") {
result = result[:len(result)-2]
} else if strings.HasSuffix(result, "\n") {
result = result[:len(result)-1]
}
newNode := candidate.CreateReplacement(ScalarNode, "!!str", result)
results.PushBack(newNode)
}
return context.ChildContext(results), nil
}

View File

@ -1,123 +0,0 @@
package yqlib
import (
"os/exec"
"testing"
)
func findExec(t *testing.T, name string) string {
t.Helper()
path, err := exec.LookPath(name)
if err != nil {
t.Skipf("skipping: %v not found: %v", name, err)
}
return path
}
var systemOperatorDisabledScenarios = []expressionScenario{
{
description: "system operator returns error when disabled",
subdescription: "Use `--security-enable-system-operator` to enable the system operator.",
document: "country: Australia",
expression: `.country = system("/usr/bin/echo"; "test")`,
expectedError: "system operations are disabled, use --security-enable-system-operator to enable",
},
}
func TestSystemOperatorDisabledScenarios(t *testing.T) {
originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps
defer func() {
ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps
}()
ConfiguredSecurityPreferences.EnableSystemOps = false
for _, tt := range systemOperatorDisabledScenarios {
testScenario(t, &tt)
}
documentOperatorScenarios(t, "system-operators", systemOperatorDisabledScenarios)
}
func TestSystemOperatorEnabledScenarios(t *testing.T) {
echoPath := findExec(t, "echo")
falsePath := findExec(t, "false")
originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps
defer func() {
ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps
}()
ConfiguredSecurityPreferences.EnableSystemOps = true
scenarios := []expressionScenario{
{
description: "Run a command with an argument",
subdescription: "Use `--security-enable-system-operator` to enable the system operator.",
yqFlags: "--security-enable-system-operator",
document: "country: Australia",
expression: `.country = system("` + echoPath + `"; "test")`,
expected: []string{
"D0, P[], (!!map)::country: test\n",
},
},
{
description: "Run a command without arguments",
subdescription: "Omit the semicolon and args to run the command with no extra arguments.",
yqFlags: "--security-enable-system-operator",
document: "a: hello",
expression: `.a = system("` + echoPath + `")`,
expected: []string{
"D0, P[], (!!map)::a: \"\"\n",
},
},
{
description: "Run a command with multiple arguments",
subdescription: "Pass an array of arguments.",
skipDoc: true,
document: "a: hello",
expression: `.a = system("` + echoPath + `"; ["foo", "bar"])`,
expected: []string{
"D0, P[], (!!map)::a: foo bar\n",
},
},
{
description: "Command and args are evaluated per matched node",
skipDoc: true,
document: "cmd: " + echoPath + "\narg: hello",
expression: `.result = system(.cmd; .arg)`,
expected: []string{
"D0, P[], (!!map)::cmd: " + echoPath + "\narg: hello\nresult: hello\n",
},
},
{
description: "Command failure returns error",
skipDoc: true,
document: "a: hello",
expression: `.a = system("` + falsePath + `")`,
expectedError: "system command '" + falsePath + "' failed: exit status 1",
},
{
description: "Null command returns error",
skipDoc: true,
document: "a: hello",
expression: `.a = system(null)`,
expectedError: "system operator: command must be a string scalar",
},
{
description: "System operator processes multiple matched nodes",
skipDoc: true,
document: "a: first",
document2: "a: second",
expression: `.a = system("` + echoPath + `"; "replaced")`,
expected: []string{
"D0, P[], (!!map)::a: replaced\n",
"D0, P[], (!!map)::a: replaced\n",
},
},
}
for _, tt := range scenarios {
testScenario(t, &tt)
}
appendOperatorDocumentScenario(t, "system-operators", scenarios)
}

View File

@ -36,33 +36,9 @@ func traversePathOperator(_ *dataTreeNavigator, context Context, expressionNode
return context.ChildContext(matches), nil return context.ChildContext(matches), nil
} }
// resolveAliasChain follows an alias chain iteratively, returning the
// first non-alias node. Returns an error if a cycle is detected.
func resolveAliasChain(node *CandidateNode) (*CandidateNode, error) {
if node.Kind != AliasNode {
return node, nil
}
visited := map[*CandidateNode]bool{}
for node.Kind == AliasNode {
if visited[node] {
return nil, fmt.Errorf("alias cycle detected")
}
visited[node] = true
log.Debug("its an alias!")
node = node.Alias
}
return node, nil
}
func traverse(context Context, matchingNode *CandidateNode, operation *Operation) (*list.List, error) { func traverse(context Context, matchingNode *CandidateNode, operation *Operation) (*list.List, error) {
log.Debugf("Traversing %v", NodeToString(matchingNode)) log.Debugf("Traversing %v", NodeToString(matchingNode))
var err error
matchingNode, err = resolveAliasChain(matchingNode)
if err != nil {
return nil, err
}
if matchingNode.Tag == "!!null" && operation.Value != "[]" && !context.DontAutoCreate { if matchingNode.Tag == "!!null" && operation.Value != "[]" && !context.DontAutoCreate {
log.Debugf("Guessing kind") log.Debugf("Guessing kind")
// we must have added this automatically, lets guess what it should be now // we must have added this automatically, lets guess what it should be now
@ -86,6 +62,10 @@ func traverse(context Context, matchingNode *CandidateNode, operation *Operation
log.Debugf("its a sequence of %v things!", len(matchingNode.Content)) log.Debugf("its a sequence of %v things!", len(matchingNode.Content))
return traverseArray(matchingNode, operation, operation.Preferences.(traversePreferences)) return traverseArray(matchingNode, operation, operation.Preferences.(traversePreferences))
case AliasNode:
log.Debug("its an alias!")
matchingNode = matchingNode.Alias
return traverse(context, matchingNode, operation)
default: default:
return list.New(), nil return list.New(), nil
} }
@ -99,11 +79,7 @@ func traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode
log.Debugf("--traverseArrayOperator") log.Debugf("--traverseArrayOperator")
if expressionNode.RHS != nil && expressionNode.RHS.RHS != nil && expressionNode.RHS.RHS.Operation.OperationType == createMapOpType { if expressionNode.RHS != nil && expressionNode.RHS.RHS != nil && expressionNode.RHS.RHS.Operation.OperationType == createMapOpType {
lhsContext, err := d.GetMatchingNodes(context, expressionNode.LHS) return sliceArrayOperator(d, context, expressionNode.RHS.RHS)
if err != nil {
return Context{}, err
}
return sliceArrayOperator(d, lhsContext, expressionNode.RHS.RHS)
} }
lhs, err := d.GetMatchingNodes(context, expressionNode.LHS) lhs, err := d.GetMatchingNodes(context, expressionNode.LHS)
@ -149,13 +125,7 @@ func traverseNodesWithArrayIndices(context Context, indicesToTraverse []*Candida
return context.ChildContext(matchingNodeMap), nil return context.ChildContext(matchingNodeMap), nil
} }
func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesToTraverse []*CandidateNode, prefs traversePreferences) (*list.List, error) { func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesToTraverse []*CandidateNode, prefs traversePreferences) (*list.List, error) { // call this if doc / alias like the other traverse
var err error
matchingNode, err = resolveAliasChain(matchingNode)
if err != nil {
return nil, err
}
if matchingNode.Tag == "!!null" { if matchingNode.Tag == "!!null" {
log.Debugf("OperatorArrayTraverse got a null - turning it into an empty array") log.Debugf("OperatorArrayTraverse got a null - turning it into an empty array")
// auto vivification // auto vivification
@ -168,6 +138,9 @@ func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesT
} }
switch matchingNode.Kind { switch matchingNode.Kind {
case AliasNode:
matchingNode = matchingNode.Alias
return traverseArrayIndices(context, matchingNode, indicesToTraverse, prefs)
case SequenceNode: case SequenceNode:
return traverseArrayWithIndices(matchingNode, indicesToTraverse, prefs) return traverseArrayWithIndices(matchingNode, indicesToTraverse, prefs)
case MappingNode: case MappingNode:

View File

@ -665,16 +665,6 @@ var traversePathOperatorScenarios = []expressionScenario{
"D0, P[a], (!!null)::null\n", "D0, P[a], (!!null)::null\n",
}, },
}, },
{
// Regression test for https://issues.oss-fuzz.com/issues/390467412
// go-yaml accepts cross-document alias references (invalid per
// YAML spec). A nested assignment on such an alias can create a
// circular alias node, which must not cause a stack overflow.
skipDoc: true,
document: "&-- a\n---\n*--",
expression: ". = (.x = 1)",
expectedError: "alias cycle detected",
},
} }
func TestTraversePathOperatorScenarios(t *testing.T) { func TestTraversePathOperatorScenarios(t *testing.T) {
@ -692,58 +682,3 @@ func TestTraversePathOperatorAlignedToSpecScenarios(t *testing.T) {
appendOperatorDocumentScenario(t, "traverse-read", fixedTraversePathOperatorScenarios) appendOperatorDocumentScenario(t, "traverse-read", fixedTraversePathOperatorScenarios)
ConfiguredYamlPreferences.FixMergeAnchorToSpec = false ConfiguredYamlPreferences.FixMergeAnchorToSpec = false
} }
// Regression test for https://issues.oss-fuzz.com/issues/390467412
// A circular alias (alias pointing back to itself) must not cause a
// stack overflow. resolveAliasChain should detect the cycle and return
// an error; both traverse() and traverseArrayIndices() use it.
func TestTraverseAliasCycle(t *testing.T) {
aliasNode := &CandidateNode{
Kind: AliasNode,
}
aliasNode.Alias = aliasNode // A -> A
op := &Operation{
OperationType: traversePathOpType,
Value: "key",
StringValue: "key",
Preferences: traversePreferences{},
}
_, err := traverse(Context{}, aliasNode, op)
if err == nil {
t.Fatal("expected error for alias cycle, got nil")
}
if err.Error() != "alias cycle detected" {
t.Fatalf("expected 'alias cycle detected', got %q", err.Error())
}
// Same cycle must be caught through the array traversal path.
_, err = traverseArrayIndices(Context{}, aliasNode, nil, traversePreferences{})
if err == nil {
t.Fatal("expected error for alias cycle via traverseArrayIndices, got nil")
}
if err.Error() != "alias cycle detected" {
t.Fatalf("expected 'alias cycle detected', got %q", err.Error())
}
}
func TestTraverseAliasCycleChain(t *testing.T) {
nodeA := &CandidateNode{Kind: AliasNode}
nodeB := &CandidateNode{Kind: AliasNode}
nodeA.Alias = nodeB
nodeB.Alias = nodeA // A -> B -> A
op := &Operation{
OperationType: traversePathOpType,
Value: "key",
StringValue: "key",
Preferences: traversePreferences{},
}
_, err := traverse(Context{}, nodeA, op)
if err == nil {
t.Fatal("expected error for alias cycle chain, got nil")
}
if err.Error() != "alias cycle detected" {
t.Fatalf("expected 'alias cycle detected', got %q", err.Error())
}
}

View File

@ -31,7 +31,6 @@ type expressionScenario struct {
dontFormatInputForDoc bool // dont format input doc for documentation generation dontFormatInputForDoc bool // dont format input doc for documentation generation
requiresFormat string requiresFormat string
skipForGoccy bool skipForGoccy bool
yqFlags string // extra yq flags to include in generated doc command snippets
} }
var goccyTesting = false var goccyTesting = false
@ -357,22 +356,14 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
writeOrPanic(w, "then\n") writeOrPanic(w, "then\n")
flagsPrefix := ""
if s.yqFlags != "" {
flagsPrefix = s.yqFlags + " "
}
if s.expression != "" { if s.expression != "" {
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v'%v' %v\n```\n", envCommand, flagsPrefix, command, strings.ReplaceAll(s.expression, "'", `'\''`), files)) writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v'%v' %v\n```\n", envCommand, command, strings.ReplaceAll(s.expression, "'", `'\''`), files))
} else { } else {
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v%v\n```\n", envCommand, flagsPrefix, command, files)) writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v\n```\n", envCommand, command, files))
} }
} else { } else {
writeOrPanic(w, "Running\n") writeOrPanic(w, "Running\n")
flagsPrefix := "" writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v--null-input '%v'\n```\n", envCommand, command, s.expression))
if s.yqFlags != "" {
flagsPrefix = s.yqFlags + " "
}
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v--null-input '%v'\n```\n", envCommand, flagsPrefix, command, s.expression))
} }
return formattedDoc, formattedDoc2 return formattedDoc, formattedDoc2
} }

View File

@ -1,13 +1,11 @@
package yqlib package yqlib
type SecurityPreferences struct { type SecurityPreferences struct {
DisableEnvOps bool DisableEnvOps bool
DisableFileOps bool DisableFileOps bool
EnableSystemOps bool
} }
var ConfiguredSecurityPreferences = SecurityPreferences{ var ConfiguredSecurityPreferences = SecurityPreferences{
DisableEnvOps: false, DisableEnvOps: false,
DisableFileOps: false, DisableFileOps: false,
EnableSystemOps: false,
} }

View File

@ -287,18 +287,6 @@ var expectedSubArrays = `array:
- {} - {}
` `
// Keys with special characters that require quoting in TOML
var rtSpecialKeyInlineTable = `host = { "http://sealos.hub:5000" = { capabilities = ["pull", "resolve", "push"], skip_verify = true } }
`
var rtSpecialKeyTableSection = `["/tmp/blah"]
value = "hello"
`
var rtSpecialKeyDottedTableSection = `[servers."http://localhost:8080"]
ip = "127.0.0.1"
`
var tomlScenarios = []formatScenario{ var tomlScenarios = []formatScenario{
{ {
skipDoc: true, skipDoc: true,
@ -626,30 +614,6 @@ var tomlScenarios = []formatScenario{
expected: tomlTableWithComments, expected: tomlTableWithComments,
scenarioType: "roundtrip", scenarioType: "roundtrip",
}, },
{
skipDoc: true,
description: "Roundtrip: key with special characters in inline table",
input: rtSpecialKeyInlineTable,
expression: ".",
expected: rtSpecialKeyInlineTable,
scenarioType: "roundtrip",
},
{
skipDoc: true,
description: "Roundtrip: key with special characters in table section",
input: rtSpecialKeyTableSection,
expression: ".",
expected: rtSpecialKeyTableSection,
scenarioType: "roundtrip",
},
{
skipDoc: true,
description: "Roundtrip: special character key in dotted table section header",
input: rtSpecialKeyDottedTableSection,
expression: ".",
expected: rtSpecialKeyDottedTableSection,
scenarioType: "roundtrip",
},
} }
func testTomlScenario(t *testing.T, s formatScenario) { func testTomlScenario(t *testing.T, s formatScenario) {

View File

@ -298,9 +298,4 @@ subsubarray
Ffile Ffile
Fquery Fquery
coverpkg coverpkg
gsub gsub
ralia
Austr
ustrali
héllo
alia