Compare commits

..

1 Commits

Author SHA1 Message Date
Jiri Tyr
1ad3677589
Merge ee21f7591f into 17f66dc6c6 2026-03-26 11:35:10 +01:00
31 changed files with 64 additions and 890 deletions

View File

@ -44,7 +44,7 @@ jobs:
./scripts/xcompile.sh
- name: Release
uses: softprops/action-gh-release@v2
uses: softprops/action-gh-release@v1
with:
files: build/*
draft: true

View File

@ -1,4 +1,4 @@
FROM golang:1.26.2 AS builder
FROM golang:1.26.1 AS builder
WORKDIR /go/src/mikefarah/yq

View File

@ -1,4 +1,4 @@
FROM golang:1.26.2
FROM golang:1.26.1
RUN apt-get update && \
apt-get install -y npm && \

View File

@ -212,7 +212,6 @@ yq -P -oy sample.json
rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableEnvOps, "security-disable-env-ops", "", false, "Disable env related operations.")
rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableFileOps, "security-disable-file-ops", "", false, "Disable file related operations (e.g. load)")
rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.EnableSystemOps, "security-enable-system-operator", "", false, "Enable system operator to allow execution of external commands.")
rootCmd.AddCommand(
createEvaluateSequenceCommand(),

4
go.mod
View File

@ -17,9 +17,9 @@ require (
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
github.com/spf13/cobra v1.10.2
github.com/spf13/pflag v1.0.10
github.com/yuin/gopher-lua v1.1.2
github.com/yuin/gopher-lua v1.1.1
github.com/zclconf/go-cty v1.18.0
go.yaml.in/yaml/v4 v4.0.0-rc.4
go.yaml.in/yaml/v4 v4.0.0-rc.3
golang.org/x/mod v0.34.0
golang.org/x/net v0.52.0
golang.org/x/text v0.35.0

8
go.sum
View File

@ -61,15 +61,15 @@ github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3A
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/yuin/gopher-lua v1.1.2 h1:yF/FjE3hD65tBbt0VXLE13HWS9h34fdzJmrWRXwobGA=
github.com/yuin/gopher-lua v1.1.2/go.mod h1:7aRmXIWl37SqRf0koeyylBEzJ+aPt8A+mmkQ4f1ntR8=
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
github.com/zclconf/go-cty v1.18.0 h1:pJ8+HNI4gFoyRNqVE37wWbJWVw43BZczFo7KUoRczaA=
github.com/zclconf/go-cty v1.18.0/go.mod h1:qpnV6EDNgC1sns/AleL1fvatHw72j+S+nS+MJ+T2CSg=
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo=
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
go.yaml.in/yaml/v4 v4.0.0-rc.4 h1:UP4+v6fFrBIb1l934bDl//mmnoIZEDK0idg1+AIvX5U=
go.yaml.in/yaml/v4 v4.0.0-rc.4/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go=
go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=

View File

@ -1,5 +1,5 @@
# Slice Array or String
# Slice/Splice Array
The slice operator works on both arrays and strings. Like the `jq` equivalent, `.[10:15]` will return a subarray (or substring) of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array or string.
The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.
You may leave out the first or second number, which will refer to the start or end of the array or string respectively.
You may leave out the first or second number, which will refer to the start or end of the array respectively.

View File

@ -1,27 +0,0 @@
# System Operators
The `system` operator allows you to run an external command and use its output as a value in your expression.
**Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it.
**Note:** When enabled, the system operator can replicate the functionality of `env` and `load`
operators via external commands. Enabling it effectively overrides `--security-disable-env-ops`
and `--security-disable-file-ops`.
## Usage
```bash
yq --security-enable-system-operator --null-input '.field = system("command"; "arg1")'
```
The operator takes:
- A command string (required)
- An argument (or an array of arguments), separated from the command by `;` (optional)
The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string.
## Disabling the system operator
The system operator is disabled by default. When disabled, an error is returned instead of running the command, consistent with `--security-disable-env-ops` and `--security-disable-file-ops`.
Use `--security-enable-system-operator` flag to enable it.

View File

@ -1,8 +1,8 @@
# Slice Array or String
# Slice/Splice Array
The slice operator works on both arrays and strings. Like the `jq` equivalent, `.[10:15]` will return a subarray (or substring) of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array or string.
The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.
You may leave out the first or second number, which will refer to the start or end of the array or string respectively.
You may leave out the first or second number, which will refer to the start or end of the array respectively.
## Slicing arrays
Given a sample.yml file of:
@ -103,81 +103,3 @@ will output
- cow
```
## Slicing strings
Given a sample.yml file of:
```yaml
country: Australia
```
then
```bash
yq '.country[0:5]' sample.yml
```
will output
```yaml
Austr
```
## Slicing strings - without the second number
Finishes at the end of the string
Given a sample.yml file of:
```yaml
country: Australia
```
then
```bash
yq '.country[5:]' sample.yml
```
will output
```yaml
alia
```
## Slicing strings - without the first number
Starts from the start of the string
Given a sample.yml file of:
```yaml
country: Australia
```
then
```bash
yq '.country[:5]' sample.yml
```
will output
```yaml
Austr
```
## Slicing strings - use negative numbers to count backwards from the end
Negative indices count from the end of the string
Given a sample.yml file of:
```yaml
country: Australia
```
then
```bash
yq '.country[-5:]' sample.yml
```
will output
```yaml
ralia
```
## Slicing strings - Unicode
Indices are rune-based, so multi-byte characters are handled correctly
Given a sample.yml file of:
```yaml
greeting: héllo
```
then
```bash
yq '.greeting[1:3]' sample.yml
```
will output
```yaml
él
```

View File

@ -1,76 +0,0 @@
# System Operators
The `system` operator allows you to run an external command and use its output as a value in your expression.
**Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it.
**Note:** When enabled, the system operator can replicate the functionality of `env` and `load`
operators via external commands. Enabling it effectively overrides `--security-disable-env-ops`
and `--security-disable-file-ops`.
## Usage
```bash
yq --security-enable-system-operator --null-input '.field = system("command"; "arg1")'
```
The operator takes:
- A command string (required)
- An argument (or an array of arguments), separated from the command by `;` (optional)
The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string.
## Disabling the system operator
The system operator is disabled by default. When disabled, an error is returned instead of running the command, consistent with `--security-disable-env-ops` and `--security-disable-file-ops`.
Use `--security-enable-system-operator` flag to enable it.
## system operator returns error when disabled
Use `--security-enable-system-operator` to enable the system operator.
Given a sample.yml file of:
```yaml
country: Australia
```
then
```bash
yq '.country = system("/usr/bin/echo"; "test")' sample.yml
```
will output
```bash
Error: system operations are disabled, use --security-enable-system-operator to enable
```
## Run a command with an argument
Use `--security-enable-system-operator` to enable the system operator.
Given a sample.yml file of:
```yaml
country: Australia
```
then
```bash
yq --security-enable-system-operator '.country = system("/usr/bin/echo"; "test")' sample.yml
```
will output
```yaml
country: test
```
## Run a command without arguments
Omit the semicolon and args to run the command with no extra arguments.
Given a sample.yml file of:
```yaml
a: hello
```
then
```bash
yq --security-enable-system-operator '.a = system("/usr/bin/echo")' sample.yml
```
will output
```yaml
a: ""
```

View File

@ -69,27 +69,6 @@ func (te *tomlEncoder) CanHandleAliases() bool {
// ---- helpers ----
// tomlKey returns the key quoted if it contains characters that are not valid
// in a TOML bare key. TOML bare keys may only contain ASCII letters, ASCII
// digits, underscores, and dashes.
func tomlKey(key string) string {
for _, r := range key {
if (r < 'A' || r > 'Z') && (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '_' && r != '-' {
return fmt.Sprintf("%q", key)
}
}
return key
}
// tomlDottedKey joins path components, quoting any that require it.
func tomlDottedKey(path []string) string {
parts := make([]string, len(path))
for i, p := range path {
parts[i] = tomlKey(p)
}
return strings.Join(parts, ".")
}
func (te *tomlEncoder) writeComment(w io.Writer, comment string) error {
if comment == "" {
return nil
@ -169,10 +148,9 @@ func (te *tomlEncoder) encodeTopLevelEntry(w io.Writer, path []string, node *Can
}
if allMaps {
key := path[len(path)-1]
quotedKey := tomlKey(key)
for _, it := range node.Content {
// [[key]] then body
if _, err := w.Write([]byte("[[" + quotedKey + "]]\n")); err != nil {
if _, err := w.Write([]byte("[[" + key + "]]\n")); err != nil {
return err
}
if err := te.encodeMappingBodyWithPath(w, []string{key}, it); err != nil {
@ -207,7 +185,7 @@ func (te *tomlEncoder) writeAttribute(w io.Writer, key string, value *CandidateN
}
// Write the attribute
line := tomlKey(key) + " = " + te.formatScalar(value)
line := key + " = " + te.formatScalar(value)
// Add line comment if present
if value.LineComment != "" {
@ -232,7 +210,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida
// Handle empty arrays
if len(seq.Content) == 0 {
line := tomlKey(key) + " = []"
line := key + " = []"
if seq.LineComment != "" {
lineComment := strings.TrimSpace(seq.LineComment)
if !strings.HasPrefix(lineComment, "#") {
@ -255,7 +233,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida
if hasElementComments {
// Write multiline array format with comments
if _, err := w.Write([]byte(tomlKey(key) + " = [\n")); err != nil {
if _, err := w.Write([]byte(key + " = [\n")); err != nil {
return err
}
@ -346,7 +324,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida
}
}
line := tomlKey(key) + " = [" + strings.Join(items, ", ") + "]"
line := key + " = [" + strings.Join(items, ", ") + "]"
// Add line comment if present
if seq.LineComment != "" {
@ -394,21 +372,21 @@ func (te *tomlEncoder) mappingToInlineTable(m *CandidateNode) (string, error) {
v := m.Content[i+1]
switch v.Kind {
case ScalarNode:
parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), te.formatScalar(v)))
parts = append(parts, fmt.Sprintf("%s = %s", k, te.formatScalar(v)))
case SequenceNode:
// inline array in inline table
arr, err := te.sequenceToInlineArray(v)
if err != nil {
return "", err
}
parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), arr))
parts = append(parts, fmt.Sprintf("%s = %s", k, arr))
case MappingNode:
// nested inline table
inline, err := te.mappingToInlineTable(v)
if err != nil {
return "", err
}
parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), inline))
parts = append(parts, fmt.Sprintf("%s = %s", k, inline))
default:
return "", fmt.Errorf("unsupported inline table value kind: %v", v.Kind)
}
@ -421,7 +399,7 @@ func (te *tomlEncoder) writeInlineTableAttribute(w io.Writer, key string, m *Can
if err != nil {
return err
}
_, err = w.Write([]byte(tomlKey(key) + " = " + inline + "\n"))
_, err = w.Write([]byte(key + " = " + inline + "\n"))
return err
}
@ -443,7 +421,7 @@ func (te *tomlEncoder) writeTableHeader(w io.Writer, path []string, m *Candidate
}
// Write table header [a.b.c]
header := "[" + tomlDottedKey(path) + "]\n"
header := "[" + strings.Join(path, ".") + "]\n"
_, err := w.Write([]byte(header))
return err
}
@ -510,7 +488,7 @@ func (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *Cand
}
}
if allMaps {
key := tomlDottedKey(append(append([]string{}, path...), k))
key := strings.Join(append(append([]string{}, path...), k), ".")
for _, it := range v.Content {
if _, err := w.Write([]byte("[[" + key + "]]\n")); err != nil {
return err
@ -608,7 +586,7 @@ func (te *tomlEncoder) encodeMappingBodyWithPath(w io.Writer, path []string, m *
}
}
if allMaps {
dotted := tomlDottedKey(append(append([]string{}, path...), k))
dotted := strings.Join(append(append([]string{}, path...), k), ".")
for _, it := range v.Content {
if _, err := w.Write([]byte("[[" + dotted + "]]\n")); err != nil {
return err

View File

@ -131,11 +131,6 @@ func handleToken(tokens []*token, index int, postProcessedTokens []*token) (toke
log.Debugf("previous token is : traverseArrayOpType")
// need to put the number 0 before this token, as that is implied
postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: createValueOperation(0, "0")})
} else if index >= 2 && tokens[index-1].TokenType == openCollect &&
(tokens[index-2].TokenType == operationToken || tokens[index-2].TokenType == closeCollect || tokens[index-2].TokenType == closeCollectObject) {
log.Debugf("previous token is : openCollect following a traversal, implying 0 start")
// need to put the number 0 before this token, as that is implied
postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: createValueOperation(0, "0")})
}
}

View File

@ -96,8 +96,6 @@ var participleYqRules = []*participleYqRule{
simpleOp("load_?str|str_?load", loadStringOpType),
{"LoadYaml", `load`, loadOp(NewYamlDecoder(LoadYamlPreferences)), 0},
simpleOp("system", systemOpType),
{"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0},
simpleOp("select", selectOpType),

View File

@ -80,7 +80,7 @@ func recurseNodeObjectEqual(lhs *CandidateNode, rhs *CandidateNode) bool {
key := lhs.Content[index]
value := lhs.Content[index+1]
indexInRHS := findKeyInMap(rhs, key)
indexInRHS := findInArray(rhs, key)
if indexInRHS == -1 || !recursiveNodeEqual(value, rhs.Content[indexInRHS+1]) {
return false

View File

@ -24,7 +24,7 @@ type parseSnippetScenario struct {
var parseSnippetScenarios = []parseSnippetScenario{
{
snippet: ":",
expectedError: "yaml: while parsing a block mapping at <unknown position>: did not find expected key",
expectedError: "yaml: did not find expected key",
},
{
snippet: "",
@ -300,24 +300,6 @@ func TestRecurseNodeObjectEqual(t *testing.T) {
test.AssertResult(t, true, recurseNodeObjectEqual(obj1, obj2))
test.AssertResult(t, false, recurseNodeObjectEqual(obj1, obj3))
test.AssertResult(t, false, recurseNodeObjectEqual(obj1, obj4))
// A null key must not match a null value in the other map.
// Regression test for https://issues.oss-fuzz.com/issues/383860504
nullKey := &CandidateNode{Kind: ScalarNode, Tag: "!!null"}
nullVal := &CandidateNode{Kind: ScalarNode, Tag: "!!null"}
intKey := createScalarNode(2, "2")
intKey.Tag = "!!int"
intVal := &CandidateNode{Kind: ScalarNode, Tag: "!!null"}
mapWithNullKey := &CandidateNode{
Kind: MappingNode,
Content: []*CandidateNode{nullKey, nullVal},
}
mapWithIntKey := &CandidateNode{
Kind: MappingNode,
Content: []*CandidateNode{intKey, intVal},
}
test.AssertResult(t, false, recurseNodeObjectEqual(mapWithNullKey, mapWithIntKey))
}
func TestParseInt(t *testing.T) {

View File

@ -164,8 +164,6 @@ var stringInterpolationOpType = &operationType{Type: "STRING_INT", NumArgs: 0, P
var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 52, Handler: loadOperator, CheckForPostTraverse: true}
var loadStringOpType = &operationType{Type: "LOAD_STRING", NumArgs: 1, Precedence: 52, Handler: loadStringOperator}
var systemOpType = &operationType{Type: "SYSTEM", NumArgs: 1, Precedence: 50, Handler: systemOperator}
var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 52, Handler: keysOperator, CheckForPostTraverse: true}
var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator}

View File

@ -527,18 +527,6 @@ var addOperatorScenarios = []expressionScenario{
expression: `.a += [2]`,
expectedError: "!!seq () cannot be added to a !!str (a)",
},
{
// Regression test for https://issues.oss-fuzz.com/issues/383860504
// Adding a map to itself must not panic when sequence keys contain
// single-entry mappings with a null key in one and a non-null key
// in the other.
skipDoc: true,
document: "? [{~: ~}]\n: v1\n? [{2: ~}]\n: v2",
expression: `. += .`,
expected: []string{
"D0, P[], (!!map)::? [{~: ~}]\n: v1\n? [{2: ~}]\n: v2\n",
},
},
}
func TestAddOperatorScenarios(t *testing.T) {

View File

@ -46,9 +46,9 @@ func containsObject(lhs *CandidateNode, rhs *CandidateNode) (bool, error) {
rhsKey := rhs.Content[index]
rhsValue := rhs.Content[index+1]
log.Debugf("Looking for %v in the lhs", rhsKey.Value)
lhsKeyIndex := findKeyInMap(lhs, rhsKey)
lhsKeyIndex := findInArray(lhs, rhsKey)
log.Debugf("index is %v", lhsKeyIndex)
if lhsKeyIndex < 0 {
if lhsKeyIndex < 0 || lhsKeyIndex%2 != 0 {
return false, nil
}
lhsValue := lhs.Content[lhsKeyIndex+1]

View File

@ -65,16 +65,6 @@ var containsOperatorScenarios = []expressionScenario{
"D0, P[], (!!bool)::false\n",
},
},
{
// Regression: findInArray could match a null key against a null
// value at an earlier odd index, producing a false negative.
skipDoc: true,
document: "? 1\n: ~\n? ~\n: x",
expression: `contains({~: "x"})`,
expected: []string{
"D0, P[], (!!bool)::true\n",
},
},
{
description: "String contains substring",
document: `"foobar"`,

View File

@ -155,10 +155,8 @@ func repeatString(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error
return nil, err
} else if count < 0 {
return nil, fmt.Errorf("cannot repeat string by a negative number (%v)", count)
}
maxResultLen := 10 * 1024 * 1024 // 10 MiB
if count > 0 && len(stringNode.Value) > maxResultLen/count {
return nil, fmt.Errorf("result of repeating string (%v bytes) by %v would exceed %v bytes", len(stringNode.Value), count, maxResultLen)
} else if count > 10000000 {
return nil, fmt.Errorf("cannot repeat string by more than 100 million (%v)", count)
}
target.Value = strings.Repeat(stringNode.Value, count)

View File

@ -237,11 +237,12 @@ var multiplyOperatorScenarios = []expressionScenario{
expectedError: "cannot repeat string by a negative number (-4)",
},
{
description: "Multiply string by count that exceeds result size limit",
description: "Multiply string X by more than 100 million",
// very large string.repeats causes a panic
skipDoc: true,
document: `n: 100000001`,
expression: `"banana" * .n`,
expectedError: "result of repeating string (6 bytes) by 100000001 would exceed 10485760 bytes",
expectedError: "cannot repeat string by more than 100 million (100000001)",
},
{
description: "Multiply int node X string",
@ -553,7 +554,7 @@ var multiplyOperatorScenarios = []expressionScenario{
document: document,
expression: `.b * .c`,
expected: []string{
"D0, P[b], (!!map)::{name: dog, \"<<\": *cat}\n",
"D0, P[b], (!!map)::{name: dog, <<: *cat}\n",
},
},
{
@ -692,27 +693,6 @@ var multiplyOperatorScenarios = []expressionScenario{
"D0, P[], (!!null)::null\n",
},
},
{
// Regression test for https://issues.oss-fuzz.com/issues/418818862
// Large repeat count with a long string must not panic.
skipDoc: true,
expression: `"abc" * 99999999`,
expectedError: "result of repeating string (3 bytes) by 99999999 would exceed 10485760 bytes",
},
{
// Regression test for https://issues.oss-fuzz.com/issues/383195001
// Product of string length * repeat count must be bounded.
skipDoc: true,
expression: `"x" * 99999999`,
expectedError: "result of repeating string (1 bytes) by 99999999 would exceed 10485760 bytes",
},
{
// The size guard must not overflow: len * count can wrap to
// a negative or small value on 64-bit, bypassing the check.
skipDoc: true,
expression: `"ab" * 4611686018427387904`,
expectedError: "result of repeating string (2 bytes) by 4611686018427387904 would exceed 10485760 bytes",
},
}
func TestMultiplyOperatorScenarios(t *testing.T) {

View File

@ -16,39 +16,6 @@ func getSliceNumber(d *dataTreeNavigator, context Context, node *CandidateNode,
return parseInt(result.MatchingNodes.Front().Value.(*CandidateNode).Value)
}
// clampSliceIndex resolves a possibly-negative slice index against
// length and clamps the result to [0, length].
func clampSliceIndex(index, length int) int {
if index < 0 {
index += length
}
if index < 0 {
return 0
}
if index > length {
return length
}
return index
}
func sliceStringNode(lhsNode *CandidateNode, firstNumber int, secondNumber int) *CandidateNode {
runes := []rune(lhsNode.Value)
length := len(runes)
relativeFirstNumber := clampSliceIndex(firstNumber, length)
relativeSecondNumber := clampSliceIndex(secondNumber, length)
if relativeSecondNumber < relativeFirstNumber {
relativeSecondNumber = relativeFirstNumber
}
log.Debugf("sliceStringNode: slice from %v to %v", relativeFirstNumber, relativeSecondNumber)
slicedString := string(runes[relativeFirstNumber:relativeSecondNumber])
replacement := lhsNode.CreateReplacement(ScalarNode, lhsNode.Tag, slicedString)
replacement.Style = lhsNode.Style
return replacement
}
func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debug("slice array operator!")
@ -61,23 +28,27 @@ func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *E
lhsNode := el.Value.(*CandidateNode)
firstNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.LHS)
if err != nil {
return Context{}, err
}
relativeFirstNumber := firstNumber
if relativeFirstNumber < 0 {
relativeFirstNumber = len(lhsNode.Content) + firstNumber
}
secondNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.RHS)
if err != nil {
return Context{}, err
}
if lhsNode.Kind == ScalarNode && lhsNode.guessTagFromCustomType() == "!!str" {
results.PushBack(sliceStringNode(lhsNode, firstNumber, secondNumber))
continue
relativeSecondNumber := secondNumber
if relativeSecondNumber < 0 {
relativeSecondNumber = len(lhsNode.Content) + secondNumber
} else if relativeSecondNumber > len(lhsNode.Content) {
relativeSecondNumber = len(lhsNode.Content)
}
relativeFirstNumber := clampSliceIndex(firstNumber, len(lhsNode.Content))
relativeSecondNumber := clampSliceIndex(secondNumber, len(lhsNode.Content))
log.Debugf("calculateIndicesToTraverse: slice from %v to %v", relativeFirstNumber, relativeSecondNumber)
var newResults []*CandidateNode

View File

@ -98,115 +98,6 @@ var sliceArrayScenarios = []expressionScenario{
"D0, P[], (!!seq)::- cat1\n",
},
},
{
// Regression test for https://issues.oss-fuzz.com/issues/438776028
// Negative second index that underflows after adjustment must
// clamp to zero, yielding an empty sequence.
skipDoc: true,
document: `[a, b, c]`,
expression: `.[0:-99999]`,
expected: []string{
"D0, P[], (!!seq)::[]\n",
},
},
{
// First-index underflow: without clamping, the loop starts at a
// negative index and panics on Content access.
skipDoc: true,
document: `[a, b, c]`,
expression: `.[-99999:3]`,
expected: []string{
"D0, P[], (!!seq)::- a\n- b\n- c\n",
},
},
{
// Both indices underflow: both clamp to zero, yielding an empty
// sequence.
skipDoc: true,
document: `[a, b, c]`,
expression: `.[-99999:-99998]`,
expected: []string{
"D0, P[], (!!seq)::[]\n",
},
},
{
description: "Slicing strings",
document: `country: Australia`,
expression: `.country[0:5]`,
expected: []string{
"D0, P[country], (!!str)::Austr\n",
},
},
{
description: "Slicing strings - without the second number",
subdescription: "Finishes at the end of the string",
document: `country: Australia`,
expression: `.country[5:]`,
expected: []string{
"D0, P[country], (!!str)::alia\n",
},
},
{
description: "Slicing strings - without the first number",
subdescription: "Starts from the start of the string",
document: `country: Australia`,
expression: `.country[:5]`,
expected: []string{
"D0, P[country], (!!str)::Austr\n",
},
},
{
description: "Slicing strings - use negative numbers to count backwards from the end",
subdescription: "Negative indices count from the end of the string",
document: `country: Australia`,
expression: `.country[-5:]`,
expected: []string{
"D0, P[country], (!!str)::ralia\n",
},
},
{
skipDoc: true,
document: `country: Australia`,
expression: `.country[1:-1]`,
expected: []string{
"D0, P[country], (!!str)::ustrali\n",
},
},
{
skipDoc: true,
document: `country: Australia`,
expression: `.country[:]`,
expected: []string{
"D0, P[country], (!!str)::Australia\n",
},
},
{
skipDoc: true,
description: "second index beyond string length clamps",
document: `country: Australia`,
expression: `.country[:100]`,
expected: []string{
"D0, P[country], (!!str)::Australia\n",
},
},
{
skipDoc: true,
description: "first index beyond string length returns empty string",
document: `country: Australia`,
expression: `.country[100:]`,
expected: []string{
"D0, P[country], (!!str)::\n",
},
},
{
description: "Slicing strings - Unicode",
subdescription: "Indices are rune-based, so multi-byte characters are handled correctly",
document: `greeting: héllo`,
expression: `.greeting[1:3]`,
expected: []string{
"D0, P[greeting], (!!str)::él\n",
},
},
}
func TestSliceOperatorScenarios(t *testing.T) {

View File

@ -1,146 +0,0 @@
package yqlib
import (
"bytes"
"container/list"
"fmt"
"os/exec"
"strings"
)
func resolveSystemArgs(argsNode *CandidateNode) ([]string, error) {
if argsNode == nil {
return nil, nil
}
if argsNode.Kind == SequenceNode {
args := make([]string, 0, len(argsNode.Content))
for _, child := range argsNode.Content {
// Only non-null scalar children are valid arguments.
if child == nil {
continue
}
if child.Kind != ScalarNode || child.Tag == "!!null" {
return nil, fmt.Errorf("system operator: argument must be a non-null scalar; got kind=%v tag=%v", child.Kind, child.Tag)
}
args = append(args, child.Value)
}
if len(args) == 0 {
return nil, nil
}
return args, nil
}
// Single-argument case: only accept a non-null scalar node.
if argsNode.Tag == "!!null" {
return nil, nil
}
if argsNode.Kind != ScalarNode {
return nil, fmt.Errorf("system operator: args must be a non-null scalar or sequence of non-null scalars; got kind=%v tag=%v", argsNode.Kind, argsNode.Tag)
}
return []string{argsNode.Value}, nil
}
func resolveCommandNode(commandNodes Context) (string, error) {
if commandNodes.MatchingNodes.Front() == nil {
return "", fmt.Errorf("system operator: command expression returned no results")
}
if commandNodes.MatchingNodes.Len() > 1 {
log.Debugf("system operator: command expression returned %d results, using first", commandNodes.MatchingNodes.Len())
}
cmdNode := commandNodes.MatchingNodes.Front().Value.(*CandidateNode)
if cmdNode.Kind != ScalarNode || cmdNode.guessTagFromCustomType() != "!!str" {
return "", fmt.Errorf("system operator: command must be a string scalar")
}
if cmdNode.Value == "" {
return "", fmt.Errorf("system operator: command must be a non-empty string")
}
return cmdNode.Value, nil
}
func systemOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
if !ConfiguredSecurityPreferences.EnableSystemOps {
return Context{}, fmt.Errorf("system operations are disabled, use --security-enable-system-operator to enable")
}
// determine at parse time whether we have (command; args) or just (command)
hasArgs := expressionNode.RHS.Operation.OperationType == blockOpType
var results = list.New()
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
nodeContext := context.SingleReadonlyChildContext(candidate)
var command string
var args []string
if hasArgs {
block := expressionNode.RHS
commandNodes, err := d.GetMatchingNodes(nodeContext, block.LHS)
if err != nil {
return Context{}, err
}
command, err = resolveCommandNode(commandNodes)
if err != nil {
return Context{}, err
}
argsNodes, err := d.GetMatchingNodes(nodeContext, block.RHS)
if err != nil {
return Context{}, err
}
if argsNodes.MatchingNodes.Len() > 1 {
log.Debugf("system operator: args expression returned %d results, using first", argsNodes.MatchingNodes.Len())
}
if argsNodes.MatchingNodes.Front() != nil {
args, err = resolveSystemArgs(argsNodes.MatchingNodes.Front().Value.(*CandidateNode))
if err != nil {
return Context{}, err
}
}
} else {
commandNodes, err := d.GetMatchingNodes(nodeContext, expressionNode.RHS)
if err != nil {
return Context{}, err
}
command, err = resolveCommandNode(commandNodes)
if err != nil {
return Context{}, err
}
}
var stdin bytes.Buffer
encoded, err := encodeToYamlString(candidate)
if err != nil {
return Context{}, err
}
stdin.WriteString(encoded)
// #nosec G204 - intentional: user must explicitly enable this operator
cmd := exec.Command(command, args...)
cmd.Stdin = &stdin
var stderr bytes.Buffer
cmd.Stderr = &stderr
output, err := cmd.Output()
if err != nil {
stderrStr := strings.TrimSpace(stderr.String())
if stderrStr != "" {
return Context{}, fmt.Errorf("system command '%v' failed: %w\nstderr: %v", command, err, stderrStr)
}
return Context{}, fmt.Errorf("system command '%v' failed: %w", command, err)
}
result := string(output)
if strings.HasSuffix(result, "\r\n") {
result = result[:len(result)-2]
} else if strings.HasSuffix(result, "\n") {
result = result[:len(result)-1]
}
newNode := candidate.CreateReplacement(ScalarNode, "!!str", result)
results.PushBack(newNode)
}
return context.ChildContext(results), nil
}

View File

@ -1,123 +0,0 @@
package yqlib
import (
"os/exec"
"testing"
)
func findExec(t *testing.T, name string) string {
t.Helper()
path, err := exec.LookPath(name)
if err != nil {
t.Skipf("skipping: %v not found: %v", name, err)
}
return path
}
var systemOperatorDisabledScenarios = []expressionScenario{
{
description: "system operator returns error when disabled",
subdescription: "Use `--security-enable-system-operator` to enable the system operator.",
document: "country: Australia",
expression: `.country = system("/usr/bin/echo"; "test")`,
expectedError: "system operations are disabled, use --security-enable-system-operator to enable",
},
}
func TestSystemOperatorDisabledScenarios(t *testing.T) {
originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps
defer func() {
ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps
}()
ConfiguredSecurityPreferences.EnableSystemOps = false
for _, tt := range systemOperatorDisabledScenarios {
testScenario(t, &tt)
}
documentOperatorScenarios(t, "system-operators", systemOperatorDisabledScenarios)
}
func TestSystemOperatorEnabledScenarios(t *testing.T) {
echoPath := findExec(t, "echo")
falsePath := findExec(t, "false")
originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps
defer func() {
ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps
}()
ConfiguredSecurityPreferences.EnableSystemOps = true
scenarios := []expressionScenario{
{
description: "Run a command with an argument",
subdescription: "Use `--security-enable-system-operator` to enable the system operator.",
yqFlags: "--security-enable-system-operator",
document: "country: Australia",
expression: `.country = system("` + echoPath + `"; "test")`,
expected: []string{
"D0, P[], (!!map)::country: test\n",
},
},
{
description: "Run a command without arguments",
subdescription: "Omit the semicolon and args to run the command with no extra arguments.",
yqFlags: "--security-enable-system-operator",
document: "a: hello",
expression: `.a = system("` + echoPath + `")`,
expected: []string{
"D0, P[], (!!map)::a: \"\"\n",
},
},
{
description: "Run a command with multiple arguments",
subdescription: "Pass an array of arguments.",
skipDoc: true,
document: "a: hello",
expression: `.a = system("` + echoPath + `"; ["foo", "bar"])`,
expected: []string{
"D0, P[], (!!map)::a: foo bar\n",
},
},
{
description: "Command and args are evaluated per matched node",
skipDoc: true,
document: "cmd: " + echoPath + "\narg: hello",
expression: `.result = system(.cmd; .arg)`,
expected: []string{
"D0, P[], (!!map)::cmd: " + echoPath + "\narg: hello\nresult: hello\n",
},
},
{
description: "Command failure returns error",
skipDoc: true,
document: "a: hello",
expression: `.a = system("` + falsePath + `")`,
expectedError: "system command '" + falsePath + "' failed: exit status 1",
},
{
description: "Null command returns error",
skipDoc: true,
document: "a: hello",
expression: `.a = system(null)`,
expectedError: "system operator: command must be a string scalar",
},
{
description: "System operator processes multiple matched nodes",
skipDoc: true,
document: "a: first",
document2: "a: second",
expression: `.a = system("` + echoPath + `"; "replaced")`,
expected: []string{
"D0, P[], (!!map)::a: replaced\n",
"D0, P[], (!!map)::a: replaced\n",
},
},
}
for _, tt := range scenarios {
testScenario(t, &tt)
}
appendOperatorDocumentScenario(t, "system-operators", scenarios)
}

View File

@ -36,33 +36,9 @@ func traversePathOperator(_ *dataTreeNavigator, context Context, expressionNode
return context.ChildContext(matches), nil
}
// resolveAliasChain follows an alias chain iteratively, returning the
// first non-alias node. Returns an error if a cycle is detected.
func resolveAliasChain(node *CandidateNode) (*CandidateNode, error) {
if node.Kind != AliasNode {
return node, nil
}
visited := map[*CandidateNode]bool{}
for node.Kind == AliasNode {
if visited[node] {
return nil, fmt.Errorf("alias cycle detected")
}
visited[node] = true
log.Debug("its an alias!")
node = node.Alias
}
return node, nil
}
func traverse(context Context, matchingNode *CandidateNode, operation *Operation) (*list.List, error) {
log.Debugf("Traversing %v", NodeToString(matchingNode))
var err error
matchingNode, err = resolveAliasChain(matchingNode)
if err != nil {
return nil, err
}
if matchingNode.Tag == "!!null" && operation.Value != "[]" && !context.DontAutoCreate {
log.Debugf("Guessing kind")
// we must have added this automatically, lets guess what it should be now
@ -86,6 +62,10 @@ func traverse(context Context, matchingNode *CandidateNode, operation *Operation
log.Debugf("its a sequence of %v things!", len(matchingNode.Content))
return traverseArray(matchingNode, operation, operation.Preferences.(traversePreferences))
case AliasNode:
log.Debug("its an alias!")
matchingNode = matchingNode.Alias
return traverse(context, matchingNode, operation)
default:
return list.New(), nil
}
@ -99,11 +79,7 @@ func traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode
log.Debugf("--traverseArrayOperator")
if expressionNode.RHS != nil && expressionNode.RHS.RHS != nil && expressionNode.RHS.RHS.Operation.OperationType == createMapOpType {
lhsContext, err := d.GetMatchingNodes(context, expressionNode.LHS)
if err != nil {
return Context{}, err
}
return sliceArrayOperator(d, lhsContext, expressionNode.RHS.RHS)
return sliceArrayOperator(d, context, expressionNode.RHS.RHS)
}
lhs, err := d.GetMatchingNodes(context, expressionNode.LHS)
@ -149,13 +125,7 @@ func traverseNodesWithArrayIndices(context Context, indicesToTraverse []*Candida
return context.ChildContext(matchingNodeMap), nil
}
func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesToTraverse []*CandidateNode, prefs traversePreferences) (*list.List, error) {
var err error
matchingNode, err = resolveAliasChain(matchingNode)
if err != nil {
return nil, err
}
func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesToTraverse []*CandidateNode, prefs traversePreferences) (*list.List, error) { // call this if doc / alias like the other traverse
if matchingNode.Tag == "!!null" {
log.Debugf("OperatorArrayTraverse got a null - turning it into an empty array")
// auto vivification
@ -168,6 +138,9 @@ func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesT
}
switch matchingNode.Kind {
case AliasNode:
matchingNode = matchingNode.Alias
return traverseArrayIndices(context, matchingNode, indicesToTraverse, prefs)
case SequenceNode:
return traverseArrayWithIndices(matchingNode, indicesToTraverse, prefs)
case MappingNode:

View File

@ -665,16 +665,6 @@ var traversePathOperatorScenarios = []expressionScenario{
"D0, P[a], (!!null)::null\n",
},
},
{
// Regression test for https://issues.oss-fuzz.com/issues/390467412
// go-yaml accepts cross-document alias references (invalid per
// YAML spec). A nested assignment on such an alias can create a
// circular alias node, which must not cause a stack overflow.
skipDoc: true,
document: "&-- a\n---\n*--",
expression: ". = (.x = 1)",
expectedError: "alias cycle detected",
},
}
func TestTraversePathOperatorScenarios(t *testing.T) {
@ -692,58 +682,3 @@ func TestTraversePathOperatorAlignedToSpecScenarios(t *testing.T) {
appendOperatorDocumentScenario(t, "traverse-read", fixedTraversePathOperatorScenarios)
ConfiguredYamlPreferences.FixMergeAnchorToSpec = false
}
// Regression test for https://issues.oss-fuzz.com/issues/390467412
// A circular alias (alias pointing back to itself) must not cause a
// stack overflow. resolveAliasChain should detect the cycle and return
// an error; both traverse() and traverseArrayIndices() use it.
func TestTraverseAliasCycle(t *testing.T) {
aliasNode := &CandidateNode{
Kind: AliasNode,
}
aliasNode.Alias = aliasNode // A -> A
op := &Operation{
OperationType: traversePathOpType,
Value: "key",
StringValue: "key",
Preferences: traversePreferences{},
}
_, err := traverse(Context{}, aliasNode, op)
if err == nil {
t.Fatal("expected error for alias cycle, got nil")
}
if err.Error() != "alias cycle detected" {
t.Fatalf("expected 'alias cycle detected', got %q", err.Error())
}
// Same cycle must be caught through the array traversal path.
_, err = traverseArrayIndices(Context{}, aliasNode, nil, traversePreferences{})
if err == nil {
t.Fatal("expected error for alias cycle via traverseArrayIndices, got nil")
}
if err.Error() != "alias cycle detected" {
t.Fatalf("expected 'alias cycle detected', got %q", err.Error())
}
}
func TestTraverseAliasCycleChain(t *testing.T) {
nodeA := &CandidateNode{Kind: AliasNode}
nodeB := &CandidateNode{Kind: AliasNode}
nodeA.Alias = nodeB
nodeB.Alias = nodeA // A -> B -> A
op := &Operation{
OperationType: traversePathOpType,
Value: "key",
StringValue: "key",
Preferences: traversePreferences{},
}
_, err := traverse(Context{}, nodeA, op)
if err == nil {
t.Fatal("expected error for alias cycle chain, got nil")
}
if err.Error() != "alias cycle detected" {
t.Fatalf("expected 'alias cycle detected', got %q", err.Error())
}
}

View File

@ -31,7 +31,6 @@ type expressionScenario struct {
dontFormatInputForDoc bool // dont format input doc for documentation generation
requiresFormat string
skipForGoccy bool
yqFlags string // extra yq flags to include in generated doc command snippets
}
var goccyTesting = false
@ -357,22 +356,14 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
writeOrPanic(w, "then\n")
flagsPrefix := ""
if s.yqFlags != "" {
flagsPrefix = s.yqFlags + " "
}
if s.expression != "" {
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v'%v' %v\n```\n", envCommand, flagsPrefix, command, strings.ReplaceAll(s.expression, "'", `'\''`), files))
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v'%v' %v\n```\n", envCommand, command, strings.ReplaceAll(s.expression, "'", `'\''`), files))
} else {
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v%v\n```\n", envCommand, flagsPrefix, command, files))
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v\n```\n", envCommand, command, files))
}
} else {
writeOrPanic(w, "Running\n")
flagsPrefix := ""
if s.yqFlags != "" {
flagsPrefix = s.yqFlags + " "
}
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v--null-input '%v'\n```\n", envCommand, flagsPrefix, command, s.expression))
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v--null-input '%v'\n```\n", envCommand, command, s.expression))
}
return formattedDoc, formattedDoc2
}

View File

@ -1,13 +1,11 @@
package yqlib
type SecurityPreferences struct {
DisableEnvOps bool
DisableFileOps bool
EnableSystemOps bool
DisableEnvOps bool
DisableFileOps bool
}
var ConfiguredSecurityPreferences = SecurityPreferences{
DisableEnvOps: false,
DisableFileOps: false,
EnableSystemOps: false,
DisableEnvOps: false,
DisableFileOps: false,
}

View File

@ -287,18 +287,6 @@ var expectedSubArrays = `array:
- {}
`
// Keys with special characters that require quoting in TOML
var rtSpecialKeyInlineTable = `host = { "http://sealos.hub:5000" = { capabilities = ["pull", "resolve", "push"], skip_verify = true } }
`
var rtSpecialKeyTableSection = `["/tmp/blah"]
value = "hello"
`
var rtSpecialKeyDottedTableSection = `[servers."http://localhost:8080"]
ip = "127.0.0.1"
`
var tomlScenarios = []formatScenario{
{
skipDoc: true,
@ -626,30 +614,6 @@ var tomlScenarios = []formatScenario{
expected: tomlTableWithComments,
scenarioType: "roundtrip",
},
{
skipDoc: true,
description: "Roundtrip: key with special characters in inline table",
input: rtSpecialKeyInlineTable,
expression: ".",
expected: rtSpecialKeyInlineTable,
scenarioType: "roundtrip",
},
{
skipDoc: true,
description: "Roundtrip: key with special characters in table section",
input: rtSpecialKeyTableSection,
expression: ".",
expected: rtSpecialKeyTableSection,
scenarioType: "roundtrip",
},
{
skipDoc: true,
description: "Roundtrip: special character key in dotted table section header",
input: rtSpecialKeyDottedTableSection,
expression: ".",
expected: rtSpecialKeyDottedTableSection,
scenarioType: "roundtrip",
},
}
func testTomlScenario(t *testing.T, s formatScenario) {

View File

@ -298,9 +298,4 @@ subsubarray
Ffile
Fquery
coverpkg
gsub
ralia
Austr
ustrali
héllo
alia
gsub