mirror of
https://github.com/mikefarah/yq.git
synced 2026-07-05 12:10:37 +00:00
Compare commits
1 Commits
9f3a12f512
...
1ad3677589
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1ad3677589 |
2
.github/workflows/release.yml
vendored
2
.github/workflows/release.yml
vendored
@ -44,7 +44,7 @@ jobs:
|
||||
./scripts/xcompile.sh
|
||||
|
||||
- name: Release
|
||||
uses: softprops/action-gh-release@v2
|
||||
uses: softprops/action-gh-release@v1
|
||||
with:
|
||||
files: build/*
|
||||
draft: true
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
FROM golang:1.26.2 AS builder
|
||||
FROM golang:1.26.1 AS builder
|
||||
|
||||
WORKDIR /go/src/mikefarah/yq
|
||||
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
FROM golang:1.26.2
|
||||
FROM golang:1.26.1
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y npm && \
|
||||
|
||||
@ -212,7 +212,6 @@ yq -P -oy sample.json
|
||||
|
||||
rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableEnvOps, "security-disable-env-ops", "", false, "Disable env related operations.")
|
||||
rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.DisableFileOps, "security-disable-file-ops", "", false, "Disable file related operations (e.g. load)")
|
||||
rootCmd.PersistentFlags().BoolVarP(&yqlib.ConfiguredSecurityPreferences.EnableSystemOps, "security-enable-system-operator", "", false, "Enable system operator to allow execution of external commands.")
|
||||
|
||||
rootCmd.AddCommand(
|
||||
createEvaluateSequenceCommand(),
|
||||
|
||||
4
go.mod
4
go.mod
@ -17,9 +17,9 @@ require (
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
|
||||
github.com/spf13/cobra v1.10.2
|
||||
github.com/spf13/pflag v1.0.10
|
||||
github.com/yuin/gopher-lua v1.1.2
|
||||
github.com/yuin/gopher-lua v1.1.1
|
||||
github.com/zclconf/go-cty v1.18.0
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.4
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.3
|
||||
golang.org/x/mod v0.34.0
|
||||
golang.org/x/net v0.52.0
|
||||
golang.org/x/text v0.35.0
|
||||
|
||||
8
go.sum
8
go.sum
@ -61,15 +61,15 @@ github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3A
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/yuin/gopher-lua v1.1.2 h1:yF/FjE3hD65tBbt0VXLE13HWS9h34fdzJmrWRXwobGA=
|
||||
github.com/yuin/gopher-lua v1.1.2/go.mod h1:7aRmXIWl37SqRf0koeyylBEzJ+aPt8A+mmkQ4f1ntR8=
|
||||
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
|
||||
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
|
||||
github.com/zclconf/go-cty v1.18.0 h1:pJ8+HNI4gFoyRNqVE37wWbJWVw43BZczFo7KUoRczaA=
|
||||
github.com/zclconf/go-cty v1.18.0/go.mod h1:qpnV6EDNgC1sns/AleL1fvatHw72j+S+nS+MJ+T2CSg=
|
||||
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo=
|
||||
github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.4 h1:UP4+v6fFrBIb1l934bDl//mmnoIZEDK0idg1+AIvX5U=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.4/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.3 h1:3h1fjsh1CTAPjW7q/EMe+C8shx5d8ctzZTrLcs/j8Go=
|
||||
go.yaml.in/yaml/v4 v4.0.0-rc.3/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
|
||||
golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
|
||||
golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
|
||||
golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
# Slice Array or String
|
||||
# Slice/Splice Array
|
||||
|
||||
The slice operator works on both arrays and strings. Like the `jq` equivalent, `.[10:15]` will return a subarray (or substring) of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array or string.
|
||||
The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.
|
||||
|
||||
You may leave out the first or second number, which will refer to the start or end of the array or string respectively.
|
||||
You may leave out the first or second number, which will refer to the start or end of the array respectively.
|
||||
|
||||
@ -1,27 +0,0 @@
|
||||
# System Operators
|
||||
|
||||
The `system` operator allows you to run an external command and use its output as a value in your expression.
|
||||
|
||||
**Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it.
|
||||
|
||||
**Note:** When enabled, the system operator can replicate the functionality of `env` and `load`
|
||||
operators via external commands. Enabling it effectively overrides `--security-disable-env-ops`
|
||||
and `--security-disable-file-ops`.
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
yq --security-enable-system-operator --null-input '.field = system("command"; "arg1")'
|
||||
```
|
||||
|
||||
The operator takes:
|
||||
- A command string (required)
|
||||
- An argument (or an array of arguments), separated from the command by `;` (optional)
|
||||
|
||||
The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string.
|
||||
|
||||
## Disabling the system operator
|
||||
|
||||
The system operator is disabled by default. When disabled, an error is returned instead of running the command, consistent with `--security-disable-env-ops` and `--security-disable-file-ops`.
|
||||
|
||||
Use `--security-enable-system-operator` flag to enable it.
|
||||
@ -1,8 +1,8 @@
|
||||
# Slice Array or String
|
||||
# Slice/Splice Array
|
||||
|
||||
The slice operator works on both arrays and strings. Like the `jq` equivalent, `.[10:15]` will return a subarray (or substring) of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array or string.
|
||||
The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.
|
||||
|
||||
You may leave out the first or second number, which will refer to the start or end of the array or string respectively.
|
||||
You may leave out the first or second number, which will refer to the start or end of the array respectively.
|
||||
|
||||
## Slicing arrays
|
||||
Given a sample.yml file of:
|
||||
@ -103,81 +103,3 @@ will output
|
||||
- cow
|
||||
```
|
||||
|
||||
## Slicing strings
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
country: Australia
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.country[0:5]' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
Austr
|
||||
```
|
||||
|
||||
## Slicing strings - without the second number
|
||||
Finishes at the end of the string
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
country: Australia
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.country[5:]' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
alia
|
||||
```
|
||||
|
||||
## Slicing strings - without the first number
|
||||
Starts from the start of the string
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
country: Australia
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.country[:5]' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
Austr
|
||||
```
|
||||
|
||||
## Slicing strings - use negative numbers to count backwards from the end
|
||||
Negative indices count from the end of the string
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
country: Australia
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.country[-5:]' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
ralia
|
||||
```
|
||||
|
||||
## Slicing strings - Unicode
|
||||
Indices are rune-based, so multi-byte characters are handled correctly
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
greeting: héllo
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.greeting[1:3]' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
él
|
||||
```
|
||||
|
||||
|
||||
@ -1,76 +0,0 @@
|
||||
# System Operators
|
||||
|
||||
The `system` operator allows you to run an external command and use its output as a value in your expression.
|
||||
|
||||
**Security warning**: The system operator is disabled by default. You must explicitly pass `--security-enable-system-operator` to use it.
|
||||
|
||||
**Note:** When enabled, the system operator can replicate the functionality of `env` and `load`
|
||||
operators via external commands. Enabling it effectively overrides `--security-disable-env-ops`
|
||||
and `--security-disable-file-ops`.
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
yq --security-enable-system-operator --null-input '.field = system("command"; "arg1")'
|
||||
```
|
||||
|
||||
The operator takes:
|
||||
- A command string (required)
|
||||
- An argument (or an array of arguments), separated from the command by `;` (optional)
|
||||
|
||||
The current matched node's value is serialised and piped to the command via stdin. The command's stdout (with trailing newline stripped) is returned as a string.
|
||||
|
||||
## Disabling the system operator
|
||||
|
||||
The system operator is disabled by default. When disabled, an error is returned instead of running the command, consistent with `--security-disable-env-ops` and `--security-disable-file-ops`.
|
||||
|
||||
Use `--security-enable-system-operator` flag to enable it.
|
||||
|
||||
## system operator returns error when disabled
|
||||
Use `--security-enable-system-operator` to enable the system operator.
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
country: Australia
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq '.country = system("/usr/bin/echo"; "test")' sample.yml
|
||||
```
|
||||
will output
|
||||
```bash
|
||||
Error: system operations are disabled, use --security-enable-system-operator to enable
|
||||
```
|
||||
|
||||
## Run a command with an argument
|
||||
Use `--security-enable-system-operator` to enable the system operator.
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
country: Australia
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq --security-enable-system-operator '.country = system("/usr/bin/echo"; "test")' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
country: test
|
||||
```
|
||||
|
||||
## Run a command without arguments
|
||||
Omit the semicolon and args to run the command with no extra arguments.
|
||||
|
||||
Given a sample.yml file of:
|
||||
```yaml
|
||||
a: hello
|
||||
```
|
||||
then
|
||||
```bash
|
||||
yq --security-enable-system-operator '.a = system("/usr/bin/echo")' sample.yml
|
||||
```
|
||||
will output
|
||||
```yaml
|
||||
a: ""
|
||||
```
|
||||
|
||||
@ -69,27 +69,6 @@ func (te *tomlEncoder) CanHandleAliases() bool {
|
||||
|
||||
// ---- helpers ----
|
||||
|
||||
// tomlKey returns the key quoted if it contains characters that are not valid
|
||||
// in a TOML bare key. TOML bare keys may only contain ASCII letters, ASCII
|
||||
// digits, underscores, and dashes.
|
||||
func tomlKey(key string) string {
|
||||
for _, r := range key {
|
||||
if (r < 'A' || r > 'Z') && (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '_' && r != '-' {
|
||||
return fmt.Sprintf("%q", key)
|
||||
}
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
// tomlDottedKey joins path components, quoting any that require it.
|
||||
func tomlDottedKey(path []string) string {
|
||||
parts := make([]string, len(path))
|
||||
for i, p := range path {
|
||||
parts[i] = tomlKey(p)
|
||||
}
|
||||
return strings.Join(parts, ".")
|
||||
}
|
||||
|
||||
func (te *tomlEncoder) writeComment(w io.Writer, comment string) error {
|
||||
if comment == "" {
|
||||
return nil
|
||||
@ -169,10 +148,9 @@ func (te *tomlEncoder) encodeTopLevelEntry(w io.Writer, path []string, node *Can
|
||||
}
|
||||
if allMaps {
|
||||
key := path[len(path)-1]
|
||||
quotedKey := tomlKey(key)
|
||||
for _, it := range node.Content {
|
||||
// [[key]] then body
|
||||
if _, err := w.Write([]byte("[[" + quotedKey + "]]\n")); err != nil {
|
||||
if _, err := w.Write([]byte("[[" + key + "]]\n")); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := te.encodeMappingBodyWithPath(w, []string{key}, it); err != nil {
|
||||
@ -207,7 +185,7 @@ func (te *tomlEncoder) writeAttribute(w io.Writer, key string, value *CandidateN
|
||||
}
|
||||
|
||||
// Write the attribute
|
||||
line := tomlKey(key) + " = " + te.formatScalar(value)
|
||||
line := key + " = " + te.formatScalar(value)
|
||||
|
||||
// Add line comment if present
|
||||
if value.LineComment != "" {
|
||||
@ -232,7 +210,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida
|
||||
|
||||
// Handle empty arrays
|
||||
if len(seq.Content) == 0 {
|
||||
line := tomlKey(key) + " = []"
|
||||
line := key + " = []"
|
||||
if seq.LineComment != "" {
|
||||
lineComment := strings.TrimSpace(seq.LineComment)
|
||||
if !strings.HasPrefix(lineComment, "#") {
|
||||
@ -255,7 +233,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida
|
||||
|
||||
if hasElementComments {
|
||||
// Write multiline array format with comments
|
||||
if _, err := w.Write([]byte(tomlKey(key) + " = [\n")); err != nil {
|
||||
if _, err := w.Write([]byte(key + " = [\n")); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@ -346,7 +324,7 @@ func (te *tomlEncoder) writeArrayAttribute(w io.Writer, key string, seq *Candida
|
||||
}
|
||||
}
|
||||
|
||||
line := tomlKey(key) + " = [" + strings.Join(items, ", ") + "]"
|
||||
line := key + " = [" + strings.Join(items, ", ") + "]"
|
||||
|
||||
// Add line comment if present
|
||||
if seq.LineComment != "" {
|
||||
@ -394,21 +372,21 @@ func (te *tomlEncoder) mappingToInlineTable(m *CandidateNode) (string, error) {
|
||||
v := m.Content[i+1]
|
||||
switch v.Kind {
|
||||
case ScalarNode:
|
||||
parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), te.formatScalar(v)))
|
||||
parts = append(parts, fmt.Sprintf("%s = %s", k, te.formatScalar(v)))
|
||||
case SequenceNode:
|
||||
// inline array in inline table
|
||||
arr, err := te.sequenceToInlineArray(v)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), arr))
|
||||
parts = append(parts, fmt.Sprintf("%s = %s", k, arr))
|
||||
case MappingNode:
|
||||
// nested inline table
|
||||
inline, err := te.mappingToInlineTable(v)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
parts = append(parts, fmt.Sprintf("%s = %s", tomlKey(k), inline))
|
||||
parts = append(parts, fmt.Sprintf("%s = %s", k, inline))
|
||||
default:
|
||||
return "", fmt.Errorf("unsupported inline table value kind: %v", v.Kind)
|
||||
}
|
||||
@ -421,7 +399,7 @@ func (te *tomlEncoder) writeInlineTableAttribute(w io.Writer, key string, m *Can
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = w.Write([]byte(tomlKey(key) + " = " + inline + "\n"))
|
||||
_, err = w.Write([]byte(key + " = " + inline + "\n"))
|
||||
return err
|
||||
}
|
||||
|
||||
@ -443,7 +421,7 @@ func (te *tomlEncoder) writeTableHeader(w io.Writer, path []string, m *Candidate
|
||||
}
|
||||
|
||||
// Write table header [a.b.c]
|
||||
header := "[" + tomlDottedKey(path) + "]\n"
|
||||
header := "[" + strings.Join(path, ".") + "]\n"
|
||||
_, err := w.Write([]byte(header))
|
||||
return err
|
||||
}
|
||||
@ -510,7 +488,7 @@ func (te *tomlEncoder) encodeSeparateMapping(w io.Writer, path []string, m *Cand
|
||||
}
|
||||
}
|
||||
if allMaps {
|
||||
key := tomlDottedKey(append(append([]string{}, path...), k))
|
||||
key := strings.Join(append(append([]string{}, path...), k), ".")
|
||||
for _, it := range v.Content {
|
||||
if _, err := w.Write([]byte("[[" + key + "]]\n")); err != nil {
|
||||
return err
|
||||
@ -608,7 +586,7 @@ func (te *tomlEncoder) encodeMappingBodyWithPath(w io.Writer, path []string, m *
|
||||
}
|
||||
}
|
||||
if allMaps {
|
||||
dotted := tomlDottedKey(append(append([]string{}, path...), k))
|
||||
dotted := strings.Join(append(append([]string{}, path...), k), ".")
|
||||
for _, it := range v.Content {
|
||||
if _, err := w.Write([]byte("[[" + dotted + "]]\n")); err != nil {
|
||||
return err
|
||||
|
||||
@ -131,11 +131,6 @@ func handleToken(tokens []*token, index int, postProcessedTokens []*token) (toke
|
||||
log.Debugf("previous token is : traverseArrayOpType")
|
||||
// need to put the number 0 before this token, as that is implied
|
||||
postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: createValueOperation(0, "0")})
|
||||
} else if index >= 2 && tokens[index-1].TokenType == openCollect &&
|
||||
(tokens[index-2].TokenType == operationToken || tokens[index-2].TokenType == closeCollect || tokens[index-2].TokenType == closeCollectObject) {
|
||||
log.Debugf("previous token is : openCollect following a traversal, implying 0 start")
|
||||
// need to put the number 0 before this token, as that is implied
|
||||
postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: createValueOperation(0, "0")})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -96,8 +96,6 @@ var participleYqRules = []*participleYqRule{
|
||||
simpleOp("load_?str|str_?load", loadStringOpType),
|
||||
{"LoadYaml", `load`, loadOp(NewYamlDecoder(LoadYamlPreferences)), 0},
|
||||
|
||||
simpleOp("system", systemOpType),
|
||||
|
||||
{"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0},
|
||||
|
||||
simpleOp("select", selectOpType),
|
||||
|
||||
@ -80,7 +80,7 @@ func recurseNodeObjectEqual(lhs *CandidateNode, rhs *CandidateNode) bool {
|
||||
key := lhs.Content[index]
|
||||
value := lhs.Content[index+1]
|
||||
|
||||
indexInRHS := findKeyInMap(rhs, key)
|
||||
indexInRHS := findInArray(rhs, key)
|
||||
|
||||
if indexInRHS == -1 || !recursiveNodeEqual(value, rhs.Content[indexInRHS+1]) {
|
||||
return false
|
||||
|
||||
@ -24,7 +24,7 @@ type parseSnippetScenario struct {
|
||||
var parseSnippetScenarios = []parseSnippetScenario{
|
||||
{
|
||||
snippet: ":",
|
||||
expectedError: "yaml: while parsing a block mapping at <unknown position>: did not find expected key",
|
||||
expectedError: "yaml: did not find expected key",
|
||||
},
|
||||
{
|
||||
snippet: "",
|
||||
@ -300,24 +300,6 @@ func TestRecurseNodeObjectEqual(t *testing.T) {
|
||||
test.AssertResult(t, true, recurseNodeObjectEqual(obj1, obj2))
|
||||
test.AssertResult(t, false, recurseNodeObjectEqual(obj1, obj3))
|
||||
test.AssertResult(t, false, recurseNodeObjectEqual(obj1, obj4))
|
||||
|
||||
// A null key must not match a null value in the other map.
|
||||
// Regression test for https://issues.oss-fuzz.com/issues/383860504
|
||||
nullKey := &CandidateNode{Kind: ScalarNode, Tag: "!!null"}
|
||||
nullVal := &CandidateNode{Kind: ScalarNode, Tag: "!!null"}
|
||||
intKey := createScalarNode(2, "2")
|
||||
intKey.Tag = "!!int"
|
||||
intVal := &CandidateNode{Kind: ScalarNode, Tag: "!!null"}
|
||||
|
||||
mapWithNullKey := &CandidateNode{
|
||||
Kind: MappingNode,
|
||||
Content: []*CandidateNode{nullKey, nullVal},
|
||||
}
|
||||
mapWithIntKey := &CandidateNode{
|
||||
Kind: MappingNode,
|
||||
Content: []*CandidateNode{intKey, intVal},
|
||||
}
|
||||
test.AssertResult(t, false, recurseNodeObjectEqual(mapWithNullKey, mapWithIntKey))
|
||||
}
|
||||
|
||||
func TestParseInt(t *testing.T) {
|
||||
|
||||
@ -164,8 +164,6 @@ var stringInterpolationOpType = &operationType{Type: "STRING_INT", NumArgs: 0, P
|
||||
var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 52, Handler: loadOperator, CheckForPostTraverse: true}
|
||||
var loadStringOpType = &operationType{Type: "LOAD_STRING", NumArgs: 1, Precedence: 52, Handler: loadStringOperator}
|
||||
|
||||
var systemOpType = &operationType{Type: "SYSTEM", NumArgs: 1, Precedence: 50, Handler: systemOperator}
|
||||
|
||||
var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 52, Handler: keysOperator, CheckForPostTraverse: true}
|
||||
|
||||
var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator}
|
||||
|
||||
@ -527,18 +527,6 @@ var addOperatorScenarios = []expressionScenario{
|
||||
expression: `.a += [2]`,
|
||||
expectedError: "!!seq () cannot be added to a !!str (a)",
|
||||
},
|
||||
{
|
||||
// Regression test for https://issues.oss-fuzz.com/issues/383860504
|
||||
// Adding a map to itself must not panic when sequence keys contain
|
||||
// single-entry mappings with a null key in one and a non-null key
|
||||
// in the other.
|
||||
skipDoc: true,
|
||||
document: "? [{~: ~}]\n: v1\n? [{2: ~}]\n: v2",
|
||||
expression: `. += .`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::? [{~: ~}]\n: v1\n? [{2: ~}]\n: v2\n",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func TestAddOperatorScenarios(t *testing.T) {
|
||||
|
||||
@ -46,9 +46,9 @@ func containsObject(lhs *CandidateNode, rhs *CandidateNode) (bool, error) {
|
||||
rhsKey := rhs.Content[index]
|
||||
rhsValue := rhs.Content[index+1]
|
||||
log.Debugf("Looking for %v in the lhs", rhsKey.Value)
|
||||
lhsKeyIndex := findKeyInMap(lhs, rhsKey)
|
||||
lhsKeyIndex := findInArray(lhs, rhsKey)
|
||||
log.Debugf("index is %v", lhsKeyIndex)
|
||||
if lhsKeyIndex < 0 {
|
||||
if lhsKeyIndex < 0 || lhsKeyIndex%2 != 0 {
|
||||
return false, nil
|
||||
}
|
||||
lhsValue := lhs.Content[lhsKeyIndex+1]
|
||||
|
||||
@ -65,16 +65,6 @@ var containsOperatorScenarios = []expressionScenario{
|
||||
"D0, P[], (!!bool)::false\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
// Regression: findInArray could match a null key against a null
|
||||
// value at an earlier odd index, producing a false negative.
|
||||
skipDoc: true,
|
||||
document: "? 1\n: ~\n? ~\n: x",
|
||||
expression: `contains({~: "x"})`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!bool)::true\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "String contains substring",
|
||||
document: `"foobar"`,
|
||||
|
||||
@ -155,10 +155,8 @@ func repeatString(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error
|
||||
return nil, err
|
||||
} else if count < 0 {
|
||||
return nil, fmt.Errorf("cannot repeat string by a negative number (%v)", count)
|
||||
}
|
||||
maxResultLen := 10 * 1024 * 1024 // 10 MiB
|
||||
if count > 0 && len(stringNode.Value) > maxResultLen/count {
|
||||
return nil, fmt.Errorf("result of repeating string (%v bytes) by %v would exceed %v bytes", len(stringNode.Value), count, maxResultLen)
|
||||
} else if count > 10000000 {
|
||||
return nil, fmt.Errorf("cannot repeat string by more than 100 million (%v)", count)
|
||||
}
|
||||
target.Value = strings.Repeat(stringNode.Value, count)
|
||||
|
||||
|
||||
@ -237,11 +237,12 @@ var multiplyOperatorScenarios = []expressionScenario{
|
||||
expectedError: "cannot repeat string by a negative number (-4)",
|
||||
},
|
||||
{
|
||||
description: "Multiply string by count that exceeds result size limit",
|
||||
description: "Multiply string X by more than 100 million",
|
||||
// very large string.repeats causes a panic
|
||||
skipDoc: true,
|
||||
document: `n: 100000001`,
|
||||
expression: `"banana" * .n`,
|
||||
expectedError: "result of repeating string (6 bytes) by 100000001 would exceed 10485760 bytes",
|
||||
expectedError: "cannot repeat string by more than 100 million (100000001)",
|
||||
},
|
||||
{
|
||||
description: "Multiply int node X string",
|
||||
@ -553,7 +554,7 @@ var multiplyOperatorScenarios = []expressionScenario{
|
||||
document: document,
|
||||
expression: `.b * .c`,
|
||||
expected: []string{
|
||||
"D0, P[b], (!!map)::{name: dog, \"<<\": *cat}\n",
|
||||
"D0, P[b], (!!map)::{name: dog, <<: *cat}\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
@ -692,27 +693,6 @@ var multiplyOperatorScenarios = []expressionScenario{
|
||||
"D0, P[], (!!null)::null\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
// Regression test for https://issues.oss-fuzz.com/issues/418818862
|
||||
// Large repeat count with a long string must not panic.
|
||||
skipDoc: true,
|
||||
expression: `"abc" * 99999999`,
|
||||
expectedError: "result of repeating string (3 bytes) by 99999999 would exceed 10485760 bytes",
|
||||
},
|
||||
{
|
||||
// Regression test for https://issues.oss-fuzz.com/issues/383195001
|
||||
// Product of string length * repeat count must be bounded.
|
||||
skipDoc: true,
|
||||
expression: `"x" * 99999999`,
|
||||
expectedError: "result of repeating string (1 bytes) by 99999999 would exceed 10485760 bytes",
|
||||
},
|
||||
{
|
||||
// The size guard must not overflow: len * count can wrap to
|
||||
// a negative or small value on 64-bit, bypassing the check.
|
||||
skipDoc: true,
|
||||
expression: `"ab" * 4611686018427387904`,
|
||||
expectedError: "result of repeating string (2 bytes) by 4611686018427387904 would exceed 10485760 bytes",
|
||||
},
|
||||
}
|
||||
|
||||
func TestMultiplyOperatorScenarios(t *testing.T) {
|
||||
|
||||
@ -16,39 +16,6 @@ func getSliceNumber(d *dataTreeNavigator, context Context, node *CandidateNode,
|
||||
return parseInt(result.MatchingNodes.Front().Value.(*CandidateNode).Value)
|
||||
}
|
||||
|
||||
// clampSliceIndex resolves a possibly-negative slice index against
|
||||
// length and clamps the result to [0, length].
|
||||
func clampSliceIndex(index, length int) int {
|
||||
if index < 0 {
|
||||
index += length
|
||||
}
|
||||
if index < 0 {
|
||||
return 0
|
||||
}
|
||||
if index > length {
|
||||
return length
|
||||
}
|
||||
return index
|
||||
}
|
||||
|
||||
func sliceStringNode(lhsNode *CandidateNode, firstNumber int, secondNumber int) *CandidateNode {
|
||||
runes := []rune(lhsNode.Value)
|
||||
length := len(runes)
|
||||
|
||||
relativeFirstNumber := clampSliceIndex(firstNumber, length)
|
||||
relativeSecondNumber := clampSliceIndex(secondNumber, length)
|
||||
if relativeSecondNumber < relativeFirstNumber {
|
||||
relativeSecondNumber = relativeFirstNumber
|
||||
}
|
||||
|
||||
log.Debugf("sliceStringNode: slice from %v to %v", relativeFirstNumber, relativeSecondNumber)
|
||||
|
||||
slicedString := string(runes[relativeFirstNumber:relativeSecondNumber])
|
||||
replacement := lhsNode.CreateReplacement(ScalarNode, lhsNode.Tag, slicedString)
|
||||
replacement.Style = lhsNode.Style
|
||||
return replacement
|
||||
}
|
||||
|
||||
func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
|
||||
|
||||
log.Debug("slice array operator!")
|
||||
@ -61,23 +28,27 @@ func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *E
|
||||
lhsNode := el.Value.(*CandidateNode)
|
||||
|
||||
firstNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.LHS)
|
||||
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
relativeFirstNumber := firstNumber
|
||||
if relativeFirstNumber < 0 {
|
||||
relativeFirstNumber = len(lhsNode.Content) + firstNumber
|
||||
}
|
||||
|
||||
secondNumber, err := getSliceNumber(d, context, lhsNode, expressionNode.RHS)
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
|
||||
if lhsNode.Kind == ScalarNode && lhsNode.guessTagFromCustomType() == "!!str" {
|
||||
results.PushBack(sliceStringNode(lhsNode, firstNumber, secondNumber))
|
||||
continue
|
||||
relativeSecondNumber := secondNumber
|
||||
if relativeSecondNumber < 0 {
|
||||
relativeSecondNumber = len(lhsNode.Content) + secondNumber
|
||||
} else if relativeSecondNumber > len(lhsNode.Content) {
|
||||
relativeSecondNumber = len(lhsNode.Content)
|
||||
}
|
||||
|
||||
relativeFirstNumber := clampSliceIndex(firstNumber, len(lhsNode.Content))
|
||||
relativeSecondNumber := clampSliceIndex(secondNumber, len(lhsNode.Content))
|
||||
|
||||
log.Debugf("calculateIndicesToTraverse: slice from %v to %v", relativeFirstNumber, relativeSecondNumber)
|
||||
|
||||
var newResults []*CandidateNode
|
||||
|
||||
@ -98,115 +98,6 @@ var sliceArrayScenarios = []expressionScenario{
|
||||
"D0, P[], (!!seq)::- cat1\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
// Regression test for https://issues.oss-fuzz.com/issues/438776028
|
||||
// Negative second index that underflows after adjustment must
|
||||
// clamp to zero, yielding an empty sequence.
|
||||
skipDoc: true,
|
||||
document: `[a, b, c]`,
|
||||
expression: `.[0:-99999]`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!seq)::[]\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
// First-index underflow: without clamping, the loop starts at a
|
||||
// negative index and panics on Content access.
|
||||
skipDoc: true,
|
||||
document: `[a, b, c]`,
|
||||
expression: `.[-99999:3]`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!seq)::- a\n- b\n- c\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
// Both indices underflow: both clamp to zero, yielding an empty
|
||||
// sequence.
|
||||
skipDoc: true,
|
||||
document: `[a, b, c]`,
|
||||
expression: `.[-99999:-99998]`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!seq)::[]\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Slicing strings",
|
||||
document: `country: Australia`,
|
||||
expression: `.country[0:5]`,
|
||||
expected: []string{
|
||||
"D0, P[country], (!!str)::Austr\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Slicing strings - without the second number",
|
||||
subdescription: "Finishes at the end of the string",
|
||||
document: `country: Australia`,
|
||||
expression: `.country[5:]`,
|
||||
expected: []string{
|
||||
"D0, P[country], (!!str)::alia\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Slicing strings - without the first number",
|
||||
subdescription: "Starts from the start of the string",
|
||||
document: `country: Australia`,
|
||||
expression: `.country[:5]`,
|
||||
expected: []string{
|
||||
"D0, P[country], (!!str)::Austr\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Slicing strings - use negative numbers to count backwards from the end",
|
||||
subdescription: "Negative indices count from the end of the string",
|
||||
document: `country: Australia`,
|
||||
expression: `.country[-5:]`,
|
||||
expected: []string{
|
||||
"D0, P[country], (!!str)::ralia\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
document: `country: Australia`,
|
||||
expression: `.country[1:-1]`,
|
||||
expected: []string{
|
||||
"D0, P[country], (!!str)::ustrali\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
document: `country: Australia`,
|
||||
expression: `.country[:]`,
|
||||
expected: []string{
|
||||
"D0, P[country], (!!str)::Australia\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "second index beyond string length clamps",
|
||||
document: `country: Australia`,
|
||||
expression: `.country[:100]`,
|
||||
expected: []string{
|
||||
"D0, P[country], (!!str)::Australia\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "first index beyond string length returns empty string",
|
||||
document: `country: Australia`,
|
||||
expression: `.country[100:]`,
|
||||
expected: []string{
|
||||
"D0, P[country], (!!str)::\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Slicing strings - Unicode",
|
||||
subdescription: "Indices are rune-based, so multi-byte characters are handled correctly",
|
||||
document: `greeting: héllo`,
|
||||
expression: `.greeting[1:3]`,
|
||||
expected: []string{
|
||||
"D0, P[greeting], (!!str)::él\n",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func TestSliceOperatorScenarios(t *testing.T) {
|
||||
|
||||
@ -1,146 +0,0 @@
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"container/list"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func resolveSystemArgs(argsNode *CandidateNode) ([]string, error) {
|
||||
if argsNode == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if argsNode.Kind == SequenceNode {
|
||||
args := make([]string, 0, len(argsNode.Content))
|
||||
for _, child := range argsNode.Content {
|
||||
// Only non-null scalar children are valid arguments.
|
||||
if child == nil {
|
||||
continue
|
||||
}
|
||||
if child.Kind != ScalarNode || child.Tag == "!!null" {
|
||||
return nil, fmt.Errorf("system operator: argument must be a non-null scalar; got kind=%v tag=%v", child.Kind, child.Tag)
|
||||
}
|
||||
args = append(args, child.Value)
|
||||
}
|
||||
if len(args) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return args, nil
|
||||
}
|
||||
|
||||
// Single-argument case: only accept a non-null scalar node.
|
||||
if argsNode.Tag == "!!null" {
|
||||
return nil, nil
|
||||
}
|
||||
if argsNode.Kind != ScalarNode {
|
||||
return nil, fmt.Errorf("system operator: args must be a non-null scalar or sequence of non-null scalars; got kind=%v tag=%v", argsNode.Kind, argsNode.Tag)
|
||||
}
|
||||
return []string{argsNode.Value}, nil
|
||||
}
|
||||
|
||||
func resolveCommandNode(commandNodes Context) (string, error) {
|
||||
if commandNodes.MatchingNodes.Front() == nil {
|
||||
return "", fmt.Errorf("system operator: command expression returned no results")
|
||||
}
|
||||
if commandNodes.MatchingNodes.Len() > 1 {
|
||||
log.Debugf("system operator: command expression returned %d results, using first", commandNodes.MatchingNodes.Len())
|
||||
}
|
||||
cmdNode := commandNodes.MatchingNodes.Front().Value.(*CandidateNode)
|
||||
if cmdNode.Kind != ScalarNode || cmdNode.guessTagFromCustomType() != "!!str" {
|
||||
return "", fmt.Errorf("system operator: command must be a string scalar")
|
||||
}
|
||||
if cmdNode.Value == "" {
|
||||
return "", fmt.Errorf("system operator: command must be a non-empty string")
|
||||
}
|
||||
return cmdNode.Value, nil
|
||||
}
|
||||
|
||||
func systemOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
|
||||
if !ConfiguredSecurityPreferences.EnableSystemOps {
|
||||
return Context{}, fmt.Errorf("system operations are disabled, use --security-enable-system-operator to enable")
|
||||
}
|
||||
|
||||
// determine at parse time whether we have (command; args) or just (command)
|
||||
hasArgs := expressionNode.RHS.Operation.OperationType == blockOpType
|
||||
|
||||
var results = list.New()
|
||||
|
||||
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
|
||||
candidate := el.Value.(*CandidateNode)
|
||||
nodeContext := context.SingleReadonlyChildContext(candidate)
|
||||
|
||||
var command string
|
||||
var args []string
|
||||
|
||||
if hasArgs {
|
||||
block := expressionNode.RHS
|
||||
commandNodes, err := d.GetMatchingNodes(nodeContext, block.LHS)
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
command, err = resolveCommandNode(commandNodes)
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
|
||||
argsNodes, err := d.GetMatchingNodes(nodeContext, block.RHS)
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
if argsNodes.MatchingNodes.Len() > 1 {
|
||||
log.Debugf("system operator: args expression returned %d results, using first", argsNodes.MatchingNodes.Len())
|
||||
}
|
||||
if argsNodes.MatchingNodes.Front() != nil {
|
||||
args, err = resolveSystemArgs(argsNodes.MatchingNodes.Front().Value.(*CandidateNode))
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
commandNodes, err := d.GetMatchingNodes(nodeContext, expressionNode.RHS)
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
command, err = resolveCommandNode(commandNodes)
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
}
|
||||
|
||||
var stdin bytes.Buffer
|
||||
encoded, err := encodeToYamlString(candidate)
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
stdin.WriteString(encoded)
|
||||
|
||||
// #nosec G204 - intentional: user must explicitly enable this operator
|
||||
cmd := exec.Command(command, args...)
|
||||
cmd.Stdin = &stdin
|
||||
var stderr bytes.Buffer
|
||||
cmd.Stderr = &stderr
|
||||
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
stderrStr := strings.TrimSpace(stderr.String())
|
||||
if stderrStr != "" {
|
||||
return Context{}, fmt.Errorf("system command '%v' failed: %w\nstderr: %v", command, err, stderrStr)
|
||||
}
|
||||
return Context{}, fmt.Errorf("system command '%v' failed: %w", command, err)
|
||||
}
|
||||
|
||||
result := string(output)
|
||||
if strings.HasSuffix(result, "\r\n") {
|
||||
result = result[:len(result)-2]
|
||||
} else if strings.HasSuffix(result, "\n") {
|
||||
result = result[:len(result)-1]
|
||||
}
|
||||
newNode := candidate.CreateReplacement(ScalarNode, "!!str", result)
|
||||
results.PushBack(newNode)
|
||||
}
|
||||
|
||||
return context.ChildContext(results), nil
|
||||
}
|
||||
@ -1,123 +0,0 @@
|
||||
package yqlib
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func findExec(t *testing.T, name string) string {
|
||||
t.Helper()
|
||||
path, err := exec.LookPath(name)
|
||||
if err != nil {
|
||||
t.Skipf("skipping: %v not found: %v", name, err)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
||||
var systemOperatorDisabledScenarios = []expressionScenario{
|
||||
{
|
||||
description: "system operator returns error when disabled",
|
||||
subdescription: "Use `--security-enable-system-operator` to enable the system operator.",
|
||||
document: "country: Australia",
|
||||
expression: `.country = system("/usr/bin/echo"; "test")`,
|
||||
expectedError: "system operations are disabled, use --security-enable-system-operator to enable",
|
||||
},
|
||||
}
|
||||
|
||||
func TestSystemOperatorDisabledScenarios(t *testing.T) {
|
||||
originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps
|
||||
defer func() {
|
||||
ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps
|
||||
}()
|
||||
|
||||
ConfiguredSecurityPreferences.EnableSystemOps = false
|
||||
|
||||
for _, tt := range systemOperatorDisabledScenarios {
|
||||
testScenario(t, &tt)
|
||||
}
|
||||
documentOperatorScenarios(t, "system-operators", systemOperatorDisabledScenarios)
|
||||
}
|
||||
|
||||
func TestSystemOperatorEnabledScenarios(t *testing.T) {
|
||||
echoPath := findExec(t, "echo")
|
||||
falsePath := findExec(t, "false")
|
||||
|
||||
originalEnableSystemOps := ConfiguredSecurityPreferences.EnableSystemOps
|
||||
defer func() {
|
||||
ConfiguredSecurityPreferences.EnableSystemOps = originalEnableSystemOps
|
||||
}()
|
||||
|
||||
ConfiguredSecurityPreferences.EnableSystemOps = true
|
||||
|
||||
scenarios := []expressionScenario{
|
||||
{
|
||||
description: "Run a command with an argument",
|
||||
subdescription: "Use `--security-enable-system-operator` to enable the system operator.",
|
||||
yqFlags: "--security-enable-system-operator",
|
||||
document: "country: Australia",
|
||||
expression: `.country = system("` + echoPath + `"; "test")`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::country: test\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Run a command without arguments",
|
||||
subdescription: "Omit the semicolon and args to run the command with no extra arguments.",
|
||||
yqFlags: "--security-enable-system-operator",
|
||||
document: "a: hello",
|
||||
expression: `.a = system("` + echoPath + `")`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::a: \"\"\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Run a command with multiple arguments",
|
||||
subdescription: "Pass an array of arguments.",
|
||||
skipDoc: true,
|
||||
document: "a: hello",
|
||||
expression: `.a = system("` + echoPath + `"; ["foo", "bar"])`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::a: foo bar\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Command and args are evaluated per matched node",
|
||||
skipDoc: true,
|
||||
document: "cmd: " + echoPath + "\narg: hello",
|
||||
expression: `.result = system(.cmd; .arg)`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::cmd: " + echoPath + "\narg: hello\nresult: hello\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
description: "Command failure returns error",
|
||||
skipDoc: true,
|
||||
document: "a: hello",
|
||||
expression: `.a = system("` + falsePath + `")`,
|
||||
expectedError: "system command '" + falsePath + "' failed: exit status 1",
|
||||
},
|
||||
{
|
||||
description: "Null command returns error",
|
||||
skipDoc: true,
|
||||
document: "a: hello",
|
||||
expression: `.a = system(null)`,
|
||||
expectedError: "system operator: command must be a string scalar",
|
||||
},
|
||||
{
|
||||
description: "System operator processes multiple matched nodes",
|
||||
skipDoc: true,
|
||||
document: "a: first",
|
||||
document2: "a: second",
|
||||
expression: `.a = system("` + echoPath + `"; "replaced")`,
|
||||
expected: []string{
|
||||
"D0, P[], (!!map)::a: replaced\n",
|
||||
"D0, P[], (!!map)::a: replaced\n",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range scenarios {
|
||||
testScenario(t, &tt)
|
||||
}
|
||||
appendOperatorDocumentScenario(t, "system-operators", scenarios)
|
||||
}
|
||||
@ -36,33 +36,9 @@ func traversePathOperator(_ *dataTreeNavigator, context Context, expressionNode
|
||||
return context.ChildContext(matches), nil
|
||||
}
|
||||
|
||||
// resolveAliasChain follows an alias chain iteratively, returning the
|
||||
// first non-alias node. Returns an error if a cycle is detected.
|
||||
func resolveAliasChain(node *CandidateNode) (*CandidateNode, error) {
|
||||
if node.Kind != AliasNode {
|
||||
return node, nil
|
||||
}
|
||||
visited := map[*CandidateNode]bool{}
|
||||
for node.Kind == AliasNode {
|
||||
if visited[node] {
|
||||
return nil, fmt.Errorf("alias cycle detected")
|
||||
}
|
||||
visited[node] = true
|
||||
log.Debug("its an alias!")
|
||||
node = node.Alias
|
||||
}
|
||||
return node, nil
|
||||
}
|
||||
|
||||
func traverse(context Context, matchingNode *CandidateNode, operation *Operation) (*list.List, error) {
|
||||
log.Debugf("Traversing %v", NodeToString(matchingNode))
|
||||
|
||||
var err error
|
||||
matchingNode, err = resolveAliasChain(matchingNode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if matchingNode.Tag == "!!null" && operation.Value != "[]" && !context.DontAutoCreate {
|
||||
log.Debugf("Guessing kind")
|
||||
// we must have added this automatically, lets guess what it should be now
|
||||
@ -86,6 +62,10 @@ func traverse(context Context, matchingNode *CandidateNode, operation *Operation
|
||||
log.Debugf("its a sequence of %v things!", len(matchingNode.Content))
|
||||
return traverseArray(matchingNode, operation, operation.Preferences.(traversePreferences))
|
||||
|
||||
case AliasNode:
|
||||
log.Debug("its an alias!")
|
||||
matchingNode = matchingNode.Alias
|
||||
return traverse(context, matchingNode, operation)
|
||||
default:
|
||||
return list.New(), nil
|
||||
}
|
||||
@ -99,11 +79,7 @@ func traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode
|
||||
log.Debugf("--traverseArrayOperator")
|
||||
|
||||
if expressionNode.RHS != nil && expressionNode.RHS.RHS != nil && expressionNode.RHS.RHS.Operation.OperationType == createMapOpType {
|
||||
lhsContext, err := d.GetMatchingNodes(context, expressionNode.LHS)
|
||||
if err != nil {
|
||||
return Context{}, err
|
||||
}
|
||||
return sliceArrayOperator(d, lhsContext, expressionNode.RHS.RHS)
|
||||
return sliceArrayOperator(d, context, expressionNode.RHS.RHS)
|
||||
}
|
||||
|
||||
lhs, err := d.GetMatchingNodes(context, expressionNode.LHS)
|
||||
@ -149,13 +125,7 @@ func traverseNodesWithArrayIndices(context Context, indicesToTraverse []*Candida
|
||||
return context.ChildContext(matchingNodeMap), nil
|
||||
}
|
||||
|
||||
func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesToTraverse []*CandidateNode, prefs traversePreferences) (*list.List, error) {
|
||||
var err error
|
||||
matchingNode, err = resolveAliasChain(matchingNode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesToTraverse []*CandidateNode, prefs traversePreferences) (*list.List, error) { // call this if doc / alias like the other traverse
|
||||
if matchingNode.Tag == "!!null" {
|
||||
log.Debugf("OperatorArrayTraverse got a null - turning it into an empty array")
|
||||
// auto vivification
|
||||
@ -168,6 +138,9 @@ func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesT
|
||||
}
|
||||
|
||||
switch matchingNode.Kind {
|
||||
case AliasNode:
|
||||
matchingNode = matchingNode.Alias
|
||||
return traverseArrayIndices(context, matchingNode, indicesToTraverse, prefs)
|
||||
case SequenceNode:
|
||||
return traverseArrayWithIndices(matchingNode, indicesToTraverse, prefs)
|
||||
case MappingNode:
|
||||
|
||||
@ -665,16 +665,6 @@ var traversePathOperatorScenarios = []expressionScenario{
|
||||
"D0, P[a], (!!null)::null\n",
|
||||
},
|
||||
},
|
||||
{
|
||||
// Regression test for https://issues.oss-fuzz.com/issues/390467412
|
||||
// go-yaml accepts cross-document alias references (invalid per
|
||||
// YAML spec). A nested assignment on such an alias can create a
|
||||
// circular alias node, which must not cause a stack overflow.
|
||||
skipDoc: true,
|
||||
document: "&-- a\n---\n*--",
|
||||
expression: ". = (.x = 1)",
|
||||
expectedError: "alias cycle detected",
|
||||
},
|
||||
}
|
||||
|
||||
func TestTraversePathOperatorScenarios(t *testing.T) {
|
||||
@ -692,58 +682,3 @@ func TestTraversePathOperatorAlignedToSpecScenarios(t *testing.T) {
|
||||
appendOperatorDocumentScenario(t, "traverse-read", fixedTraversePathOperatorScenarios)
|
||||
ConfiguredYamlPreferences.FixMergeAnchorToSpec = false
|
||||
}
|
||||
|
||||
// Regression test for https://issues.oss-fuzz.com/issues/390467412
|
||||
// A circular alias (alias pointing back to itself) must not cause a
|
||||
// stack overflow. resolveAliasChain should detect the cycle and return
|
||||
// an error; both traverse() and traverseArrayIndices() use it.
|
||||
func TestTraverseAliasCycle(t *testing.T) {
|
||||
aliasNode := &CandidateNode{
|
||||
Kind: AliasNode,
|
||||
}
|
||||
aliasNode.Alias = aliasNode // A -> A
|
||||
|
||||
op := &Operation{
|
||||
OperationType: traversePathOpType,
|
||||
Value: "key",
|
||||
StringValue: "key",
|
||||
Preferences: traversePreferences{},
|
||||
}
|
||||
_, err := traverse(Context{}, aliasNode, op)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for alias cycle, got nil")
|
||||
}
|
||||
if err.Error() != "alias cycle detected" {
|
||||
t.Fatalf("expected 'alias cycle detected', got %q", err.Error())
|
||||
}
|
||||
|
||||
// Same cycle must be caught through the array traversal path.
|
||||
_, err = traverseArrayIndices(Context{}, aliasNode, nil, traversePreferences{})
|
||||
if err == nil {
|
||||
t.Fatal("expected error for alias cycle via traverseArrayIndices, got nil")
|
||||
}
|
||||
if err.Error() != "alias cycle detected" {
|
||||
t.Fatalf("expected 'alias cycle detected', got %q", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
func TestTraverseAliasCycleChain(t *testing.T) {
|
||||
nodeA := &CandidateNode{Kind: AliasNode}
|
||||
nodeB := &CandidateNode{Kind: AliasNode}
|
||||
nodeA.Alias = nodeB
|
||||
nodeB.Alias = nodeA // A -> B -> A
|
||||
|
||||
op := &Operation{
|
||||
OperationType: traversePathOpType,
|
||||
Value: "key",
|
||||
StringValue: "key",
|
||||
Preferences: traversePreferences{},
|
||||
}
|
||||
_, err := traverse(Context{}, nodeA, op)
|
||||
if err == nil {
|
||||
t.Fatal("expected error for alias cycle chain, got nil")
|
||||
}
|
||||
if err.Error() != "alias cycle detected" {
|
||||
t.Fatalf("expected 'alias cycle detected', got %q", err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
@ -31,7 +31,6 @@ type expressionScenario struct {
|
||||
dontFormatInputForDoc bool // dont format input doc for documentation generation
|
||||
requiresFormat string
|
||||
skipForGoccy bool
|
||||
yqFlags string // extra yq flags to include in generated doc command snippets
|
||||
}
|
||||
|
||||
var goccyTesting = false
|
||||
@ -357,22 +356,14 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
|
||||
|
||||
writeOrPanic(w, "then\n")
|
||||
|
||||
flagsPrefix := ""
|
||||
if s.yqFlags != "" {
|
||||
flagsPrefix = s.yqFlags + " "
|
||||
}
|
||||
if s.expression != "" {
|
||||
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v'%v' %v\n```\n", envCommand, flagsPrefix, command, strings.ReplaceAll(s.expression, "'", `'\''`), files))
|
||||
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v'%v' %v\n```\n", envCommand, command, strings.ReplaceAll(s.expression, "'", `'\''`), files))
|
||||
} else {
|
||||
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v%v\n```\n", envCommand, flagsPrefix, command, files))
|
||||
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v\n```\n", envCommand, command, files))
|
||||
}
|
||||
} else {
|
||||
writeOrPanic(w, "Running\n")
|
||||
flagsPrefix := ""
|
||||
if s.yqFlags != "" {
|
||||
flagsPrefix = s.yqFlags + " "
|
||||
}
|
||||
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v%v--null-input '%v'\n```\n", envCommand, flagsPrefix, command, s.expression))
|
||||
writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v--null-input '%v'\n```\n", envCommand, command, s.expression))
|
||||
}
|
||||
return formattedDoc, formattedDoc2
|
||||
}
|
||||
|
||||
@ -1,13 +1,11 @@
|
||||
package yqlib
|
||||
|
||||
type SecurityPreferences struct {
|
||||
DisableEnvOps bool
|
||||
DisableFileOps bool
|
||||
EnableSystemOps bool
|
||||
DisableEnvOps bool
|
||||
DisableFileOps bool
|
||||
}
|
||||
|
||||
var ConfiguredSecurityPreferences = SecurityPreferences{
|
||||
DisableEnvOps: false,
|
||||
DisableFileOps: false,
|
||||
EnableSystemOps: false,
|
||||
DisableEnvOps: false,
|
||||
DisableFileOps: false,
|
||||
}
|
||||
|
||||
@ -287,18 +287,6 @@ var expectedSubArrays = `array:
|
||||
- {}
|
||||
`
|
||||
|
||||
// Keys with special characters that require quoting in TOML
|
||||
var rtSpecialKeyInlineTable = `host = { "http://sealos.hub:5000" = { capabilities = ["pull", "resolve", "push"], skip_verify = true } }
|
||||
`
|
||||
|
||||
var rtSpecialKeyTableSection = `["/tmp/blah"]
|
||||
value = "hello"
|
||||
`
|
||||
|
||||
var rtSpecialKeyDottedTableSection = `[servers."http://localhost:8080"]
|
||||
ip = "127.0.0.1"
|
||||
`
|
||||
|
||||
var tomlScenarios = []formatScenario{
|
||||
{
|
||||
skipDoc: true,
|
||||
@ -626,30 +614,6 @@ var tomlScenarios = []formatScenario{
|
||||
expected: tomlTableWithComments,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "Roundtrip: key with special characters in inline table",
|
||||
input: rtSpecialKeyInlineTable,
|
||||
expression: ".",
|
||||
expected: rtSpecialKeyInlineTable,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "Roundtrip: key with special characters in table section",
|
||||
input: rtSpecialKeyTableSection,
|
||||
expression: ".",
|
||||
expected: rtSpecialKeyTableSection,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
{
|
||||
skipDoc: true,
|
||||
description: "Roundtrip: special character key in dotted table section header",
|
||||
input: rtSpecialKeyDottedTableSection,
|
||||
expression: ".",
|
||||
expected: rtSpecialKeyDottedTableSection,
|
||||
scenarioType: "roundtrip",
|
||||
},
|
||||
}
|
||||
|
||||
func testTomlScenario(t *testing.T, s formatScenario) {
|
||||
|
||||
@ -298,9 +298,4 @@ subsubarray
|
||||
Ffile
|
||||
Fquery
|
||||
coverpkg
|
||||
gsub
|
||||
ralia
|
||||
Austr
|
||||
ustrali
|
||||
héllo
|
||||
alia
|
||||
gsub
|
||||
Loading…
Reference in New Issue
Block a user