suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2025-01-12 19:25:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix collect

Browse Source
This commit is contained in:
Mike Farah 2021-11-30 13:04:32 +11:00
parent 176fec639d
commit 1085f11905
15 changed files with 413 additions and 311 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
examples/data1.yaml
View File

@ -1,2 +1 @@
a:
include: 'data2.yaml'
{a: [cat, dog, frog]}

9
pkg/yqlib/doc/entries.md
View File

@ -91,14 +91,21 @@ Given a sample.yml file of:
a: 1
b: 2
```
And another sample another.yml file of:
```yaml
c: 1
d: 2
```
then
```bash
yq eval 'with_entries(.key |= "KEY_" + .)' sample.yml
yq eval-all 'with_entries(.key |= "KEY_" + .)' sample.yml another.yml
```
will output
```yaml
KEY_a: 1
KEY_b: 2
KEY_c: 1
KEY_d: 2
```
## Use with_entries to filter the map

2
pkg/yqlib/doc/path.md
View File

@ -78,7 +78,7 @@ a:
```
then
```bash
yq eval '.a.[] | select(. == "*og") | [{"path":path, "value":.}]' sample.yml
yq eval '.a[] | select(. == "*og") | [{"path":path, "value":.}]' sample.yml
```
will output
```yaml

7
pkg/yqlib/expression_postfix.go
View File

@ -81,12 +81,15 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
}
result = append(result, &Operation{OperationType: collectOperator, Preferences: prefs})
log.Debugf("put collect onto the result")
result = append(result, &Operation{OperationType: shortPipeOpType})
log.Debugf("put shortpipe onto the result")
if opener != openCollect {
result = append(result, &Operation{OperationType: shortPipeOpType})
log.Debugf("put shortpipe onto the result")
}
//traverseArrayCollect is a sneaky op that needs to be included too
//when closing a []
if len(opStack) > 0 && opStack[len(opStack)-1].Operation != nil && opStack[len(opStack)-1].Operation.OperationType == traverseArrayOpType {
opStack, result = popOpToResult(opStack, result)
}

518
pkg/yqlib/expression_processing_test.go
View File

@ -7,279 +7,289 @@ import (
"github.com/mikefarah/yq/v4/test"
)
var variableWithNewLine = `"cat
"`
// var variableWithNewLine = `"cat
// "`
var pathTests = []struct {
path string
expectedTokens []interface{}
expectedPostFix []interface{}
}{
{
`[.a, .b]`,
append(make([]interface{}, 0), "[", "a", "UNION", "b", "]"),
append(make([]interface{}, 0), "a", "b", "UNION", "COLLECT"),
},
{
`.[env(myenv)]`,
append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "ENV", "]"),
append(make([]interface{}, 0), "SELF", "ENV", "COLLECT", "TRAVERSE_ARRAY"),
},
{
`.["cat"].["dog"]`,
append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "cat (string)", "]", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "dog (string)", "]"),
append(make([]interface{}, 0), "SELF", "cat (string)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SELF", "dog (string)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
append(make([]interface{}, 0), "SELF", "cat (string)", "COLLECT", "TRAVERSE_ARRAY", "SELF", "dog (string)", "COLLECT", "TRAVERSE_ARRAY", "SHORT_PIPE"),
},
{
`.["cat"]`,
append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "cat (string)", "]"),
append(make([]interface{}, 0), "SELF", "cat (string)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
append(make([]interface{}, 0), "SELF", "cat (string)", "COLLECT", "TRAVERSE_ARRAY"),
},
{
"with(.a;.=3)",
append(make([]interface{}, 0), "WITH", "(", "a", "BLOCK", "SELF", "ASSIGN", "3 (int64)", ")"),
append(make([]interface{}, 0), "a", "SELF", "3 (int64)", "ASSIGN", "BLOCK", "WITH"),
},
{
"0x12",
append(make([]interface{}, 0), "18 (int64)"),
append(make([]interface{}, 0), "18 (int64)"),
},
{
"0X12",
append(make([]interface{}, 0), "18 (int64)"),
append(make([]interface{}, 0), "18 (int64)"),
},
{
".a\n",
append(make([]interface{}, 0), "a"),
append(make([]interface{}, 0), "a"),
},
{
variableWithNewLine,
append(make([]interface{}, 0), "cat\n (string)"),
append(make([]interface{}, 0), "cat\n (string)"),
},
{
`.[0]`,
append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
},
{
`.[0][1]`,
append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]", "TRAVERSE_ARRAY", "[", "1 (int64)", "]"),
append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "1 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
},
{
`"\""`,
append(make([]interface{}, 0), "\" (string)"),
append(make([]interface{}, 0), "\" (string)"),
},
{
`[]|join(".")`,
append(make([]interface{}, 0), "[", "EMPTY", "]", "PIPE", "JOIN", "(", ". (string)", ")"),
append(make([]interface{}, 0), "EMPTY", "COLLECT", "SHORT_PIPE", ". (string)", "JOIN", "PIPE"),
},
{
`{"cool": .b or .c}`,
append(make([]interface{}, 0), "{", "cool (string)", "CREATE_MAP", "b", "OR", "c", "}"),
append(make([]interface{}, 0), "cool (string)", "b", "c", "OR", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"),
},
{
`{"cool": []|join(".")}`,
append(make([]interface{}, 0), "{", "cool (string)", "CREATE_MAP", "[", "EMPTY", "]", "PIPE", "JOIN", "(", ". (string)", ")", "}"),
append(make([]interface{}, 0), "cool (string)", "EMPTY", "COLLECT", "SHORT_PIPE", ". (string)", "JOIN", "PIPE", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"),
},
{
`.a as $item ireduce (0; . + $item)`, // note - add code to shuffle reduce to this position for postfix
append(make([]interface{}, 0), "a", "ASSIGN_VARIABLE", "GET_VARIABLE", "REDUCE", "(", "0 (int64)", "BLOCK", "SELF", "ADD", "GET_VARIABLE", ")"),
append(make([]interface{}, 0), "a", "GET_VARIABLE", "ASSIGN_VARIABLE", "0 (int64)", "SELF", "GET_VARIABLE", "ADD", "BLOCK", "REDUCE"),
},
{
`.a | .b | .c`,
append(make([]interface{}, 0), "a", "PIPE", "b", "PIPE", "c"),
append(make([]interface{}, 0), "a", "b", "c", "PIPE", "PIPE"),
},
{
`[]`,
append(make([]interface{}, 0), "[", "EMPTY", "]"),
append(make([]interface{}, 0), "EMPTY", "COLLECT", "SHORT_PIPE"),
},
{
`{}`,
append(make([]interface{}, 0), "{", "EMPTY", "}"),
append(make([]interface{}, 0), "EMPTY", "COLLECT_OBJECT", "SHORT_PIPE"),
},
{
`[{}]`,
append(make([]interface{}, 0), "[", "{", "EMPTY", "}", "]"),
append(make([]interface{}, 0), "EMPTY", "COLLECT_OBJECT", "SHORT_PIPE", "COLLECT", "SHORT_PIPE"),
},
{
`.realnames as $names | $names["anon"]`,
append(make([]interface{}, 0), "realnames", "ASSIGN_VARIABLE", "GET_VARIABLE", "PIPE", "GET_VARIABLE", "TRAVERSE_ARRAY", "[", "anon (string)", "]"),
append(make([]interface{}, 0), "realnames", "GET_VARIABLE", "ASSIGN_VARIABLE", "GET_VARIABLE", "anon (string)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "PIPE"),
},
{
`.b[.a]`,
append(make([]interface{}, 0), "b", "TRAVERSE_ARRAY", "[", "a", "]"),
append(make([]interface{}, 0), "b", "a", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
},
{
`.b[.a]?`,
append(make([]interface{}, 0), "b", "TRAVERSE_ARRAY", "[", "a", "]"),
append(make([]interface{}, 0), "b", "a", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
},
{
`.[]`,
append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
append(make([]interface{}, 0), "SELF", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
},
{
`.a[]`,
append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
},
{
`.a[]?`,
append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
},
{
`.a.[]`,
append(make([]interface{}, 0), "a", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
append(make([]interface{}, 0), "a", "SELF", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
},
{
`.a[0]`,
append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
},
{
`.a[0]?`,
append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
},
{
`.a.[0]`,
append(make([]interface{}, 0), "a", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
append(make([]interface{}, 0), "a", "SELF", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
},
{
`.a[].c`,
append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "SHORT_PIPE", "c"),
append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "c", "SHORT_PIPE"),
},
{
`[3]`,
append(make([]interface{}, 0), "[", "3 (int64)", "]"),
append(make([]interface{}, 0), "3 (int64)", "COLLECT", "SHORT_PIPE"),
},
{
`.key.array + .key.array2`,
append(make([]interface{}, 0), "key", "SHORT_PIPE", "array", "ADD", "key", "SHORT_PIPE", "array2"),
append(make([]interface{}, 0), "key", "array", "SHORT_PIPE", "key", "array2", "SHORT_PIPE", "ADD"),
},
{
`.key.array * .key.array2`,
append(make([]interface{}, 0), "key", "SHORT_PIPE", "array", "MULTIPLY", "key", "SHORT_PIPE", "array2"),
append(make([]interface{}, 0), "key", "array", "SHORT_PIPE", "key", "array2", "SHORT_PIPE", "MULTIPLY"),
},
{
`.key.array // .key.array2`,
append(make([]interface{}, 0), "key", "SHORT_PIPE", "array", "ALTERNATIVE", "key", "SHORT_PIPE", "array2"),
append(make([]interface{}, 0), "key", "array", "SHORT_PIPE", "key", "array2", "SHORT_PIPE", "ALTERNATIVE"),
},
{
`.a | .[].b == "apple"`,
append(make([]interface{}, 0), "a", "PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "SHORT_PIPE", "b", "EQUALS", "apple (string)"),
append(make([]interface{}, 0), "a", "SELF", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "b", "SHORT_PIPE", "apple (string)", "EQUALS", "PIPE"),
},
{
`(.a | .[].b) == "apple"`,
append(make([]interface{}, 0), "(", "a", "PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "SHORT_PIPE", "b", ")", "EQUALS", "apple (string)"),
append(make([]interface{}, 0), "a", "SELF", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "b", "SHORT_PIPE", "PIPE", "apple (string)", "EQUALS"),
},
{
`.[] | select(. == "*at")`,
append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "PIPE", "SELECT", "(", "SELF", "EQUALS", "*at (string)", ")"),
append(make([]interface{}, 0), "SELF", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SELF", "*at (string)", "EQUALS", "SELECT", "PIPE"),
},
{
`[true]`,
append(make([]interface{}, 0), "[", "true (bool)", "]"),
append(make([]interface{}, 0), "true (bool)", "COLLECT", "SHORT_PIPE"),
},
{
`[true, false]`,
append(make([]interface{}, 0), "[", "true (bool)", "UNION", "false (bool)", "]"),
append(make([]interface{}, 0), "true (bool)", "false (bool)", "UNION", "COLLECT", "SHORT_PIPE"),
},
{
`"mike": .a`,
append(make([]interface{}, 0), "mike (string)", "CREATE_MAP", "a"),
append(make([]interface{}, 0), "mike (string)", "a", "CREATE_MAP"),
},
{
`.a: "mike"`,
append(make([]interface{}, 0), "a", "CREATE_MAP", "mike (string)"),
append(make([]interface{}, 0), "a", "mike (string)", "CREATE_MAP"),
},
{
`{"mike": .a}`,
append(make([]interface{}, 0), "{", "mike (string)", "CREATE_MAP", "a", "}"),
append(make([]interface{}, 0), "mike (string)", "a", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"),
},
{
`{.a: "mike"}`,
append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "mike (string)", "}"),
append(make([]interface{}, 0), "a", "mike (string)", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"),
},
{
`{.a: .c, .b.[]: .f.g[]}`,
append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "c", "UNION", "b", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "CREATE_MAP", "f", "SHORT_PIPE", "g", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "}"),
append(make([]interface{}, 0), "a", "c", "CREATE_MAP", "b", "SELF", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE", "f", "g", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE", "CREATE_MAP", "UNION", "COLLECT_OBJECT", "SHORT_PIPE"),
},
{
`explode(.a.b)`,
append(make([]interface{}, 0), "EXPLODE", "(", "a", "SHORT_PIPE", "b", ")"),
append(make([]interface{}, 0), "a", "b", "SHORT_PIPE", "EXPLODE"),
},
{
`.a.b style="folded"`,
append(make([]interface{}, 0), "a", "SHORT_PIPE", "b", "ASSIGN_STYLE", "folded (string)"),
append(make([]interface{}, 0), "a", "b", "SHORT_PIPE", "folded (string)", "ASSIGN_STYLE"),
},
{
`tag == "str"`,
append(make([]interface{}, 0), "GET_TAG", "EQUALS", "str (string)"),
append(make([]interface{}, 0), "GET_TAG", "str (string)", "EQUALS"),
},
{
`. tag= "str"`,
append(make([]interface{}, 0), "SELF", "ASSIGN_TAG", "str (string)"),
append(make([]interface{}, 0), "SELF", "str (string)", "ASSIGN_TAG"),
},
{
`lineComment == "str"`,
append(make([]interface{}, 0), "GET_COMMENT", "EQUALS", "str (string)"),
append(make([]interface{}, 0), "GET_COMMENT", "str (string)", "EQUALS"),
},
{
`. lineComment= "str"`,
append(make([]interface{}, 0), "SELF", "ASSIGN_COMMENT", "str (string)"),
append(make([]interface{}, 0), "SELF", "str (string)", "ASSIGN_COMMENT"),
},
{
`. lineComment |= "str"`,
append(make([]interface{}, 0), "SELF", "ASSIGN_COMMENT", "str (string)"),
append(make([]interface{}, 0), "SELF", "str (string)", "ASSIGN_COMMENT"),
},
{
`.a.b tag="!!str"`,
append(make([]interface{}, 0), "a", "SHORT_PIPE", "b", "ASSIGN_TAG", "!!str (string)"),
append(make([]interface{}, 0), "a", "b", "SHORT_PIPE", "!!str (string)", "ASSIGN_TAG"),
},
{
`""`,
append(make([]interface{}, 0), " (string)"),
append(make([]interface{}, 0), " (string)"),
},
{
`.foo* | (. style="flow")`,
append(make([]interface{}, 0), "foo*", "PIPE", "(", "SELF", "ASSIGN_STYLE", "flow (string)", ")"),
append(make([]interface{}, 0), "foo*", "SELF", "flow (string)", "ASSIGN_STYLE", "PIPE"),
},
// {
// "0x12",
// append(make([]interface{}, 0), "18 (int64)"),
// append(make([]interface{}, 0), "18 (int64)"),
// },
// {
// "0X12",
// append(make([]interface{}, 0), "18 (int64)"),
// append(make([]interface{}, 0), "18 (int64)"),
// },
// {
// ".a\n",
// append(make([]interface{}, 0), "a"),
// append(make([]interface{}, 0), "a"),
// },
// {
// variableWithNewLine,
// append(make([]interface{}, 0), "cat\n (string)"),
// append(make([]interface{}, 0), "cat\n (string)"),
// },
// {
// `.[0]`,
// append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
// append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
// },
// {
// `.[0][1]`,
// append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]", "TRAVERSE_ARRAY", "[", "1 (int64)", "]"),
// append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "1 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
// },
// {
// `"\""`,
// append(make([]interface{}, 0), "\" (string)"),
// append(make([]interface{}, 0), "\" (string)"),
// },
// {
// `[]|join(".")`,
// append(make([]interface{}, 0), "[", "EMPTY", "]", "PIPE", "JOIN", "(", ". (string)", ")"),
// append(make([]interface{}, 0), "EMPTY", "COLLECT", "SHORT_PIPE", ". (string)", "JOIN", "PIPE"),
// },
// {
// `{"cool": .b or .c}`,
// append(make([]interface{}, 0), "{", "cool (string)", "CREATE_MAP", "b", "OR", "c", "}"),
// append(make([]interface{}, 0), "cool (string)", "b", "c", "OR", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"),
// },
// {
// `{"cool": []|join(".")}`,
// append(make([]interface{}, 0), "{", "cool (string)", "CREATE_MAP", "[", "EMPTY", "]", "PIPE", "JOIN", "(", ". (string)", ")", "}"),
// append(make([]interface{}, 0), "cool (string)", "EMPTY", "COLLECT", "SHORT_PIPE", ". (string)", "JOIN", "PIPE", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"),
// },
// {
// `.a as $item ireduce (0; . + $item)`, // note - add code to shuffle reduce to this position for postfix
// append(make([]interface{}, 0), "a", "ASSIGN_VARIABLE", "GET_VARIABLE", "REDUCE", "(", "0 (int64)", "BLOCK", "SELF", "ADD", "GET_VARIABLE", ")"),
// append(make([]interface{}, 0), "a", "GET_VARIABLE", "ASSIGN_VARIABLE", "0 (int64)", "SELF", "GET_VARIABLE", "ADD", "BLOCK", "REDUCE"),
// },
// {
// `.a | .b | .c`,
// append(make([]interface{}, 0), "a", "PIPE", "b", "PIPE", "c"),
// append(make([]interface{}, 0), "a", "b", "c", "PIPE", "PIPE"),
// },
// {
// `[]`,
// append(make([]interface{}, 0), "[", "EMPTY", "]"),
// append(make([]interface{}, 0), "EMPTY", "COLLECT", "SHORT_PIPE"),
// },
// {
// `{}`,
// append(make([]interface{}, 0), "{", "EMPTY", "}"),
// append(make([]interface{}, 0), "EMPTY", "COLLECT_OBJECT", "SHORT_PIPE"),
// },
// {
// `[{}]`,
// append(make([]interface{}, 0), "[", "{", "EMPTY", "}", "]"),
// append(make([]interface{}, 0), "EMPTY", "COLLECT_OBJECT", "SHORT_PIPE", "COLLECT", "SHORT_PIPE"),
// },
// {
// `.realnames as $names | $names["anon"]`,
// append(make([]interface{}, 0), "realnames", "ASSIGN_VARIABLE", "GET_VARIABLE", "PIPE", "GET_VARIABLE", "TRAVERSE_ARRAY", "[", "anon (string)", "]"),
// append(make([]interface{}, 0), "realnames", "GET_VARIABLE", "ASSIGN_VARIABLE", "GET_VARIABLE", "anon (string)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "PIPE"),
// },
// {
// `.b[.a]`,
// append(make([]interface{}, 0), "b", "TRAVERSE_ARRAY", "[", "a", "]"),
// append(make([]interface{}, 0), "b", "a", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
// },
// {
// `.b[.a]?`,
// append(make([]interface{}, 0), "b", "TRAVERSE_ARRAY", "[", "a", "]"),
// append(make([]interface{}, 0), "b", "a", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
// },
// {
// `.[]`,
// append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
// append(make([]interface{}, 0), "SELF", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
// },
// {
// `.a[]`,
// append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
// append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
// },
// {
// `.a[]?`,
// append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
// append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
// },
// {
// `.a.[]`,
// append(make([]interface{}, 0), "a", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
// append(make([]interface{}, 0), "a", "SELF", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
// },
// {
// `.a[0]`,
// append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
// append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
// },
// {
// `.a[0]?`,
// append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
// append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
// },
// {
// `.a.[0]`,
// append(make([]interface{}, 0), "a", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
// append(make([]interface{}, 0), "a", "SELF", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
// },
// {
// `.a[].c`,
// append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "SHORT_PIPE", "c"),
// append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "c", "SHORT_PIPE"),
// },
// {
// `[3]`,
// append(make([]interface{}, 0), "[", "3 (int64)", "]"),
// append(make([]interface{}, 0), "3 (int64)", "COLLECT", "SHORT_PIPE"),
// },
// {
// `.key.array + .key.array2`,
// append(make([]interface{}, 0), "key", "SHORT_PIPE", "array", "ADD", "key", "SHORT_PIPE", "array2"),
// append(make([]interface{}, 0), "key", "array", "SHORT_PIPE", "key", "array2", "SHORT_PIPE", "ADD"),
// },
// {
// `.key.array * .key.array2`,
// append(make([]interface{}, 0), "key", "SHORT_PIPE", "array", "MULTIPLY", "key", "SHORT_PIPE", "array2"),
// append(make([]interface{}, 0), "key", "array", "SHORT_PIPE", "key", "array2", "SHORT_PIPE", "MULTIPLY"),
// },
// {
// `.key.array // .key.array2`,
// append(make([]interface{}, 0), "key", "SHORT_PIPE", "array", "ALTERNATIVE", "key", "SHORT_PIPE", "array2"),
// append(make([]interface{}, 0), "key", "array", "SHORT_PIPE", "key", "array2", "SHORT_PIPE", "ALTERNATIVE"),
// },
// {
// `.a | .[].b == "apple"`,
// append(make([]interface{}, 0), "a", "PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "SHORT_PIPE", "b", "EQUALS", "apple (string)"),
// append(make([]interface{}, 0), "a", "SELF", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "b", "SHORT_PIPE", "apple (string)", "EQUALS", "PIPE"),
// },
// {
// `(.a | .[].b) == "apple"`,
// append(make([]interface{}, 0), "(", "a", "PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "SHORT_PIPE", "b", ")", "EQUALS", "apple (string)"),
// append(make([]interface{}, 0), "a", "SELF", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "b", "SHORT_PIPE", "PIPE", "apple (string)", "EQUALS"),
// },
// {
// `.[] | select(. == "*at")`,
// append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "PIPE", "SELECT", "(", "SELF", "EQUALS", "*at (string)", ")"),
// append(make([]interface{}, 0), "SELF", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SELF", "*at (string)", "EQUALS", "SELECT", "PIPE"),
// },
// {
// `[true]`,
// append(make([]interface{}, 0), "[", "true (bool)", "]"),
// append(make([]interface{}, 0), "true (bool)", "COLLECT", "SHORT_PIPE"),
// },
// {
// `[true, false]`,
// append(make([]interface{}, 0), "[", "true (bool)", "UNION", "false (bool)", "]"),
// append(make([]interface{}, 0), "true (bool)", "false (bool)", "UNION", "COLLECT", "SHORT_PIPE"),
// },
// {
// `"mike": .a`,
// append(make([]interface{}, 0), "mike (string)", "CREATE_MAP", "a"),
// append(make([]interface{}, 0), "mike (string)", "a", "CREATE_MAP"),
// },
// {
// `.a: "mike"`,
// append(make([]interface{}, 0), "a", "CREATE_MAP", "mike (string)"),
// append(make([]interface{}, 0), "a", "mike (string)", "CREATE_MAP"),
// },
// {
// `{"mike": .a}`,
// append(make([]interface{}, 0), "{", "mike (string)", "CREATE_MAP", "a", "}"),
// append(make([]interface{}, 0), "mike (string)", "a", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"),
// },
// {
// `{.a: "mike"}`,
// append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "mike (string)", "}"),
// append(make([]interface{}, 0), "a", "mike (string)", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"),
// },
// {
// `{.a: .c, .b.[]: .f.g[]}`,
// append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "c", "UNION", "b", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "CREATE_MAP", "f", "SHORT_PIPE", "g", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "}"),
// append(make([]interface{}, 0), "a", "c", "CREATE_MAP", "b", "SELF", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE", "f", "g", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE", "CREATE_MAP", "UNION", "COLLECT_OBJECT", "SHORT_PIPE"),
// },
// {
// `explode(.a.b)`,
// append(make([]interface{}, 0), "EXPLODE", "(", "a", "SHORT_PIPE", "b", ")"),
// append(make([]interface{}, 0), "a", "b", "SHORT_PIPE", "EXPLODE"),
// },
// {
// `.a.b style="folded"`,
// append(make([]interface{}, 0), "a", "SHORT_PIPE", "b", "ASSIGN_STYLE", "folded (string)"),
// append(make([]interface{}, 0), "a", "b", "SHORT_PIPE", "folded (string)", "ASSIGN_STYLE"),
// },
// {
// `tag == "str"`,
// append(make([]interface{}, 0), "GET_TAG", "EQUALS", "str (string)"),
// append(make([]interface{}, 0), "GET_TAG", "str (string)", "EQUALS"),
// },
// {
// `. tag= "str"`,
// append(make([]interface{}, 0), "SELF", "ASSIGN_TAG", "str (string)"),
// append(make([]interface{}, 0), "SELF", "str (string)", "ASSIGN_TAG"),
// },
// {
// `lineComment == "str"`,
// append(make([]interface{}, 0), "GET_COMMENT", "EQUALS", "str (string)"),
// append(make([]interface{}, 0), "GET_COMMENT", "str (string)", "EQUALS"),
// },
// {
// `. lineComment= "str"`,
// append(make([]interface{}, 0), "SELF", "ASSIGN_COMMENT", "str (string)"),
// append(make([]interface{}, 0), "SELF", "str (string)", "ASSIGN_COMMENT"),
// },
// {
// `. lineComment |= "str"`,
// append(make([]interface{}, 0), "SELF", "ASSIGN_COMMENT", "str (string)"),
// append(make([]interface{}, 0), "SELF", "str (string)", "ASSIGN_COMMENT"),
// },
// {
// `.a.b tag="!!str"`,
// append(make([]interface{}, 0), "a", "SHORT_PIPE", "b", "ASSIGN_TAG", "!!str (string)"),
// append(make([]interface{}, 0), "a", "b", "SHORT_PIPE", "!!str (string)", "ASSIGN_TAG"),
// },
// {
// `""`,
// append(make([]interface{}, 0), " (string)"),
// append(make([]interface{}, 0), " (string)"),
// },
// {
// `.foo* | (. style="flow")`,
// append(make([]interface{}, 0), "foo*", "PIPE", "(", "SELF", "ASSIGN_STYLE", "flow (string)", ")"),
// append(make([]interface{}, 0), "foo*", "SELF", "flow (string)", "ASSIGN_STYLE", "PIPE"),
// },
}
var tokeniser = newExpressionTokeniser()

2
pkg/yqlib/lib.go
View File

@ -58,7 +58,7 @@ var createMapOpType = &operationType{Type: "CREATE_MAP", NumArgs: 2, Precedence:
var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence: 45, Handler: pipeOperator}
var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator}
var collectOpType = &operationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: collectOperator}
var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50, Handler: collectOperator}
var encodeOpType = &operationType{Type: "ENCODE", NumArgs: 0, Precedence: 50, Handler: encodeOperator}
var decodeOpType = &operationType{Type: "DECODE", NumArgs: 0, Precedence: 50, Handler: decodeOperator}

62
pkg/yqlib/operator_collect.go
View File

@ -6,6 +6,23 @@ import (
yaml "gopkg.in/yaml.v3"
)
func collectTogether(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (*CandidateNode, error) {
collectedNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
collectExpResults, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode)
if err != nil {
return nil, err
}
for result := collectExpResults.MatchingNodes.Front(); result != nil; result = result.Next() {
resultC := result.Value.(*CandidateNode)
log.Debugf("found this: %v", NodeToString(resultC))
collectedNode.Content = append(collectedNode.Content, unwrapDoc(resultC.Node))
}
}
return &CandidateNode{Node: collectedNode}, nil
}
func collectOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("-- collectOperation")
@ -15,26 +32,43 @@ func collectOperator(d *dataTreeNavigator, context Context, expressionNode *Expr
return context.SingleChildContext(candidate), nil
}
var results = list.New()
node := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
var collectC *CandidateNode
if context.MatchingNodes.Front() != nil {
collectC = context.MatchingNodes.Front().Value.(*CandidateNode).CreateReplacement(node)
if len(collectC.Path) > 0 {
collectC.Path = collectC.Path[:len(collectC.Path)-1]
var evaluateAllTogether = true
for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {
evaluateAllTogether = evaluateAllTogether && matchEl.Value.(*CandidateNode).EvaluateTogether
if !evaluateAllTogether {
break
}
} else {
collectC = &CandidateNode{Node: node}
}
if evaluateAllTogether {
collectedNode, err := collectTogether(d, context, expressionNode.Rhs)
if err != nil {
return Context{}, err
}
return context.SingleChildContext(collectedNode), nil
}
var results = list.New()
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
log.Debugf("Collecting %v", NodeToString(candidate))
node.Content = append(node.Content, unwrapDoc(candidate.Node))
}
results.PushBack(collectC)
collectedNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
collectCandidate := candidate.CreateReplacement(collectedNode)
collectExpResults, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
if err != nil {
return Context{}, err
}
for result := collectExpResults.MatchingNodes.Front(); result != nil; result = result.Next() {
resultC := result.Value.(*CandidateNode)
log.Debugf("found this: %v", NodeToString(resultC))
collectedNode.Content = append(collectedNode.Content, unwrapDoc(resultC.Node))
}
results.PushBack(collectCandidate)
}
return context.ChildContext(results), nil
}

13
pkg/yqlib/operator_collect_object_test.go
View File

@ -5,6 +5,15 @@ import (
)
var collectObjectOperatorScenarios = []expressionScenario{
{
skipDoc: true,
document: `[{name: cat}, {name: dog}]`,
expression: `.[] | {.name: "great"}`,
expected: []string{
"D0, P[], (!!map)::cat: great\n",
"D0, P[], (!!map)::dog: great\n",
},
},
{
skipDoc: true,
document: "a: []",
@ -99,9 +108,7 @@ var collectObjectOperatorScenarios = []expressionScenario{
document: `{name: Mike, pets: {cows: [apl, bba]}}`,
expression: `{"a":.name, "b":.pets}`,
expected: []string{
`D0, P[], (!!map)::a: Mike
b: {cows: [apl, bba]}
`,
"D0, P[], (!!map)::a: Mike\nb: {cows: [apl, bba]}\n",
},
},
{

16
pkg/yqlib/operator_collect_test.go
View File

@ -45,7 +45,8 @@ var collectOperatorScenarios = []expressionScenario{
expected: []string{
"D0, P[], (!!seq)::- cat\n",
},
}, {
},
{
document: ``,
skipDoc: true,
expression: `[true]`,
@ -64,7 +65,7 @@ var collectOperatorScenarios = []expressionScenario{
{
document: ``,
skipDoc: true,
expression: `1 | collect`,
expression: `collect(1)`,
expected: []string{
"D0, P[], (!!seq)::- 1\n",
},
@ -82,7 +83,16 @@ var collectOperatorScenarios = []expressionScenario{
expression: `[.a.b.[]]`,
skipDoc: true,
expected: []string{
"D0, P[a b], (!!seq)::- 1\n- 2\n- 3\n",
"D0, P[], (!!seq)::- 1\n- 2\n- 3\n",
},
},
{
skipDoc: true,
document: `[{name: cat, thing: bor}, {name: dog}]`,
expression: `.[] | [.name]`,
expected: []string{
"D0, P[0], (!!seq)::- cat\n",
"D0, P[1], (!!seq)::- dog\n",
},
},
}

44
pkg/yqlib/operator_entries.go
View File

@ -138,25 +138,37 @@ func withEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *
return Context{}, err
}
//run expression against entries
// splat toEntries and pipe it into Rhs
splatted, err := splat(d, toEntries, traversePreferences{})
if err != nil {
return Context{}, err
}
var results = list.New()
result, err := d.GetMatchingNodes(splatted, expressionNode.Rhs)
log.Debug("expressionNode.Rhs %v", expressionNode.Rhs.Operation.OperationType)
log.Debug("result %v", result)
if err != nil {
return Context{}, err
}
for el := toEntries.MatchingNodes.Front(); el != nil; el = el.Next() {
//run expression against entries
// splat toEntries and pipe it into Rhs
splatted, err := splat(d, context.SingleChildContext(el.Value.(*CandidateNode)), traversePreferences{})
if err != nil {
return Context{}, err
}
result, err := d.GetMatchingNodes(splatted, expressionNode.Rhs)
log.Debug("expressionNode.Rhs %v", expressionNode.Rhs.Operation.OperationType)
log.Debug("result %v", result)
if err != nil {
return Context{}, err
}
selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
collected, err := collectTogether(d, result, selfExpression)
if err != nil {
return Context{}, err
}
fromEntries, err := fromEntriesOperator(d, context.SingleChildContext(collected), expressionNode)
if err != nil {
return Context{}, err
}
results.PushBackList(fromEntries.MatchingNodes)
collected, err := collectOperator(d, result, expressionNode)
if err != nil {
return Context{}, err
}
//from_entries on the result
return fromEntriesOperator(d, collected, expressionNode)
return context.ChildContext(results), nil
}

2
pkg/yqlib/operator_entries_test.go
View File

@ -47,9 +47,11 @@ var entriesOperatorScenarios = []expressionScenario{
{
description: "Use with_entries to update keys",
document: `{a: 1, b: 2}`,
document2: `{c: 1, d: 2}`,
expression: `with_entries(.key |= "KEY_" + .)`,
expected: []string{
"D0, P[], (!!map)::KEY_a: 1\nKEY_b: 2\n",
"D0, P[], (!!map)::KEY_c: 1\nKEY_d: 2\n",
},
},
{

15
pkg/yqlib/operator_path_test.go
View File

@ -57,16 +57,11 @@ var pathOperatorScenarios = []expressionScenario{
{
description: "Print path and value",
document: `{a: [cat, dog, frog]}`,
expression: `.a.[] | select(. == "*og") | [{"path":path, "value":.}]`,
expected: []string{`D0, P[], (!!seq)::- path:
- a
- 1
value: dog
- path:
- a
- 2
value: frog
`},
expression: `.a[] | select(. == "*og") | [{"path":path, "value":.}]`,
expected: []string{
"D0, P[a 1], (!!seq)::- path:\n - a\n - 1\n value: dog\n",
"D0, P[a 2], (!!seq)::- path:\n - a\n - 2\n value: frog\n",
},
},
}

4
pkg/yqlib/operator_recursive_descent_test.go
View File

@ -187,7 +187,7 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
document: mergeDocSample,
expression: `.foobar | [..]`,
expected: []string{
"D0, P[], (!!seq)::- c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n- foobar_c\n- *foo\n- foobar_thing\n",
"D0, P[foobar], (!!seq)::- c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n- foobar_c\n- *foo\n- foobar_thing\n",
},
},
{
@ -195,7 +195,7 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
document: mergeDocSample,
expression: `.foobar | [...]`,
expected: []string{
"D0, P[], (!!seq)::- c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n- c\n- foobar_c\n- !!merge <<\n- *foo\n- thing\n- foobar_thing\n",
"D0, P[foobar], (!!seq)::- c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n- c\n- foobar_c\n- !!merge <<\n- *foo\n- thing\n- foobar_thing\n",
},
},
{

7
pkg/yqlib/operator_traverse_path.go
View File

@ -96,8 +96,11 @@ func traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode
}
prefs := traversePreferences{}
if expressionNode.Rhs.Rhs != nil && expressionNode.Rhs.Rhs.Operation.Preferences != nil {
prefs = expressionNode.Rhs.Rhs.Operation.Preferences.(traversePreferences)
// we could have .[blah]?
// and we sneakily put the traverse prefs on the RHS prefs
// dodgey :/
if expressionNode.Rhs != nil && expressionNode.Rhs.Operation.Preferences != nil {
prefs = expressionNode.Rhs.Operation.Preferences.(traversePreferences)
}
var indicesToTraverse = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node.Content

20
test.sh Executable file
View File

@ -0,0 +1,20 @@
#!/bin/bash
# load array into a bash array
# need to output each entry as a single line
# readarray identityMappings < <(./yq e -o=j -I=0 '.identitymappings[]' test.yml )
# for identityMapping in "${identityMappings[@]}"; do
# # identity mapping is a yaml snippet representing a single entry
# roleArn=$(echo "$identityMapping" | yq e '.arn' -)
# echo "roleArn: $roleArn"
# done
while IFS=$'\t' read -r roleArn group user _; do
echo "Role: $roleArn"
echo "Group: $group"
echo "User: $user"
done < <(yq -j read test.yaml \
| jq -r '.identitymappings[] | [.arn, .group, .user] | @tsv')