suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2024-11-12 05:38:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added recurse keys operator

Browse Source
This commit is contained in:
Mike Farah 2020-12-28 11:24:42 +11:00
parent b10a9ccfc6
commit 88663a6ce3
10 changed files with 273 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
pkg/yqlib/doc/Recursive Descent.md
View File

@ -1,8 +1,55 @@
This operator recursively matches all children nodes given of a particular element, including that node itself. This is most often used to apply a filter recursively against all matches, for instance to set the `style` of all nodes in a yaml doc:
This operator recursively matches all children nodes given of a particular element, including that node itself. This is most often used to apply a filter recursively against all matches. It can be used in either the
## match values form `..`
This will, like the `jq` equivalent, recursively match all _value_ nodes. Use it to find/manipulate particular values.
For instance to set the `style` of all _value_ nodes in a yaml doc, excluding map keys:
```bash
yq eval '.. style= "flow"' file.yaml
```
## match values and map keys form `...`
The also includes map keys in the results set. This is particularly useful in YAML as unlike JSON, map keys can have their own styling, tags and use anchors and aliases.
For instance to set the `style` of all nodes in a yaml doc, including the map keys:
```bash
yq eval '... style= "flow"' file.yaml
```
## Recurse map (values only)
Given a sample.yml file of:
```yaml
a: frog
```
then
```bash
yq eval '..' sample.yml
```
will output
```yaml
a: frog
frog
```
## Recurse map (values and keys)
Note that the map key appears in the results
Given a sample.yml file of:
```yaml
a: frog
```
then
```bash
yq eval '...' sample.yml
```
will output
```yaml
a: frog
a
frog
```
## Aliases are not traversed
Given a sample.yml file of:
```yaml

30
pkg/yqlib/doc/Style.md
View File

@ -40,6 +40,26 @@ c: "3.2"
e: "true"
```
## Set double quote style on map keys too
Given a sample.yml file of:
```yaml
a: cat
b: 5
c: 3.2
e: true
```
then
```bash
yq eval '... style="double"' sample.yml
```
will output
```yaml
"a": "cat"
"b": "5"
"c": "3.2"
"e": "true"
```
## Set single quote style
Given a sample.yml file of:
```yaml
@ -126,18 +146,18 @@ will output
```
## Pretty print
Set empty (default) quote style
Set empty (default) quote style, note the usage of `...` to match keys too.
Given a sample.yml file of:
```yaml
a: cat
b: 5
c: 3.2
e: true
"b": 5
'c': 3.2
"e": true
```
then
```bash
yq eval '.. style=""' sample.yml
yq eval '... style=""' sample.yml
```
will output
```yaml

16
pkg/yqlib/doc/headers/Recursive Descent.md
View File

@ -1,5 +1,19 @@
This operator recursively matches all children nodes given of a particular element, including that node itself. This is most often used to apply a filter recursively against all matches, for instance to set the `style` of all nodes in a yaml doc:
This operator recursively matches all children nodes given of a particular element, including that node itself. This is most often used to apply a filter recursively against all matches. It can be used in either the
## match values form `..`
This will, like the `jq` equivalent, recursively match all _value_ nodes. Use it to find/manipulate particular values.
For instance to set the `style` of all _value_ nodes in a yaml doc, excluding map keys:
```bash
yq eval '.. style= "flow"' file.yaml
```
## match values and map keys form `...`
The also includes map keys in the results set. This is particularly useful in YAML as unlike JSON, map keys can have their own styling, tags and use anchors and aliases.
For instance to set the `style` of all nodes in a yaml doc, including the map keys:
```bash
yq eval '... style= "flow"' file.yaml
```

4
pkg/yqlib/operator_collect_object.go
View File

@ -67,7 +67,9 @@ func collect(d *dataTreeNavigator, aggregate *list.List, remainingMatches *list.
}
candidate := remainingMatches.Remove(remainingMatches.Front()).(*CandidateNode)
splatted, err := Splat(d, nodeToMap(candidate))
splatted, err := Splat(d, nodeToMap(candidate),
&TraversePreferences{FollowAlias: false, IncludeMapKeys: false})
for splatEl := splatted.Front(); splatEl != nil; splatEl = splatEl.Next() {
splatEl.Value.(*CandidateNode).Path = nil

4
pkg/yqlib/operator_multiply.go
View File

@ -85,7 +85,9 @@ func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode,
var results = list.New()
// shouldn't recurse arrays if appending
err := recursiveDecent(d, results, nodeToMap(rhs), !shouldAppendArrays)
prefs := &RecursiveDescentPreferences{RecurseArray: !shouldAppendArrays,
TraversePreferences: &TraversePreferences{FollowAlias: false}}
err := recursiveDecent(d, results, nodeToMap(rhs), prefs)
if err != nil {
return nil, err
}

16
pkg/yqlib/operator_recursive_descent.go
View File

@ -6,10 +6,16 @@ import (
yaml "gopkg.in/yaml.v3"
)
type RecursiveDescentPreferences struct {
TraversePreferences *TraversePreferences
RecurseArray bool
}
func RecursiveDescentOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
var results = list.New()
err := recursiveDecent(d, results, matchMap, true)
preferences := pathNode.Operation.Preferences.(*RecursiveDescentPreferences)
err := recursiveDecent(d, results, matchMap, preferences)
if err != nil {
return nil, err
}
@ -17,7 +23,7 @@ func RecursiveDescentOperator(d *dataTreeNavigator, matchMap *list.List, pathNod
return results, nil
}
func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.List, recurseArray bool) error {
func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.List, preferences *RecursiveDescentPreferences) error {
for el := matchMap.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
@ -27,14 +33,14 @@ func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.Li
results.PushBack(candidate)
if candidate.Node.Kind != yaml.AliasNode && len(candidate.Node.Content) > 0 &&
(recurseArray || candidate.Node.Kind != yaml.SequenceNode) {
(preferences.RecurseArray || candidate.Node.Kind != yaml.SequenceNode) {
children, err := Splat(d, nodeToMap(candidate))
children, err := Splat(d, nodeToMap(candidate), preferences.TraversePreferences)
if err != nil {
return err
}
err = recursiveDecent(d, results, children, recurseArray)
err = recursiveDecent(d, results, children, preferences)
if err != nil {
return err
}

113
pkg/yqlib/operator_recursive_descent_test.go
View File

@ -13,6 +13,14 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
"D0, P[], (!!map)::{}\n",
},
},
{
skipDoc: true,
document: `{}`,
expression: `...`,
expected: []string{
"D0, P[], (!!map)::{}\n",
},
},
{
skipDoc: true,
document: `[]`,
@ -21,6 +29,14 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
"D0, P[], (!!seq)::[]\n",
},
},
{
skipDoc: true,
document: `[]`,
expression: `...`,
expected: []string{
"D0, P[], (!!seq)::[]\n",
},
},
{
skipDoc: true,
document: `cat`,
@ -31,13 +47,32 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
},
{
skipDoc: true,
document: `{a: frog}`,
expression: `..`,
document: `cat`,
expression: `...`,
expected: []string{
"D0, P[], (!!str)::cat\n",
},
},
{
description: "Recurse map (values only)",
document: `{a: frog}`,
expression: `..`,
expected: []string{
"D0, P[], (!!map)::{a: frog}\n",
"D0, P[a], (!!str)::frog\n",
},
},
{
description: "Recurse map (values and keys)",
subdescription: "Note that the map key appears in the results",
document: `{a: frog}`,
expression: `...`,
expected: []string{
"D0, P[], (!!map)::{a: frog}\n",
"D0, P[a], (!!str)::a\n",
"D0, P[a], (!!str)::frog\n",
},
},
{
skipDoc: true,
document: `{a: {b: apple}}`,
@ -48,6 +83,18 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
"D0, P[a b], (!!str)::apple\n",
},
},
{
skipDoc: true,
document: `{a: {b: apple}}`,
expression: `...`,
expected: []string{
"D0, P[], (!!map)::{a: {b: apple}}\n",
"D0, P[a], (!!str)::a\n",
"D0, P[a], (!!map)::{b: apple}\n",
"D0, P[a b], (!!str)::b\n",
"D0, P[a b], (!!str)::apple\n",
},
},
{
skipDoc: true,
document: `[1,2,3]`,
@ -59,6 +106,17 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
"D0, P[2], (!!int)::3\n",
},
},
{
skipDoc: true,
document: `[1,2,3]`,
expression: `...`,
expected: []string{
"D0, P[], (!!seq)::[1, 2, 3]\n",
"D0, P[0], (!!int)::1\n",
"D0, P[1], (!!int)::2\n",
"D0, P[2], (!!int)::3\n",
},
},
{
skipDoc: true,
document: `[{a: cat},2,true]`,
@ -71,6 +129,19 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
"D0, P[2], (!!bool)::true\n",
},
},
{
skipDoc: true,
document: `[{a: cat},2,true]`,
expression: `...`,
expected: []string{
"D0, P[], (!!seq)::[{a: cat}, 2, true]\n",
"D0, P[0], (!!map)::{a: cat}\n",
"D0, P[0 a], (!!str)::a\n",
"D0, P[0 a], (!!str)::cat\n",
"D0, P[1], (!!int)::2\n",
"D0, P[2], (!!bool)::true\n",
},
},
{
description: "Aliases are not traversed",
document: `{a: &cat {c: frog}, b: *cat}`,
@ -79,6 +150,20 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
"D0, P[a], (!!seq)::- {a: &cat {c: frog}, b: *cat}\n- &cat {c: frog}\n- frog\n- *cat\n",
},
},
{
skipDoc: true,
document: `{a: &cat {c: frog}, b: *cat}`,
expression: `...`,
expected: []string{
"D0, P[], (!!map)::{a: &cat {c: frog}, b: *cat}\n",
"D0, P[a], (!!str)::a\n",
"D0, P[a], (!!map)::&cat {c: frog}\n",
"D0, P[a c], (!!str)::c\n",
"D0, P[a c], (!!str)::frog\n",
"D0, P[b], (!!str)::b\n",
"D0, P[b], (alias)::*cat\n",
},
},
{
description: "Merge docs are not traversed",
document: mergeDocSample,
@ -87,6 +172,14 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
"D0, P[foobar], (!!seq)::- c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n- foobar_c\n- *foo\n- foobar_thing\n",
},
},
{
skipDoc: true,
document: mergeDocSample,
expression: `.foobar | [...]`,
expected: []string{
"D0, P[foobar], (!!seq)::- c: foobar_c\n !!merge <<: *foo\n thing: foobar_thing\n- c\n- foobar_c\n- !!merge <<\n- *foo\n- thing\n- foobar_thing\n",
},
},
{
skipDoc: true,
document: mergeDocSample,
@ -100,6 +193,22 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
"D0, P[foobarList c], (!!str)::foobarList_c\n",
},
},
{
skipDoc: true,
document: mergeDocSample,
expression: `.foobarList | ...`,
expected: []string{
"D0, P[foobarList], (!!map)::b: foobarList_b\n!!merge <<: [*foo, *bar]\nc: foobarList_c\n",
"D0, P[foobarList b], (!!str)::b\n",
"D0, P[foobarList b], (!!str)::foobarList_b\n",
"D0, P[foobarList <<], (!!merge)::!!merge <<\n",
"D0, P[foobarList <<], (!!seq)::[*foo, *bar]\n",
"D0, P[foobarList << 0], (alias)::*foo\n",
"D0, P[foobarList << 1], (alias)::*bar\n",
"D0, P[foobarList c], (!!str)::c\n",
"D0, P[foobarList c], (!!str)::foobarList_c\n",
},
},
}
func TestRecursiveDescentOperatorScenarios(t *testing.T) {

22
pkg/yqlib/operator_style_test.go
View File

@ -21,6 +21,22 @@ var styleOperatorScenarios = []expressionScenario{
"D0, P[], (!!map)::a: \"cat\"\nb: \"5\"\nc: \"3.2\"\ne: \"true\"\n",
},
},
{
description: "Set double quote style on map keys too",
document: `{a: cat, b: 5, c: 3.2, e: true}`,
expression: `... style="double"`,
expected: []string{
"D0, P[], (!!map)::\"a\": \"cat\"\n\"b\": \"5\"\n\"c\": \"3.2\"\n\"e\": \"true\"\n",
},
},
{
skipDoc: true,
document: "bing: &foo frog\na:\n c: cat\n <<: [*foo]",
expression: `(... | select(tag=="!!str")) style="single"`,
expected: []string{
"D0, P[], (!!map)::'bing': &foo 'frog'\n'a':\n 'c': 'cat'\n !!merge <<: [*foo]\n",
},
},
{
description: "Set single quote style",
document: `{a: cat, b: 5, c: 3.2, e: true}`,
@ -71,9 +87,9 @@ e: >-
},
{
description: "Pretty print",
subdescription: "Set empty (default) quote style",
document: `{a: cat, b: 5, c: 3.2, e: true}`,
expression: `.. style=""`,
subdescription: "Set empty (default) quote style, note the usage of `...` to match keys too.",
document: `{a: cat, "b": 5, 'c': 3.2, "e": true}`,
expression: `... style=""`,
expected: []string{
"D0, P[], (!!map)::a: cat\nb: 5\nc: 3.2\ne: true\n",
},

56
pkg/yqlib/operator_traverse_path.go
View File

@ -10,11 +10,12 @@ import (
)
type TraversePreferences struct {
DontFollowAlias bool
FollowAlias bool
IncludeMapKeys bool
}
func Splat(d *dataTreeNavigator, matches *list.List) (*list.List, error) {
return traverseNodesWithArrayIndices(matches, make([]*yaml.Node, 0), false)
func Splat(d *dataTreeNavigator, matches *list.List, prefs *TraversePreferences) (*list.List, error) {
return traverseNodesWithArrayIndices(matches, make([]*yaml.Node, 0), prefs)
}
func TraversePathOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
@ -53,7 +54,8 @@ func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, operation *Oper
switch value.Kind {
case yaml.MappingNode:
log.Debug("its a map with %v entries", len(value.Content)/2)
return traverseMap(matchingNode, operation.StringValue, true, false)
prefs := &TraversePreferences{FollowAlias: true}
return traverseMap(matchingNode, operation.StringValue, prefs, false)
case yaml.SequenceNode:
log.Debug("its a sequence of %v things!", len(value.Content))
@ -84,15 +86,15 @@ func TraverseArrayOperator(d *dataTreeNavigator, matchingNodes *list.List, pathN
}
var indicesToTraverse = rhs.Front().Value.(*CandidateNode).Node.Content
return traverseNodesWithArrayIndices(matchingNodes, indicesToTraverse, true)
prefs := &TraversePreferences{FollowAlias: true}
return traverseNodesWithArrayIndices(matchingNodes, indicesToTraverse, prefs)
}
func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse []*yaml.Node, followAlias bool) (*list.List, error) {
func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse []*yaml.Node, prefs *TraversePreferences) (*list.List, error) {
var matchingNodeMap = list.New()
for el := matchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
newNodes, err := traverseArrayIndices(candidate, indicesToTraverse, followAlias)
newNodes, err := traverseArrayIndices(candidate, indicesToTraverse, prefs)
if err != nil {
return nil, err
}
@ -102,7 +104,7 @@ func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse [
return matchingNodeMap, nil
}
func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml.Node, followAlias bool) (*list.List, error) { // call this if doc / alias like the other traverse
func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml.Node, prefs *TraversePreferences) (*list.List, error) { // call this if doc / alias like the other traverse
node := matchingNode.Node
if node.Tag == "!!null" {
log.Debugf("OperatorArrayTraverse got a null - turning it into an empty array")
@ -113,32 +115,32 @@ func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml
if node.Kind == yaml.AliasNode {
matchingNode.Node = node.Alias
return traverseArrayIndices(matchingNode, indicesToTraverse, followAlias)
return traverseArrayIndices(matchingNode, indicesToTraverse, prefs)
} else if node.Kind == yaml.SequenceNode {
return traverseArrayWithIndices(matchingNode, indicesToTraverse)
} else if node.Kind == yaml.MappingNode {
return traverseMapWithIndices(matchingNode, indicesToTraverse, followAlias)
return traverseMapWithIndices(matchingNode, indicesToTraverse, prefs)
} else if node.Kind == yaml.DocumentNode {
return traverseArrayIndices(&CandidateNode{
Node: matchingNode.Node.Content[0],
Filename: matchingNode.Filename,
FileIndex: matchingNode.FileIndex,
Document: matchingNode.Document}, indicesToTraverse, followAlias)
Document: matchingNode.Document}, indicesToTraverse, prefs)
}
log.Debugf("OperatorArrayTraverse skipping %v as its a %v", matchingNode, node.Tag)
return list.New(), nil
}
func traverseMapWithIndices(candidate *CandidateNode, indices []*yaml.Node, followAlias bool) (*list.List, error) {
func traverseMapWithIndices(candidate *CandidateNode, indices []*yaml.Node, prefs *TraversePreferences) (*list.List, error) {
if len(indices) == 0 {
return traverseMap(candidate, "", followAlias, true)
return traverseMap(candidate, "", prefs, true)
}
var matchingNodeMap = list.New()
for _, indexNode := range indices {
log.Debug("traverseMapWithIndices: %v", indexNode.Value)
newNodes, err := traverseMap(candidate, indexNode.Value, followAlias, false)
newNodes, err := traverseMap(candidate, indexNode.Value, prefs, false)
if err != nil {
return nil, err
}
@ -201,9 +203,9 @@ func keyMatches(key *yaml.Node, wantedKey string) bool {
return Match(key.Value, wantedKey)
}
func traverseMap(matchingNode *CandidateNode, key string, followAlias bool, splat bool) (*list.List, error) {
func traverseMap(matchingNode *CandidateNode, key string, prefs *TraversePreferences, splat bool) (*list.List, error) {
var newMatches = orderedmap.NewOrderedMap()
err := doTraverseMap(newMatches, matchingNode, key, followAlias, splat)
err := doTraverseMap(newMatches, matchingNode, key, prefs, splat)
if err != nil {
return nil, err
@ -232,7 +234,7 @@ func traverseMap(matchingNode *CandidateNode, key string, followAlias bool, spla
return results, nil
}
func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, wantedKey string, followAlias bool, splat bool) error {
func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, wantedKey string, prefs *TraversePreferences, splat bool) error {
// value.Content is a concatenated array of key, value,
// so keys are in the even indexes, values in odd.
// merge aliases are defined first, but we only want to traverse them
@ -247,14 +249,22 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode,
log.Debug("checking %v (%v)", key.Value, key.Tag)
//skip the 'merge' tag, find a direct match first
if key.Tag == "!!merge" && followAlias {
if key.Tag == "!!merge" && prefs.FollowAlias {
log.Debug("Merge anchor")
err := traverseMergeAnchor(newMatches, candidate, value, wantedKey, splat)
err := traverseMergeAnchor(newMatches, candidate, value, wantedKey, prefs, splat)
if err != nil {
return err
}
} else if splat || keyMatches(key, wantedKey) {
log.Debug("MATCHED")
if prefs.IncludeMapKeys {
candidateNode := &CandidateNode{
Node: key,
Path: candidate.CreateChildPath(key.Value),
Document: candidate.Document,
}
newMatches.Set(fmt.Sprintf("keyOf-%v", candidateNode.GetKey()), candidateNode)
}
candidateNode := &CandidateNode{
Node: value,
Path: candidate.CreateChildPath(key.Value),
@ -267,7 +277,7 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode,
return nil
}
func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *yaml.Node, wantedKey string, splat bool) error {
func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *yaml.Node, wantedKey string, prefs *TraversePreferences, splat bool) error {
switch value.Kind {
case yaml.AliasNode:
candidateNode := &CandidateNode{
@ -275,10 +285,10 @@ func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *C
Path: originalCandidate.Path,
Document: originalCandidate.Document,
}
return doTraverseMap(newMatches, candidateNode, wantedKey, true, splat)
return doTraverseMap(newMatches, candidateNode, wantedKey, prefs, splat)
case yaml.SequenceNode:
for _, childValue := range value.Content {
err := traverseMergeAnchor(newMatches, originalCandidate, childValue, wantedKey, splat)
err := traverseMergeAnchor(newMatches, originalCandidate, childValue, wantedKey, prefs, splat)
if err != nil {
return err
}

6
pkg/yqlib/path_tokeniser.go
View File

@ -173,7 +173,11 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`\)`), literalToken(CloseBracket, true))
lexer.Add([]byte(`\.\[`), literalToken(TraverseArrayCollect, false))
lexer.Add([]byte(`\.\.`), opToken(RecursiveDescent))
lexer.Add([]byte(`\.\.`), opTokenWithPrefs(RecursiveDescent, nil, &RecursiveDescentPreferences{RecurseArray: true,
TraversePreferences: &TraversePreferences{FollowAlias: false, IncludeMapKeys: false}}))
lexer.Add([]byte(`\.\.\.`), opTokenWithPrefs(RecursiveDescent, nil, &RecursiveDescentPreferences{RecurseArray: true,
TraversePreferences: &TraversePreferences{FollowAlias: false, IncludeMapKeys: true}}))
lexer.Add([]byte(`,`), opToken(Union))
lexer.Add([]byte(`:\s*`), opToken(CreateMap))