suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2024-11-12 13:48:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added with_entries

Browse Source
This commit is contained in:
Mike Farah 2021-05-09 15:12:50 +10:00
parent 941a453163
commit cc08afc435
9 changed files with 140 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
pkg/yqlib/doc/Entries.md
View File

@ -1,4 +1,4 @@
Similar to the same named functions in `jq` these functions convert to/from an object and an array of key-value pairs. This is most useful for performing operations on keys of maps.
## to_entries Map
Given a sample.yml file of:
```yaml
@ -69,3 +69,19 @@ will output
1: b
```
## Use with_entries to update keys
Given a sample.yml file of:
```yaml
a: 1
b: 2
```
then
```bash
yq eval 'with_entries(.key |= "KEY_" + .)' sample.yml
```
will output
```yaml
KEY_a: 1
KEY_b: 2
```

17
pkg/yqlib/doc/Traverse (Read).md
View File

@ -98,6 +98,23 @@ will output
null
```
## Optional identifier
Like jq, does not output an error when the yaml is not an array or object as expected
Given a sample.yml file of:
```yaml
- 1
- 2
- 3
```
then
```bash
yq eval '.a?' sample.yml
```
will output
```yaml
```
## Wildcard matching
Given a sample.yml file of:
```yaml

1
pkg/yqlib/doc/headers/Entries.md Normal file
View File

@ -0,0 +1 @@
Similar to the same named functions in `jq` these functions convert to/from an object and an array of key-value pairs. This is most useful for performing operations on keys of maps.

14
pkg/yqlib/expression_tokeniser.go
View File

@ -63,12 +63,19 @@ func (t *token) toString(detail bool) string {
func pathToken(wrapped bool) lex.Action {
return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
value := string(m.Bytes)
prefs := traversePreferences{}
if value[len(value)-1:] == "?" {
prefs.OptionalTraverse = true
value = value[:len(value)-1]
}
value = value[1:]
if wrapped {
value = unwrap(value)
}
log.Debug("PathToken %v", value)
op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: traversePreferences{}}
op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: prefs}
return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
}
}
@ -280,6 +287,7 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`path`), opToken(getPathOpType))
lexer.Add([]byte(`to_entries`), opToken(toEntriesOpType))
lexer.Add([]byte(`from_entries`), opToken(fromEntriesOpType))
lexer.Add([]byte(`with_entries`), opToken(withEntriesOpType))
lexer.Add([]byte(`lineComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}))
@ -302,8 +310,8 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte("( |\t|\n|\r)+"), skip)
lexer.Add([]byte(`\."[^ "]+"`), pathToken(true))
lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=]+`), pathToken(false))
lexer.Add([]byte(`\."[^ "]+"\??`), pathToken(true))
lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=]+\??`), pathToken(false))
lexer.Add([]byte(`\.`), selfToken())
lexer.Add([]byte(`\|`), opToken(pipeOpType))

5
pkg/yqlib/lib.go
View File

@ -60,8 +60,11 @@ var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence:
var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator}
var collectOpType = &operationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: collectOperator}
var toEntriesOpType = &operationType{Type: "TO_ENTRIES", NumArgs: 0, Precedence: 50, Handler: toEntriesOperator}
var fromEntriesOpType = &operationType{Type: "TO_ENTRIES", NumArgs: 0, Precedence: 50, Handler: fromEntriesOperator}
var fromEntriesOpType = &operationType{Type: "FROM_ENTRIES", NumArgs: 0, Precedence: 50, Handler: fromEntriesOperator}
var withEntriesOpType = &operationType{Type: "WITH_ENTRIES", NumArgs: 1, Precedence: 50, Handler: withEntriesOperator}
var splitDocumentOpType = &operationType{Type: "SPLIT_DOC", NumArgs: 0, Precedence: 50, Handler: splitDocumentOperator}
var getVariableOpType = &operationType{Type: "GET_VARIABLE", NumArgs: 0, Precedence: 55, Handler: getVariableOperator}
var getStyleOpType = &operationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: getStyleOperator}

52
pkg/yqlib/operator_entries.go
View File

@ -3,22 +3,23 @@ package yqlib
import (
"container/list"
"fmt"
yaml "gopkg.in/yaml.v3"
)
func entrySeqFor(key *yaml.Node, value *yaml.Node) *yaml.Node {
var keyKey = &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "key"}
var valueKey = &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "value"}
var keyKey = &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "key"}
var valueKey = &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "value"}
return &yaml.Node{
Kind: yaml.MappingNode,
Tag: "!!map",
Kind: yaml.MappingNode,
Tag: "!!map",
Content: []*yaml.Node{keyKey, key, valueKey, value},
}
}
func toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode {
var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
var entriesNode = candidateNode.CreateChild(nil, sequence)
var contents = unwrapDoc(candidateNode.Node).Content
@ -32,7 +33,7 @@ func toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode {
}
func toEntriesfromSeq(candidateNode *CandidateNode) *CandidateNode {
var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
var entriesNode = candidateNode.CreateChild(nil, sequence)
var contents = unwrapDoc(candidateNode.Node).Content
@ -89,8 +90,8 @@ func parseEntry(d *dataTreeNavigator, entry *yaml.Node, position int) (*yaml.Nod
}
func fromEntries(d *dataTreeNavigator, candidateNode * CandidateNode) (*CandidateNode, error) {
var node = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
func fromEntries(d *dataTreeNavigator, candidateNode *CandidateNode) (*CandidateNode, error) {
var node = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
var mapCandidateNode = candidateNode.CreateChild(nil, node)
var contents = unwrapDoc(candidateNode.Node).Content
@ -114,7 +115,7 @@ func fromEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *
switch candidateNode.Kind {
case yaml.SequenceNode:
mapResult, err :=fromEntries(d, candidate)
mapResult, err := fromEntries(d, candidate)
if err != nil {
return Context{}, err
}
@ -125,4 +126,35 @@ func fromEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *
}
return context.ChildContext(results), nil
}
}
func withEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
//to_entries on the context
toEntries, err := toEntriesOperator(d, context, expressionNode)
if err != nil {
return Context{}, nil
}
//run expression against entries
// splat toEntries and pipe it into Rhs
splatted, err := splat(d, toEntries, traversePreferences{})
if err != nil {
return Context{}, nil
}
result, err := d.GetMatchingNodes(splatted, expressionNode.Rhs)
log.Debug("expressionNode.Rhs %v", expressionNode.Rhs.Operation.OperationType)
log.Debug("result %v", result)
if err != nil {
return Context{}, nil
}
collected, err := collectOperator(d, result, expressionNode)
if err != nil {
return Context{}, nil
}
//from_entries on the result
return fromEntriesOperator(d, collected, expressionNode)
}

26
pkg/yqlib/operator_entries_test.go
View File

@ -7,37 +7,45 @@ import (
var entriesOperatorScenarios = []expressionScenario{
{
description: "to_entries Map",
document: `{a: 1, b: 2}`,
expression: `to_entries`,
document: `{a: 1, b: 2}`,
expression: `to_entries`,
expected: []string{
"D0, P[], (!!seq)::- key: a\n value: 1\n- key: b\n value: 2\n",
},
},
{
description: "to_entries Array",
document: `[a, b]`,
expression: `to_entries`,
document: `[a, b]`,
expression: `to_entries`,
expected: []string{
"D0, P[], (!!seq)::- key: 0\n value: a\n- key: 1\n value: b\n",
},
},
{
description: "from_entries map",
document: `{a: 1, b: 2}`,
expression: `to_entries | from_entries`,
document: `{a: 1, b: 2}`,
expression: `to_entries | from_entries`,
expected: []string{
"D0, P[], (!!map)::a: 1\nb: 2\n",
},
},
{
description: "from_entries with numeric key indexes",
description: "from_entries with numeric key indexes",
subdescription: "from_entries always creates a map, even for numeric keys",
document: `[a,b]`,
expression: `to_entries | from_entries`,
document: `[a,b]`,
expression: `to_entries | from_entries`,
expected: []string{
"D0, P[], (!!map)::0: a\n1: b\n",
},
},
{
description: "Use with_entries to update keys",
document: `{a: 1, b: 2}`,
expression: `with_entries(.key |= "KEY_" + .)`,
expected: []string{
"D0, P[], (!!map)::KEY_a: 1\nKEY_b: 2\n",
},
},
}
func TestEntriesOperatorScenarios(t *testing.T) {

14
pkg/yqlib/operator_traverse_path.go
View File

@ -14,6 +14,7 @@ type traversePreferences struct {
IncludeMapKeys bool
DontAutoCreate bool // by default, we automatically create entries on the fly.
DontIncludeMapValues bool
OptionalTraverse bool // e.g. .adf?
}
func splat(d *dataTreeNavigator, context Context, prefs traversePreferences) (Context, error) {
@ -60,7 +61,7 @@ func traverse(d *dataTreeNavigator, context Context, matchingNode *CandidateNode
case yaml.SequenceNode:
log.Debug("its a sequence of %v things!", len(value.Content))
return traverseArray(matchingNode, operation)
return traverseArray(matchingNode, operation, operation.Preferences.(traversePreferences))
case yaml.AliasNode:
log.Debug("its an alias!")
@ -130,7 +131,7 @@ func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesT
matchingNode.Node = node.Alias
return traverseArrayIndices(context, matchingNode, indicesToTraverse, prefs)
} else if node.Kind == yaml.SequenceNode {
return traverseArrayWithIndices(matchingNode, indicesToTraverse)
return traverseArrayWithIndices(matchingNode, indicesToTraverse, prefs)
} else if node.Kind == yaml.MappingNode {
return traverseMapWithIndices(context, matchingNode, indicesToTraverse, prefs)
} else if node.Kind == yaml.DocumentNode {
@ -159,7 +160,7 @@ func traverseMapWithIndices(context Context, candidate *CandidateNode, indices [
return matchingNodeMap, nil
}
func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node) (*list.List, error) {
func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node, prefs traversePreferences) (*list.List, error) {
log.Debug("traverseArrayWithIndices")
var newMatches = list.New()
node := unwrapDoc(candidate.Node)
@ -177,6 +178,9 @@ func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node) (*
for _, indexNode := range indices {
log.Debug("traverseArrayWithIndices: '%v'", indexNode.Value)
index, err := strconv.ParseInt(indexNode.Value, 10, 64)
if err != nil && prefs.OptionalTraverse {
continue
}
if err != nil {
return nil, fmt.Errorf("Cannot index array with '%v' (%v)", indexNode.Value, err)
}
@ -297,8 +301,8 @@ func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *C
return nil
}
func traverseArray(candidate *CandidateNode, operation *Operation) (*list.List, error) {
func traverseArray(candidate *CandidateNode, operation *Operation, prefs traversePreferences) (*list.List, error) {
log.Debug("operation Value %v", operation.Value)
indices := []*yaml.Node{&yaml.Node{Value: operation.StringValue}}
return traverseArrayWithIndices(candidate, indices)
return traverseArrayWithIndices(candidate, indices, prefs)
}

22
pkg/yqlib/operator_traverse_path_test.go
View File

@ -45,6 +45,14 @@ var traversePathOperatorScenarios = []expressionScenario{
"D0, P[1], (!!map)::{c: banana}\n",
},
},
// {
// description: "Optional Splat",
// subdescription: "Just like splat, but won't error if you run it against scalars",
// document: `"cat"`,
// expression: `.[]?`,
// expected: []string{
// },
// },
{
description: "Special characters",
subdescription: "Use quotes with brackets around path elements with special characters",
@ -97,6 +105,20 @@ var traversePathOperatorScenarios = []expressionScenario{
"D0, P[a b], (!!null)::null\n",
},
},
{
description: "Optional identifier",
subdescription: "Like jq, does not output an error when the yaml is not an array or object as expected",
document: `[1,2,3]`,
expression: `.a?`,
expected: []string{},
},
// {
// skipDoc: true,
// document: `[1,2,3]`,
// expression: `.["a"]?`,
// expected: []string{
// },
// },
{
skipDoc: true,
document: ``,