From cc08afc4358f409c9b6b5b38aafd76c40c2f952a Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sun, 9 May 2021 15:12:50 +1000 Subject: [PATCH] Added with_entries --- pkg/yqlib/doc/Entries.md | 18 +++++++- pkg/yqlib/doc/Traverse (Read).md | 17 ++++++++ pkg/yqlib/doc/headers/Entries.md | 1 + pkg/yqlib/expression_tokeniser.go | 14 +++++-- pkg/yqlib/lib.go | 5 ++- pkg/yqlib/operator_entries.go | 52 +++++++++++++++++++----- pkg/yqlib/operator_entries_test.go | 26 ++++++++---- pkg/yqlib/operator_traverse_path.go | 14 ++++--- pkg/yqlib/operator_traverse_path_test.go | 22 ++++++++++ 9 files changed, 140 insertions(+), 29 deletions(-) create mode 100644 pkg/yqlib/doc/headers/Entries.md diff --git a/pkg/yqlib/doc/Entries.md b/pkg/yqlib/doc/Entries.md index 1acfbddf..9bb98726 100644 --- a/pkg/yqlib/doc/Entries.md +++ b/pkg/yqlib/doc/Entries.md @@ -1,4 +1,4 @@ - +Similar to the same named functions in `jq` these functions convert to/from an object and an array of key-value pairs. This is most useful for performing operations on keys of maps. ## to_entries Map Given a sample.yml file of: ```yaml @@ -69,3 +69,19 @@ will output 1: b ``` +## Use with_entries to update keys +Given a sample.yml file of: +```yaml +a: 1 +b: 2 +``` +then +```bash +yq eval 'with_entries(.key |= "KEY_" + .)' sample.yml +``` +will output +```yaml +KEY_a: 1 +KEY_b: 2 +``` + diff --git a/pkg/yqlib/doc/Traverse (Read).md b/pkg/yqlib/doc/Traverse (Read).md index c8b58905..f5acb2d7 100644 --- a/pkg/yqlib/doc/Traverse (Read).md +++ b/pkg/yqlib/doc/Traverse (Read).md @@ -98,6 +98,23 @@ will output null ``` +## Optional identifier +Like jq, does not output an error when the yaml is not an array or object as expected + +Given a sample.yml file of: +```yaml +- 1 +- 2 +- 3 +``` +then +```bash +yq eval '.a?' sample.yml +``` +will output +```yaml +``` + ## Wildcard matching Given a sample.yml file of: ```yaml diff --git a/pkg/yqlib/doc/headers/Entries.md b/pkg/yqlib/doc/headers/Entries.md new file mode 100644 index 00000000..f13824be --- /dev/null +++ b/pkg/yqlib/doc/headers/Entries.md @@ -0,0 +1 @@ +Similar to the same named functions in `jq` these functions convert to/from an object and an array of key-value pairs. This is most useful for performing operations on keys of maps. \ No newline at end of file diff --git a/pkg/yqlib/expression_tokeniser.go b/pkg/yqlib/expression_tokeniser.go index 3b6c7e81..64e672c7 100644 --- a/pkg/yqlib/expression_tokeniser.go +++ b/pkg/yqlib/expression_tokeniser.go @@ -63,12 +63,19 @@ func (t *token) toString(detail bool) string { func pathToken(wrapped bool) lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { value := string(m.Bytes) + prefs := traversePreferences{} + + if value[len(value)-1:] == "?" { + prefs.OptionalTraverse = true + value = value[:len(value)-1] + } + value = value[1:] if wrapped { value = unwrap(value) } log.Debug("PathToken %v", value) - op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: traversePreferences{}} + op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: prefs} return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil } } @@ -280,6 +287,7 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`path`), opToken(getPathOpType)) lexer.Add([]byte(`to_entries`), opToken(toEntriesOpType)) lexer.Add([]byte(`from_entries`), opToken(fromEntriesOpType)) + lexer.Add([]byte(`with_entries`), opToken(withEntriesOpType)) lexer.Add([]byte(`lineComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true})) @@ -302,8 +310,8 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte("( |\t|\n|\r)+"), skip) - lexer.Add([]byte(`\."[^ "]+"`), pathToken(true)) - lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=]+`), pathToken(false)) + lexer.Add([]byte(`\."[^ "]+"\??`), pathToken(true)) + lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=]+\??`), pathToken(false)) lexer.Add([]byte(`\.`), selfToken()) lexer.Add([]byte(`\|`), opToken(pipeOpType)) diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 0a64ecc8..bf88329a 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -60,8 +60,11 @@ var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence: var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator} var collectOpType = &operationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: collectOperator} + var toEntriesOpType = &operationType{Type: "TO_ENTRIES", NumArgs: 0, Precedence: 50, Handler: toEntriesOperator} -var fromEntriesOpType = &operationType{Type: "TO_ENTRIES", NumArgs: 0, Precedence: 50, Handler: fromEntriesOperator} +var fromEntriesOpType = &operationType{Type: "FROM_ENTRIES", NumArgs: 0, Precedence: 50, Handler: fromEntriesOperator} +var withEntriesOpType = &operationType{Type: "WITH_ENTRIES", NumArgs: 1, Precedence: 50, Handler: withEntriesOperator} + var splitDocumentOpType = &operationType{Type: "SPLIT_DOC", NumArgs: 0, Precedence: 50, Handler: splitDocumentOperator} var getVariableOpType = &operationType{Type: "GET_VARIABLE", NumArgs: 0, Precedence: 55, Handler: getVariableOperator} var getStyleOpType = &operationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: getStyleOperator} diff --git a/pkg/yqlib/operator_entries.go b/pkg/yqlib/operator_entries.go index e166ed6e..ba9ea669 100644 --- a/pkg/yqlib/operator_entries.go +++ b/pkg/yqlib/operator_entries.go @@ -3,22 +3,23 @@ package yqlib import ( "container/list" "fmt" + yaml "gopkg.in/yaml.v3" ) func entrySeqFor(key *yaml.Node, value *yaml.Node) *yaml.Node { - var keyKey = &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "key"} - var valueKey = &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "value"} + var keyKey = &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "key"} + var valueKey = &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "value"} return &yaml.Node{ - Kind: yaml.MappingNode, - Tag: "!!map", + Kind: yaml.MappingNode, + Tag: "!!map", Content: []*yaml.Node{keyKey, key, valueKey, value}, } } func toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode { - var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} + var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} var entriesNode = candidateNode.CreateChild(nil, sequence) var contents = unwrapDoc(candidateNode.Node).Content @@ -32,7 +33,7 @@ func toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode { } func toEntriesfromSeq(candidateNode *CandidateNode) *CandidateNode { - var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} + var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} var entriesNode = candidateNode.CreateChild(nil, sequence) var contents = unwrapDoc(candidateNode.Node).Content @@ -89,8 +90,8 @@ func parseEntry(d *dataTreeNavigator, entry *yaml.Node, position int) (*yaml.Nod } -func fromEntries(d *dataTreeNavigator, candidateNode * CandidateNode) (*CandidateNode, error) { - var node = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} +func fromEntries(d *dataTreeNavigator, candidateNode *CandidateNode) (*CandidateNode, error) { + var node = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} var mapCandidateNode = candidateNode.CreateChild(nil, node) var contents = unwrapDoc(candidateNode.Node).Content @@ -114,7 +115,7 @@ func fromEntriesOperator(d *dataTreeNavigator, context Context, expressionNode * switch candidateNode.Kind { case yaml.SequenceNode: - mapResult, err :=fromEntries(d, candidate) + mapResult, err := fromEntries(d, candidate) if err != nil { return Context{}, err } @@ -125,4 +126,35 @@ func fromEntriesOperator(d *dataTreeNavigator, context Context, expressionNode * } return context.ChildContext(results), nil -} \ No newline at end of file +} + +func withEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { + + //to_entries on the context + toEntries, err := toEntriesOperator(d, context, expressionNode) + if err != nil { + return Context{}, nil + } + + //run expression against entries + // splat toEntries and pipe it into Rhs + splatted, err := splat(d, toEntries, traversePreferences{}) + if err != nil { + return Context{}, nil + } + + result, err := d.GetMatchingNodes(splatted, expressionNode.Rhs) + log.Debug("expressionNode.Rhs %v", expressionNode.Rhs.Operation.OperationType) + log.Debug("result %v", result) + if err != nil { + return Context{}, nil + } + + collected, err := collectOperator(d, result, expressionNode) + if err != nil { + return Context{}, nil + } + + //from_entries on the result + return fromEntriesOperator(d, collected, expressionNode) +} diff --git a/pkg/yqlib/operator_entries_test.go b/pkg/yqlib/operator_entries_test.go index 1bb4755d..eaa49af4 100644 --- a/pkg/yqlib/operator_entries_test.go +++ b/pkg/yqlib/operator_entries_test.go @@ -7,37 +7,45 @@ import ( var entriesOperatorScenarios = []expressionScenario{ { description: "to_entries Map", - document: `{a: 1, b: 2}`, - expression: `to_entries`, + document: `{a: 1, b: 2}`, + expression: `to_entries`, expected: []string{ "D0, P[], (!!seq)::- key: a\n value: 1\n- key: b\n value: 2\n", }, }, { description: "to_entries Array", - document: `[a, b]`, - expression: `to_entries`, + document: `[a, b]`, + expression: `to_entries`, expected: []string{ "D0, P[], (!!seq)::- key: 0\n value: a\n- key: 1\n value: b\n", }, }, { description: "from_entries map", - document: `{a: 1, b: 2}`, - expression: `to_entries | from_entries`, + document: `{a: 1, b: 2}`, + expression: `to_entries | from_entries`, expected: []string{ "D0, P[], (!!map)::a: 1\nb: 2\n", }, }, { - description: "from_entries with numeric key indexes", + description: "from_entries with numeric key indexes", subdescription: "from_entries always creates a map, even for numeric keys", - document: `[a,b]`, - expression: `to_entries | from_entries`, + document: `[a,b]`, + expression: `to_entries | from_entries`, expected: []string{ "D0, P[], (!!map)::0: a\n1: b\n", }, }, + { + description: "Use with_entries to update keys", + document: `{a: 1, b: 2}`, + expression: `with_entries(.key |= "KEY_" + .)`, + expected: []string{ + "D0, P[], (!!map)::KEY_a: 1\nKEY_b: 2\n", + }, + }, } func TestEntriesOperatorScenarios(t *testing.T) { diff --git a/pkg/yqlib/operator_traverse_path.go b/pkg/yqlib/operator_traverse_path.go index b64bcb94..db24ac80 100644 --- a/pkg/yqlib/operator_traverse_path.go +++ b/pkg/yqlib/operator_traverse_path.go @@ -14,6 +14,7 @@ type traversePreferences struct { IncludeMapKeys bool DontAutoCreate bool // by default, we automatically create entries on the fly. DontIncludeMapValues bool + OptionalTraverse bool // e.g. .adf? } func splat(d *dataTreeNavigator, context Context, prefs traversePreferences) (Context, error) { @@ -60,7 +61,7 @@ func traverse(d *dataTreeNavigator, context Context, matchingNode *CandidateNode case yaml.SequenceNode: log.Debug("its a sequence of %v things!", len(value.Content)) - return traverseArray(matchingNode, operation) + return traverseArray(matchingNode, operation, operation.Preferences.(traversePreferences)) case yaml.AliasNode: log.Debug("its an alias!") @@ -130,7 +131,7 @@ func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesT matchingNode.Node = node.Alias return traverseArrayIndices(context, matchingNode, indicesToTraverse, prefs) } else if node.Kind == yaml.SequenceNode { - return traverseArrayWithIndices(matchingNode, indicesToTraverse) + return traverseArrayWithIndices(matchingNode, indicesToTraverse, prefs) } else if node.Kind == yaml.MappingNode { return traverseMapWithIndices(context, matchingNode, indicesToTraverse, prefs) } else if node.Kind == yaml.DocumentNode { @@ -159,7 +160,7 @@ func traverseMapWithIndices(context Context, candidate *CandidateNode, indices [ return matchingNodeMap, nil } -func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node) (*list.List, error) { +func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node, prefs traversePreferences) (*list.List, error) { log.Debug("traverseArrayWithIndices") var newMatches = list.New() node := unwrapDoc(candidate.Node) @@ -177,6 +178,9 @@ func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node) (* for _, indexNode := range indices { log.Debug("traverseArrayWithIndices: '%v'", indexNode.Value) index, err := strconv.ParseInt(indexNode.Value, 10, 64) + if err != nil && prefs.OptionalTraverse { + continue + } if err != nil { return nil, fmt.Errorf("Cannot index array with '%v' (%v)", indexNode.Value, err) } @@ -297,8 +301,8 @@ func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *C return nil } -func traverseArray(candidate *CandidateNode, operation *Operation) (*list.List, error) { +func traverseArray(candidate *CandidateNode, operation *Operation, prefs traversePreferences) (*list.List, error) { log.Debug("operation Value %v", operation.Value) indices := []*yaml.Node{&yaml.Node{Value: operation.StringValue}} - return traverseArrayWithIndices(candidate, indices) + return traverseArrayWithIndices(candidate, indices, prefs) } diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go index e7d0a73c..f6869461 100644 --- a/pkg/yqlib/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -45,6 +45,14 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[1], (!!map)::{c: banana}\n", }, }, + // { + // description: "Optional Splat", + // subdescription: "Just like splat, but won't error if you run it against scalars", + // document: `"cat"`, + // expression: `.[]?`, + // expected: []string{ + // }, + // }, { description: "Special characters", subdescription: "Use quotes with brackets around path elements with special characters", @@ -97,6 +105,20 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[a b], (!!null)::null\n", }, }, + { + description: "Optional identifier", + subdescription: "Like jq, does not output an error when the yaml is not an array or object as expected", + document: `[1,2,3]`, + expression: `.a?`, + expected: []string{}, + }, + // { + // skipDoc: true, + // document: `[1,2,3]`, + // expression: `.["a"]?`, + // expected: []string{ + // }, + // }, { skipDoc: true, document: ``,