Added with_entries

pkg/yqlib/doc/Entries.md

@@ -1,4 +1,4 @@
-
+Similar to the same named functions in `jq` these functions convert to/from an object and an array of key-value pairs. This is most useful for performing operations on keys of maps.
 ## to_entries Map
 Given a sample.yml file of:
 ```yaml
@@ -69,3 +69,19 @@ will output
 1: b
 ```
 
+## Use with_entries to update keys
+Given a sample.yml file of:
+```yaml
+a: 1
+b: 2
+```
+then
+```bash
+yq eval 'with_entries(.key |= "KEY_" + .)' sample.yml
+```
+will output
+```yaml
+KEY_a: 1
+KEY_b: 2
+```
+

pkg/yqlib/doc/Traverse (Read).md

@@ -98,6 +98,23 @@ will output
 null
 ```
 
+## Optional identifier
+Like jq, does not output an error when the yaml is not an array or object as expected
+
+Given a sample.yml file of:
+```yaml
+- 1
+- 2
+- 3
+```
+then
+```bash
+yq eval '.a?' sample.yml
+```
+will output
+```yaml
+```
+
 ## Wildcard matching
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/headers/Entries.md

@@ -0,0 +1 @@
+Similar to the same named functions in `jq` these functions convert to/from an object and an array of key-value pairs. This is most useful for performing operations on keys of maps.

pkg/yqlib/expression_tokeniser.go

@@ -63,12 +63,19 @@ func (t *token) toString(detail bool) string {
 func pathToken(wrapped bool) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		value := string(m.Bytes)
+		prefs := traversePreferences{}
+
+		if value[len(value)-1:] == "?" {
+			prefs.OptionalTraverse = true
+			value = value[:len(value)-1]
+		}
+
 		value = value[1:]
 		if wrapped {
 			value = unwrap(value)
 		}
 		log.Debug("PathToken %v", value)
-		op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: traversePreferences{}}
+		op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: prefs}
 		return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
 	}
 }
@@ -280,6 +287,7 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`path`), opToken(getPathOpType))
 	lexer.Add([]byte(`to_entries`), opToken(toEntriesOpType))
 	lexer.Add([]byte(`from_entries`), opToken(fromEntriesOpType))
+	lexer.Add([]byte(`with_entries`), opToken(withEntriesOpType))
 
 	lexer.Add([]byte(`lineComment`), opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}))
 
@@ -302,8 +310,8 @@ func initLexer() (*lex.Lexer, error) {
 
 	lexer.Add([]byte("( |\t|\n|\r)+"), skip)
 
-	lexer.Add([]byte(`\."[^ "]+"`), pathToken(true))
+	lexer.Add([]byte(`\."[^ "]+"\??`), pathToken(true))
-	lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=]+`), pathToken(false))
+	lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=]+\??`), pathToken(false))
 	lexer.Add([]byte(`\.`), selfToken())
 
 	lexer.Add([]byte(`\|`), opToken(pipeOpType))

pkg/yqlib/lib.go

@@ -60,8 +60,11 @@ var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence:
 
 var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator}
 var collectOpType = &operationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: collectOperator}
+
 var toEntriesOpType = &operationType{Type: "TO_ENTRIES", NumArgs: 0, Precedence: 50, Handler: toEntriesOperator}
-var fromEntriesOpType = &operationType{Type: "TO_ENTRIES", NumArgs: 0, Precedence: 50, Handler: fromEntriesOperator}
+var fromEntriesOpType = &operationType{Type: "FROM_ENTRIES", NumArgs: 0, Precedence: 50, Handler: fromEntriesOperator}
+var withEntriesOpType = &operationType{Type: "WITH_ENTRIES", NumArgs: 1, Precedence: 50, Handler: withEntriesOperator}
+
 var splitDocumentOpType = &operationType{Type: "SPLIT_DOC", NumArgs: 0, Precedence: 50, Handler: splitDocumentOperator}
 var getVariableOpType = &operationType{Type: "GET_VARIABLE", NumArgs: 0, Precedence: 55, Handler: getVariableOperator}
 var getStyleOpType = &operationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: getStyleOperator}

pkg/yqlib/operator_entries.go

@@ -3,22 +3,23 @@ package yqlib
 import (
 	"container/list"
 	"fmt"
+
 	yaml "gopkg.in/yaml.v3"
 )
 
 func entrySeqFor(key *yaml.Node, value *yaml.Node) *yaml.Node {
-	var keyKey = &yaml.Node{Kind:  yaml.ScalarNode, Tag: "!!str", Value: "key"}
+	var keyKey = &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "key"}
-	var valueKey = &yaml.Node{Kind:  yaml.ScalarNode, Tag: "!!str", Value: "value"}
+	var valueKey = &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: "value"}
 
 	return &yaml.Node{
-		Kind:  yaml.MappingNode, 
+		Kind:    yaml.MappingNode,
-		Tag: "!!map", 
+		Tag:     "!!map",
 		Content: []*yaml.Node{keyKey, key, valueKey, value},
 	}
 }
 
 func toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode {
-	var sequence = &yaml.Node{Kind:  yaml.SequenceNode, Tag: "!!seq"}
+	var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
 	var entriesNode = candidateNode.CreateChild(nil, sequence)
 
 	var contents = unwrapDoc(candidateNode.Node).Content
@@ -32,7 +33,7 @@ func toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode {
 }
 
 func toEntriesfromSeq(candidateNode *CandidateNode) *CandidateNode {
-	var sequence = &yaml.Node{Kind:  yaml.SequenceNode, Tag: "!!seq"}
+	var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
 	var entriesNode = candidateNode.CreateChild(nil, sequence)
 
 	var contents = unwrapDoc(candidateNode.Node).Content
@@ -89,8 +90,8 @@ func parseEntry(d *dataTreeNavigator, entry *yaml.Node, position int) (*yaml.Nod
 
 }
 
-func fromEntries(d *dataTreeNavigator, candidateNode * CandidateNode) (*CandidateNode, error) {
+func fromEntries(d *dataTreeNavigator, candidateNode *CandidateNode) (*CandidateNode, error) {
-	var node = &yaml.Node{Kind:  yaml.MappingNode, Tag: "!!map"}
+	var node = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
 	var mapCandidateNode = candidateNode.CreateChild(nil, node)
 
 	var contents = unwrapDoc(candidateNode.Node).Content
@@ -114,7 +115,7 @@ func fromEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *
 
 		switch candidateNode.Kind {
 		case yaml.SequenceNode:
-			mapResult, err :=fromEntries(d, candidate) 
+			mapResult, err := fromEntries(d, candidate)
 			if err != nil {
 				return Context{}, err
 			}
@@ -125,4 +126,35 @@ func fromEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *
 	}
 
 	return context.ChildContext(results), nil
-}
+}
+
+func withEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	//to_entries on the context
+	toEntries, err := toEntriesOperator(d, context, expressionNode)
+	if err != nil {
+		return Context{}, nil
+	}
+
+	//run expression against entries
+	// splat toEntries and pipe it into Rhs
+	splatted, err := splat(d, toEntries, traversePreferences{})
+	if err != nil {
+		return Context{}, nil
+	}
+
+	result, err := d.GetMatchingNodes(splatted, expressionNode.Rhs)
+	log.Debug("expressionNode.Rhs %v", expressionNode.Rhs.Operation.OperationType)
+	log.Debug("result %v", result)
+	if err != nil {
+		return Context{}, nil
+	}
+
+	collected, err := collectOperator(d, result, expressionNode)
+	if err != nil {
+		return Context{}, nil
+	}
+
+	//from_entries on the result
+	return fromEntriesOperator(d, collected, expressionNode)
+}

pkg/yqlib/operator_entries_test.go

@@ -7,37 +7,45 @@ import (
 var entriesOperatorScenarios = []expressionScenario{
 	{
 		description: "to_entries Map",
-		document:   `{a: 1, b: 2}`,
+		document:    `{a: 1, b: 2}`,
-		expression: `to_entries`,
+		expression:  `to_entries`,
 		expected: []string{
 			"D0, P[], (!!seq)::- key: a\n  value: 1\n- key: b\n  value: 2\n",
 		},
 	},
 	{
 		description: "to_entries Array",
-		document:   `[a, b]`,
+		document:    `[a, b]`,
-		expression: `to_entries`,
+		expression:  `to_entries`,
 		expected: []string{
 			"D0, P[], (!!seq)::- key: 0\n  value: a\n- key: 1\n  value: b\n",
 		},
 	},
 	{
 		description: "from_entries map",
-		document:   `{a: 1, b: 2}`,
+		document:    `{a: 1, b: 2}`,
-		expression: `to_entries | from_entries`,
+		expression:  `to_entries | from_entries`,
 		expected: []string{
 			"D0, P[], (!!map)::a: 1\nb: 2\n",
 		},
 	},
 	{
-		description: "from_entries with numeric key indexes",
+		description:    "from_entries with numeric key indexes",
 		subdescription: "from_entries always creates a map, even for numeric keys",
-		document:   `[a,b]`,
+		document:       `[a,b]`,
-		expression: `to_entries | from_entries`,
+		expression:     `to_entries | from_entries`,
 		expected: []string{
 			"D0, P[], (!!map)::0: a\n1: b\n",
 		},
 	},
+	{
+		description: "Use with_entries to update keys",
+		document:    `{a: 1, b: 2}`,
+		expression:  `with_entries(.key |= "KEY_" + .)`,
+		expected: []string{
+			"D0, P[], (!!map)::KEY_a: 1\nKEY_b: 2\n",
+		},
+	},
 }
 
 func TestEntriesOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_traverse_path.go

@@ -14,6 +14,7 @@ type traversePreferences struct {
 	IncludeMapKeys       bool
 	DontAutoCreate       bool // by default, we automatically create entries on the fly.
 	DontIncludeMapValues bool
+	OptionalTraverse     bool // e.g. .adf?
 }
 
 func splat(d *dataTreeNavigator, context Context, prefs traversePreferences) (Context, error) {
@@ -60,7 +61,7 @@ func traverse(d *dataTreeNavigator, context Context, matchingNode *CandidateNode
 
 	case yaml.SequenceNode:
 		log.Debug("its a sequence of %v things!", len(value.Content))
-		return traverseArray(matchingNode, operation)
+		return traverseArray(matchingNode, operation, operation.Preferences.(traversePreferences))
 
 	case yaml.AliasNode:
 		log.Debug("its an alias!")
@@ -130,7 +131,7 @@ func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesT
 		matchingNode.Node = node.Alias
 		return traverseArrayIndices(context, matchingNode, indicesToTraverse, prefs)
 	} else if node.Kind == yaml.SequenceNode {
-		return traverseArrayWithIndices(matchingNode, indicesToTraverse)
+		return traverseArrayWithIndices(matchingNode, indicesToTraverse, prefs)
 	} else if node.Kind == yaml.MappingNode {
 		return traverseMapWithIndices(context, matchingNode, indicesToTraverse, prefs)
 	} else if node.Kind == yaml.DocumentNode {
@@ -159,7 +160,7 @@ func traverseMapWithIndices(context Context, candidate *CandidateNode, indices [
 	return matchingNodeMap, nil
 }
 
-func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node) (*list.List, error) {
+func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node, prefs traversePreferences) (*list.List, error) {
 	log.Debug("traverseArrayWithIndices")
 	var newMatches = list.New()
 	node := unwrapDoc(candidate.Node)
@@ -177,6 +178,9 @@ func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node) (*
 	for _, indexNode := range indices {
 		log.Debug("traverseArrayWithIndices: '%v'", indexNode.Value)
 		index, err := strconv.ParseInt(indexNode.Value, 10, 64)
+		if err != nil && prefs.OptionalTraverse {
+			continue
+		}
 		if err != nil {
 			return nil, fmt.Errorf("Cannot index array with '%v' (%v)", indexNode.Value, err)
 		}
@@ -297,8 +301,8 @@ func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *C
 	return nil
 }
 
-func traverseArray(candidate *CandidateNode, operation *Operation) (*list.List, error) {
+func traverseArray(candidate *CandidateNode, operation *Operation, prefs traversePreferences) (*list.List, error) {
 	log.Debug("operation Value %v", operation.Value)
 	indices := []*yaml.Node{&yaml.Node{Value: operation.StringValue}}
-	return traverseArrayWithIndices(candidate, indices)
+	return traverseArrayWithIndices(candidate, indices, prefs)
 }

pkg/yqlib/operator_traverse_path_test.go

@@ -45,6 +45,14 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[1], (!!map)::{c: banana}\n",
 		},
 	},
+	// {
+	// 	description:    "Optional Splat",
+	// 	subdescription: "Just like splat, but won't error if you run it against scalars",
+	// 	document:       `"cat"`,
+	// 	expression:     `.[]?`,
+	// 	expected: []string{
+	// 	},
+	// },
 	{
 		description:    "Special characters",
 		subdescription: "Use quotes with brackets around path elements with special characters",
@@ -97,6 +105,20 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[a b], (!!null)::null\n",
 		},
 	},
+	{
+		description:    "Optional identifier",
+		subdescription: "Like jq, does not output an error when the yaml is not an array or object as expected",
+		document:       `[1,2,3]`,
+		expression:     `.a?`,
+		expected:       []string{},
+	},
+	// {
+	// 	skipDoc: true,
+	// 	document:       `[1,2,3]`,
+	// 	expression:     `.["a"]?`,
+	// 	expected: []string{
+	// 	},
+	// },
 	{
 		skipDoc:    true,
 		document:   ``,