diff --git a/pkg/yqlib/doc/Multiply.md b/pkg/yqlib/doc/Multiply.md index 87ea2d89..cd872c86 100644 --- a/pkg/yqlib/doc/Multiply.md +++ b/pkg/yqlib/doc/Multiply.md @@ -111,6 +111,26 @@ b: - 5 ``` +## Merge, only existing fields +Given a sample.yml file of: +```yaml +a: + thing: one + cat: frog +b: + missing: two + thing: two +``` +then +```bash +yq eval '.a *? .b' sample.yml +``` +will output +```yaml +thing: two +cat: frog +``` + ## Merge, appending arrays Given a sample.yml file of: ```yaml @@ -143,6 +163,33 @@ array: value: banana ``` +## Merge, only existing fields, appending arrays +Given a sample.yml file of: +```yaml +a: + thing: + - 1 + - 2 +b: + thing: + - 3 + - 4 + another: + - 1 +``` +then +```bash +yq eval '.a *?+ .b' sample.yml +``` +will output +```yaml +thing: + - 1 + - 2 + - 3 + - 4 +``` + ## Merge to prefix an element Given a sample.yml file of: ```yaml diff --git a/pkg/yqlib/expression_processing_test.go b/pkg/yqlib/expression_processing_test.go index 45226b70..b4d4e373 100644 --- a/pkg/yqlib/expression_processing_test.go +++ b/pkg/yqlib/expression_processing_test.go @@ -52,11 +52,6 @@ var pathTests = []struct { append(make([]interface{}, 0), "[", "3 (int64)", "]"), append(make([]interface{}, 0), "3 (int64)", "COLLECT", "SHORT_PIPE"), }, - { - `d0.a`, - append(make([]interface{}, 0), "d0", "SHORT_PIPE", "a"), - append(make([]interface{}, 0), "d0", "a", "SHORT_PIPE"), - }, { `.a | .[].b == "apple"`, append(make([]interface{}, 0), "a", "PIPE", "TRAVERSE_ARRAY", "[", "]", "SHORT_PIPE", "b", "EQUALS", "apple (string)"), diff --git a/pkg/yqlib/expression_tokeniser.go b/pkg/yqlib/expression_tokeniser.go index 1c317bc2..9cdec47c 100644 --- a/pkg/yqlib/expression_tokeniser.go +++ b/pkg/yqlib/expression_tokeniser.go @@ -3,6 +3,7 @@ package yqlib import ( "fmt" "strconv" + "strings" lex "github.com/timtadh/lexmachine" "github.com/timtadh/lexmachine/machines" @@ -65,21 +66,7 @@ func pathToken(wrapped bool) lex.Action { value = unwrap(value) } log.Debug("PathToken %v", value) - op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value} - return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil - } -} - -func documentToken() lex.Action { - return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { - var numberString = string(m.Bytes) - numberString = numberString[1:] - var number, errParsingInt = strconv.ParseInt(numberString, 10, 64) // nolint - if errParsingInt != nil { - return nil, errParsingInt - } - log.Debug("documentToken %v", string(m.Bytes)) - op := &Operation{OperationType: documentFilterOpType, Value: number, StringValue: numberString} + op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: traversePreferences{}} return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil } } @@ -101,6 +88,21 @@ func assignOpToken(updateAssign bool) lex.Action { } } +func multiplyWithPrefs() lex.Action { + return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { + prefs := multiplyPreferences{} + options := string(m.Bytes) + if strings.Contains(options, "+") { + prefs.AppendArrays = true + } + if strings.Contains(options, "?") { + prefs.TraversePrefs = traversePreferences{DontAutoCreate: true} + } + op := &Operation{OperationType: multiplyOpType, Value: multiplyOpType.Type, StringValue: options, Preferences: &prefs} + return &token{TokenType: operationToken, Operation: op}, nil + } +} + func opTokenWithPrefs(op *operationType, assignOpType *operationType, preferences interface{}) lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { log.Debug("opTokenWithPrefs %v", string(m.Bytes)) @@ -220,10 +222,10 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`\.\[`), literalToken(traverseArrayCollect, false)) lexer.Add([]byte(`\.\.`), opTokenWithPrefs(recursiveDescentOpType, nil, &recursiveDescentPreferences{RecurseArray: true, - TraversePreferences: &traversePreferences{FollowAlias: false, IncludeMapKeys: false}})) + TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: false}})) lexer.Add([]byte(`\.\.\.`), opTokenWithPrefs(recursiveDescentOpType, nil, &recursiveDescentPreferences{RecurseArray: true, - TraversePreferences: &traversePreferences{FollowAlias: false, IncludeMapKeys: true}})) + TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: true}})) lexer.Add([]byte(`,`), opToken(unionOpType)) lexer.Add([]byte(`:\s*`), opToken(createMapOpType)) @@ -270,7 +272,6 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte("( |\t|\n|\r)+"), skip) - lexer.Add([]byte(`d[0-9]+`), documentToken()) lexer.Add([]byte(`\."[^ "]+"`), pathToken(true)) lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=]+`), pathToken(false)) lexer.Add([]byte(`\.`), selfToken()) @@ -296,7 +297,7 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`\{`), literalToken(openCollectObject, false)) lexer.Add([]byte(`\}`), literalToken(closeCollectObject, true)) lexer.Add([]byte(`\*`), opTokenWithPrefs(multiplyOpType, nil, &multiplyPreferences{AppendArrays: false})) - lexer.Add([]byte(`\*\+`), opTokenWithPrefs(multiplyOpType, nil, &multiplyPreferences{AppendArrays: true})) + lexer.Add([]byte(`\*[\+|\?]*`), multiplyWithPrefs()) lexer.Add([]byte(`\+`), opToken(addOpType)) lexer.Add([]byte(`\+=`), opToken(addAssignOpType)) diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index d5aab1f3..1994cb6c 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -68,7 +68,6 @@ var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Pre var traversePathOpType = &operationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 50, Handler: traversePathOperator} var traverseArrayOpType = &operationType{Type: "TRAVERSE_ARRAY", NumArgs: 1, Precedence: 50, Handler: traverseArrayOperator} -var documentFilterOpType = &operationType{Type: "DOCUMENT_FILTER", NumArgs: 0, Precedence: 50, Handler: traversePathOperator} var selfReferenceOpType = &operationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: selfOperator} var valueOpType = &operationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: valueOperator} var envOpType = &operationType{Type: "ENV", NumArgs: 0, Precedence: 50, Handler: envOperator} @@ -120,8 +119,6 @@ func createValueOperation(value interface{}, stringValue string) *Operation { func (p *Operation) toString() string { if p.OperationType == traversePathOpType { return fmt.Sprintf("%v", p.Value) - } else if p.OperationType == documentFilterOpType { - return fmt.Sprintf("d%v", p.Value) } else if p.OperationType == selfReferenceOpType { return "SELF" } else if p.OperationType == valueOpType { diff --git a/pkg/yqlib/operator_collect_object.go b/pkg/yqlib/operator_collect_object.go index 8be51611..c9d5fda4 100644 --- a/pkg/yqlib/operator_collect_object.go +++ b/pkg/yqlib/operator_collect_object.go @@ -60,7 +60,7 @@ func collect(d *dataTreeNavigator, aggregate *list.List, remainingMatches *list. candidate := remainingMatches.Remove(remainingMatches.Front()).(*CandidateNode) splatted, err := splat(d, nodeToMap(candidate), - &traversePreferences{FollowAlias: false, IncludeMapKeys: false}) + traversePreferences{DontFollowAlias: true, IncludeMapKeys: false}) for splatEl := splatted.Front(); splatEl != nil; splatEl = splatEl.Next() { splatEl.Value.(*CandidateNode).Path = nil diff --git a/pkg/yqlib/operator_delete.go b/pkg/yqlib/operator_delete.go index cecacaeb..c84039e6 100644 --- a/pkg/yqlib/operator_delete.go +++ b/pkg/yqlib/operator_delete.go @@ -25,7 +25,7 @@ func deleteChildOperator(d *dataTreeNavigator, matchingNodes *list.List, express deleteImmediateChildOpNode := &ExpressionNode{ Operation: deleteImmediateChildOp, - Rhs: createTraversalTree(candidate.Path[0 : len(candidate.Path)-1]), + Rhs: createTraversalTree(candidate.Path[0:len(candidate.Path)-1], traversePreferences{}), } _, err := d.GetMatchingNodes(matchingNodes, deleteImmediateChildOpNode) diff --git a/pkg/yqlib/operator_multiply.go b/pkg/yqlib/operator_multiply.go index 2428c465..35d43d6d 100644 --- a/pkg/yqlib/operator_multiply.go +++ b/pkg/yqlib/operator_multiply.go @@ -44,7 +44,8 @@ func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, expressionNod } type multiplyPreferences struct { - AppendArrays bool + AppendArrays bool + TraversePrefs traversePreferences } func multiplyOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) { @@ -59,29 +60,28 @@ func multiply(preferences *multiplyPreferences) func(d *dataTreeNavigator, lhs * log.Debugf("Multipling LHS: %v", lhs.Node.Tag) log.Debugf("- RHS: %v", rhs.Node.Tag) - shouldAppendArrays := preferences.AppendArrays - if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode || (lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) { var newBlank = lhs.CreateChild(nil, &yaml.Node{}) - var newThing, err = mergeObjects(d, newBlank, lhs, false) + var newThing, err = mergeObjects(d, newBlank, lhs, &multiplyPreferences{}) if err != nil { return nil, err } - return mergeObjects(d, newThing, rhs, shouldAppendArrays) + return mergeObjects(d, newThing, rhs, preferences) } return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag) } } -func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode, shouldAppendArrays bool) (*CandidateNode, error) { +func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode, preferences *multiplyPreferences) (*CandidateNode, error) { + shouldAppendArrays := preferences.AppendArrays var results = list.New() // shouldn't recurse arrays if appending prefs := &recursiveDescentPreferences{RecurseArray: !shouldAppendArrays, - TraversePreferences: &traversePreferences{FollowAlias: false}} + TraversePreferences: traversePreferences{DontFollowAlias: true}} err := recursiveDecent(d, results, nodeToMap(rhs), prefs) if err != nil { return nil, err @@ -93,7 +93,7 @@ func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode, } for el := results.Front(); el != nil; el = el.Next() { - err := applyAssignment(d, pathIndexToStartFrom, lhs, el.Value.(*CandidateNode), shouldAppendArrays) + err := applyAssignment(d, pathIndexToStartFrom, lhs, el.Value.(*CandidateNode), preferences) if err != nil { return nil, err } @@ -101,8 +101,8 @@ func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode, return lhs, nil } -func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode, shouldAppendArrays bool) error { - +func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode, preferences *multiplyPreferences) error { + shouldAppendArrays := preferences.AppendArrays log.Debugf("merge - applyAssignment lhs %v, rhs: %v", NodeToString(lhs), NodeToString(rhs)) lhsPath := rhs.Path[pathIndexToStartFrom:] @@ -116,7 +116,7 @@ func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *Candid } rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs} - assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath), Rhs: &ExpressionNode{Operation: rhsOp}} + assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath, preferences.TraversePrefs), Rhs: &ExpressionNode{Operation: rhsOp}} _, err := d.GetMatchingNodes(nodeToMap(lhs), assignmentOpNode) diff --git a/pkg/yqlib/operator_multiply_test.go b/pkg/yqlib/operator_multiply_test.go index 40819c3a..ebca2f13 100644 --- a/pkg/yqlib/operator_multiply_test.go +++ b/pkg/yqlib/operator_multiply_test.go @@ -107,6 +107,22 @@ b: "D0, P[a], (!!seq)::[1, 2]\n", }, }, + { + description: "Merge, only existing fields", + document: `{a: {thing: one, cat: frog}, b: {missing: two, thing: two}}`, + expression: `.a *? .b`, + expected: []string{ + "D0, P[a], (!!map)::{thing: two, cat: frog}\n", + }, + }, + { + skipDoc: true, + document: `{a: [{thing: one}], b: [{missing: two, thing: two}]}`, + expression: `.a *? .b`, + expected: []string{ + "D0, P[a], (!!seq)::[{thing: two}]\n", + }, + }, { description: "Merge, appending arrays", document: `{a: {array: [1, 2, animal: dog], value: coconut}, b: {array: [3, 4, animal: cat], value: banana}}`, @@ -115,6 +131,14 @@ b: "D0, P[a], (!!map)::{array: [1, 2, {animal: dog}, 3, 4, {animal: cat}], value: banana}\n", }, }, + { + description: "Merge, only existing fields, appending arrays", + document: `{a: {thing: [1,2]}, b: {thing: [3,4], another: [1]}}`, + expression: `.a *?+ .b`, + expected: []string{ + "D0, P[a], (!!map)::{thing: [1, 2, 3, 4]}\n", + }, + }, { description: "Merge to prefix an element", document: `{a: cat, b: dog}`, diff --git a/pkg/yqlib/operator_recursive_descent.go b/pkg/yqlib/operator_recursive_descent.go index 52257312..962e8e41 100644 --- a/pkg/yqlib/operator_recursive_descent.go +++ b/pkg/yqlib/operator_recursive_descent.go @@ -7,7 +7,7 @@ import ( ) type recursiveDescentPreferences struct { - TraversePreferences *traversePreferences + TraversePreferences traversePreferences RecurseArray bool } diff --git a/pkg/yqlib/operator_traverse_path.go b/pkg/yqlib/operator_traverse_path.go index b7825bab..d63b6cec 100644 --- a/pkg/yqlib/operator_traverse_path.go +++ b/pkg/yqlib/operator_traverse_path.go @@ -10,11 +10,12 @@ import ( ) type traversePreferences struct { - FollowAlias bool - IncludeMapKeys bool + DontFollowAlias bool + IncludeMapKeys bool + DontAutoCreate bool // by default, we automatically create entries on the fly. } -func splat(d *dataTreeNavigator, matches *list.List, prefs *traversePreferences) (*list.List, error) { +func splat(d *dataTreeNavigator, matches *list.List, prefs traversePreferences) (*list.List, error) { return traverseNodesWithArrayIndices(matches, make([]*yaml.Node, 0), prefs) } @@ -54,8 +55,7 @@ func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, operation *Oper switch value.Kind { case yaml.MappingNode: log.Debug("its a map with %v entries", len(value.Content)/2) - prefs := &traversePreferences{FollowAlias: true} - return traverseMap(matchingNode, operation.StringValue, prefs, false) + return traverseMap(matchingNode, operation.StringValue, operation.Preferences.(traversePreferences), false) case yaml.SequenceNode: log.Debug("its a sequence of %v things!", len(value.Content)) @@ -83,11 +83,10 @@ func traverseArrayOperator(d *dataTreeNavigator, matchingNodes *list.List, expre } var indicesToTraverse = rhs.Front().Value.(*CandidateNode).Node.Content - prefs := &traversePreferences{FollowAlias: true} - return traverseNodesWithArrayIndices(matchingNodes, indicesToTraverse, prefs) + return traverseNodesWithArrayIndices(matchingNodes, indicesToTraverse, traversePreferences{}) } -func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse []*yaml.Node, prefs *traversePreferences) (*list.List, error) { +func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse []*yaml.Node, prefs traversePreferences) (*list.List, error) { var matchingNodeMap = list.New() for el := matchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) @@ -101,7 +100,7 @@ func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse [ return matchingNodeMap, nil } -func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml.Node, prefs *traversePreferences) (*list.List, error) { // call this if doc / alias like the other traverse +func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml.Node, prefs traversePreferences) (*list.List, error) { // call this if doc / alias like the other traverse node := matchingNode.Node if node.Tag == "!!null" { log.Debugf("OperatorArrayTraverse got a null - turning it into an empty array") @@ -124,7 +123,7 @@ func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml return list.New(), nil } -func traverseMapWithIndices(candidate *CandidateNode, indices []*yaml.Node, prefs *traversePreferences) (*list.List, error) { +func traverseMapWithIndices(candidate *CandidateNode, indices []*yaml.Node, prefs traversePreferences) (*list.List, error) { if len(indices) == 0 { return traverseMap(candidate, "", prefs, true) } @@ -188,7 +187,7 @@ func keyMatches(key *yaml.Node, wantedKey string) bool { return matchKey(key.Value, wantedKey) } -func traverseMap(matchingNode *CandidateNode, key string, prefs *traversePreferences, splat bool) (*list.List, error) { +func traverseMap(matchingNode *CandidateNode, key string, prefs traversePreferences, splat bool) (*list.List, error) { var newMatches = orderedmap.NewOrderedMap() err := doTraverseMap(newMatches, matchingNode, key, prefs, splat) @@ -196,7 +195,7 @@ func traverseMap(matchingNode *CandidateNode, key string, prefs *traversePrefere return nil, err } - if newMatches.Len() == 0 { + if !prefs.DontAutoCreate && newMatches.Len() == 0 { //no matches, create one automagically valueNode := &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"} node := matchingNode.Node @@ -214,7 +213,7 @@ func traverseMap(matchingNode *CandidateNode, key string, prefs *traversePrefere return results, nil } -func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, wantedKey string, prefs *traversePreferences, splat bool) error { +func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, wantedKey string, prefs traversePreferences, splat bool) error { // value.Content is a concatenated array of key, value, // so keys are in the even indexes, values in odd. // merge aliases are defined first, but we only want to traverse them @@ -229,7 +228,7 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, log.Debug("checking %v (%v)", key.Value, key.Tag) //skip the 'merge' tag, find a direct match first - if key.Tag == "!!merge" && prefs.FollowAlias { + if key.Tag == "!!merge" && !prefs.DontFollowAlias { log.Debug("Merge anchor") err := traverseMergeAnchor(newMatches, candidate, value, wantedKey, prefs, splat) if err != nil { @@ -249,7 +248,7 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, return nil } -func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *yaml.Node, wantedKey string, prefs *traversePreferences, splat bool) error { +func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *yaml.Node, wantedKey string, prefs traversePreferences, splat bool) error { switch value.Kind { case yaml.AliasNode: candidateNode := originalCandidate.CreateChild(nil, value.Alias) diff --git a/pkg/yqlib/operators.go b/pkg/yqlib/operators.go index 01e7087f..37d4b1eb 100644 --- a/pkg/yqlib/operators.go +++ b/pkg/yqlib/operators.go @@ -35,14 +35,16 @@ func nodeToMap(candidate *CandidateNode) *list.List { return elMap } -func createTraversalTree(path []interface{}) *ExpressionNode { +func createTraversalTree(path []interface{}, traversePrefs traversePreferences) *ExpressionNode { if len(path) == 0 { return &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}} } else if len(path) == 1 { - return &ExpressionNode{Operation: &Operation{OperationType: traversePathOpType, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}} + return &ExpressionNode{Operation: &Operation{OperationType: traversePathOpType, Preferences: traversePrefs, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}} } + return &ExpressionNode{ Operation: &Operation{OperationType: shortPipeOpType}, - Lhs: createTraversalTree(path[0:1]), - Rhs: createTraversalTree(path[1:])} + Lhs: createTraversalTree(path[0:1], traversePrefs), + Rhs: createTraversalTree(path[1:], traversePrefs), + } }