diff --git a/pkg/yqlib/doc/Multiply.md b/pkg/yqlib/doc/Multiply.md
index 87ea2d89..cd872c86 100644
--- a/pkg/yqlib/doc/Multiply.md
+++ b/pkg/yqlib/doc/Multiply.md
@@ -111,6 +111,26 @@ b:
- 5
```
+## Merge, only existing fields
+Given a sample.yml file of:
+```yaml
+a:
+ thing: one
+ cat: frog
+b:
+ missing: two
+ thing: two
+```
+then
+```bash
+yq eval '.a *? .b' sample.yml
+```
+will output
+```yaml
+thing: two
+cat: frog
+```
+
## Merge, appending arrays
Given a sample.yml file of:
```yaml
@@ -143,6 +163,33 @@ array:
value: banana
```
+## Merge, only existing fields, appending arrays
+Given a sample.yml file of:
+```yaml
+a:
+ thing:
+ - 1
+ - 2
+b:
+ thing:
+ - 3
+ - 4
+ another:
+ - 1
+```
+then
+```bash
+yq eval '.a *?+ .b' sample.yml
+```
+will output
+```yaml
+thing:
+ - 1
+ - 2
+ - 3
+ - 4
+```
+
## Merge to prefix an element
Given a sample.yml file of:
```yaml
diff --git a/pkg/yqlib/expression_processing_test.go b/pkg/yqlib/expression_processing_test.go
index 45226b70..b4d4e373 100644
--- a/pkg/yqlib/expression_processing_test.go
+++ b/pkg/yqlib/expression_processing_test.go
@@ -52,11 +52,6 @@ var pathTests = []struct {
append(make([]interface{}, 0), "[", "3 (int64)", "]"),
append(make([]interface{}, 0), "3 (int64)", "COLLECT", "SHORT_PIPE"),
},
- {
- `d0.a`,
- append(make([]interface{}, 0), "d0", "SHORT_PIPE", "a"),
- append(make([]interface{}, 0), "d0", "a", "SHORT_PIPE"),
- },
{
`.a | .[].b == "apple"`,
append(make([]interface{}, 0), "a", "PIPE", "TRAVERSE_ARRAY", "[", "]", "SHORT_PIPE", "b", "EQUALS", "apple (string)"),
diff --git a/pkg/yqlib/expression_tokeniser.go b/pkg/yqlib/expression_tokeniser.go
index 1c317bc2..9cdec47c 100644
--- a/pkg/yqlib/expression_tokeniser.go
+++ b/pkg/yqlib/expression_tokeniser.go
@@ -3,6 +3,7 @@ package yqlib
import (
"fmt"
"strconv"
+ "strings"
lex "github.com/timtadh/lexmachine"
"github.com/timtadh/lexmachine/machines"
@@ -65,21 +66,7 @@ func pathToken(wrapped bool) lex.Action {
value = unwrap(value)
}
log.Debug("PathToken %v", value)
- op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value}
- return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
- }
-}
-
-func documentToken() lex.Action {
- return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
- var numberString = string(m.Bytes)
- numberString = numberString[1:]
- var number, errParsingInt = strconv.ParseInt(numberString, 10, 64) // nolint
- if errParsingInt != nil {
- return nil, errParsingInt
- }
- log.Debug("documentToken %v", string(m.Bytes))
- op := &Operation{OperationType: documentFilterOpType, Value: number, StringValue: numberString}
+ op := &Operation{OperationType: traversePathOpType, Value: value, StringValue: value, Preferences: traversePreferences{}}
return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: true}, nil
}
}
@@ -101,6 +88,21 @@ func assignOpToken(updateAssign bool) lex.Action {
}
}
+func multiplyWithPrefs() lex.Action {
+ return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+ prefs := multiplyPreferences{}
+ options := string(m.Bytes)
+ if strings.Contains(options, "+") {
+ prefs.AppendArrays = true
+ }
+ if strings.Contains(options, "?") {
+ prefs.TraversePrefs = traversePreferences{DontAutoCreate: true}
+ }
+ op := &Operation{OperationType: multiplyOpType, Value: multiplyOpType.Type, StringValue: options, Preferences: &prefs}
+ return &token{TokenType: operationToken, Operation: op}, nil
+ }
+}
+
func opTokenWithPrefs(op *operationType, assignOpType *operationType, preferences interface{}) lex.Action {
return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
log.Debug("opTokenWithPrefs %v", string(m.Bytes))
@@ -220,10 +222,10 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`\.\[`), literalToken(traverseArrayCollect, false))
lexer.Add([]byte(`\.\.`), opTokenWithPrefs(recursiveDescentOpType, nil, &recursiveDescentPreferences{RecurseArray: true,
- TraversePreferences: &traversePreferences{FollowAlias: false, IncludeMapKeys: false}}))
+ TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: false}}))
lexer.Add([]byte(`\.\.\.`), opTokenWithPrefs(recursiveDescentOpType, nil, &recursiveDescentPreferences{RecurseArray: true,
- TraversePreferences: &traversePreferences{FollowAlias: false, IncludeMapKeys: true}}))
+ TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: true}}))
lexer.Add([]byte(`,`), opToken(unionOpType))
lexer.Add([]byte(`:\s*`), opToken(createMapOpType))
@@ -270,7 +272,6 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte("( |\t|\n|\r)+"), skip)
- lexer.Add([]byte(`d[0-9]+`), documentToken())
lexer.Add([]byte(`\."[^ "]+"`), pathToken(true))
lexer.Add([]byte(`\.[^ \}\{\:\[\],\|\.\[\(\)=]+`), pathToken(false))
lexer.Add([]byte(`\.`), selfToken())
@@ -296,7 +297,7 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`\{`), literalToken(openCollectObject, false))
lexer.Add([]byte(`\}`), literalToken(closeCollectObject, true))
lexer.Add([]byte(`\*`), opTokenWithPrefs(multiplyOpType, nil, &multiplyPreferences{AppendArrays: false}))
- lexer.Add([]byte(`\*\+`), opTokenWithPrefs(multiplyOpType, nil, &multiplyPreferences{AppendArrays: true}))
+ lexer.Add([]byte(`\*[\+|\?]*`), multiplyWithPrefs())
lexer.Add([]byte(`\+`), opToken(addOpType))
lexer.Add([]byte(`\+=`), opToken(addAssignOpType))
diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go
index d5aab1f3..1994cb6c 100644
--- a/pkg/yqlib/lib.go
+++ b/pkg/yqlib/lib.go
@@ -68,7 +68,6 @@ var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Pre
var traversePathOpType = &operationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 50, Handler: traversePathOperator}
var traverseArrayOpType = &operationType{Type: "TRAVERSE_ARRAY", NumArgs: 1, Precedence: 50, Handler: traverseArrayOperator}
-var documentFilterOpType = &operationType{Type: "DOCUMENT_FILTER", NumArgs: 0, Precedence: 50, Handler: traversePathOperator}
var selfReferenceOpType = &operationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: selfOperator}
var valueOpType = &operationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: valueOperator}
var envOpType = &operationType{Type: "ENV", NumArgs: 0, Precedence: 50, Handler: envOperator}
@@ -120,8 +119,6 @@ func createValueOperation(value interface{}, stringValue string) *Operation {
func (p *Operation) toString() string {
if p.OperationType == traversePathOpType {
return fmt.Sprintf("%v", p.Value)
- } else if p.OperationType == documentFilterOpType {
- return fmt.Sprintf("d%v", p.Value)
} else if p.OperationType == selfReferenceOpType {
return "SELF"
} else if p.OperationType == valueOpType {
diff --git a/pkg/yqlib/operator_collect_object.go b/pkg/yqlib/operator_collect_object.go
index 8be51611..c9d5fda4 100644
--- a/pkg/yqlib/operator_collect_object.go
+++ b/pkg/yqlib/operator_collect_object.go
@@ -60,7 +60,7 @@ func collect(d *dataTreeNavigator, aggregate *list.List, remainingMatches *list.
candidate := remainingMatches.Remove(remainingMatches.Front()).(*CandidateNode)
splatted, err := splat(d, nodeToMap(candidate),
- &traversePreferences{FollowAlias: false, IncludeMapKeys: false})
+ traversePreferences{DontFollowAlias: true, IncludeMapKeys: false})
for splatEl := splatted.Front(); splatEl != nil; splatEl = splatEl.Next() {
splatEl.Value.(*CandidateNode).Path = nil
diff --git a/pkg/yqlib/operator_delete.go b/pkg/yqlib/operator_delete.go
index cecacaeb..c84039e6 100644
--- a/pkg/yqlib/operator_delete.go
+++ b/pkg/yqlib/operator_delete.go
@@ -25,7 +25,7 @@ func deleteChildOperator(d *dataTreeNavigator, matchingNodes *list.List, express
deleteImmediateChildOpNode := &ExpressionNode{
Operation: deleteImmediateChildOp,
- Rhs: createTraversalTree(candidate.Path[0 : len(candidate.Path)-1]),
+ Rhs: createTraversalTree(candidate.Path[0:len(candidate.Path)-1], traversePreferences{}),
}
_, err := d.GetMatchingNodes(matchingNodes, deleteImmediateChildOpNode)
diff --git a/pkg/yqlib/operator_multiply.go b/pkg/yqlib/operator_multiply.go
index 2428c465..35d43d6d 100644
--- a/pkg/yqlib/operator_multiply.go
+++ b/pkg/yqlib/operator_multiply.go
@@ -44,7 +44,8 @@ func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, expressionNod
}
type multiplyPreferences struct {
- AppendArrays bool
+ AppendArrays bool
+ TraversePrefs traversePreferences
}
func multiplyOperator(d *dataTreeNavigator, matchingNodes *list.List, expressionNode *ExpressionNode) (*list.List, error) {
@@ -59,29 +60,28 @@ func multiply(preferences *multiplyPreferences) func(d *dataTreeNavigator, lhs *
log.Debugf("Multipling LHS: %v", lhs.Node.Tag)
log.Debugf("- RHS: %v", rhs.Node.Tag)
- shouldAppendArrays := preferences.AppendArrays
-
if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode ||
(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
var newBlank = lhs.CreateChild(nil, &yaml.Node{})
- var newThing, err = mergeObjects(d, newBlank, lhs, false)
+ var newThing, err = mergeObjects(d, newBlank, lhs, &multiplyPreferences{})
if err != nil {
return nil, err
}
- return mergeObjects(d, newThing, rhs, shouldAppendArrays)
+ return mergeObjects(d, newThing, rhs, preferences)
}
return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag)
}
}
-func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode, shouldAppendArrays bool) (*CandidateNode, error) {
+func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode, preferences *multiplyPreferences) (*CandidateNode, error) {
+ shouldAppendArrays := preferences.AppendArrays
var results = list.New()
// shouldn't recurse arrays if appending
prefs := &recursiveDescentPreferences{RecurseArray: !shouldAppendArrays,
- TraversePreferences: &traversePreferences{FollowAlias: false}}
+ TraversePreferences: traversePreferences{DontFollowAlias: true}}
err := recursiveDecent(d, results, nodeToMap(rhs), prefs)
if err != nil {
return nil, err
@@ -93,7 +93,7 @@ func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode,
}
for el := results.Front(); el != nil; el = el.Next() {
- err := applyAssignment(d, pathIndexToStartFrom, lhs, el.Value.(*CandidateNode), shouldAppendArrays)
+ err := applyAssignment(d, pathIndexToStartFrom, lhs, el.Value.(*CandidateNode), preferences)
if err != nil {
return nil, err
}
@@ -101,8 +101,8 @@ func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode,
return lhs, nil
}
-func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode, shouldAppendArrays bool) error {
-
+func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode, preferences *multiplyPreferences) error {
+ shouldAppendArrays := preferences.AppendArrays
log.Debugf("merge - applyAssignment lhs %v, rhs: %v", NodeToString(lhs), NodeToString(rhs))
lhsPath := rhs.Path[pathIndexToStartFrom:]
@@ -116,7 +116,7 @@ func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *Candid
}
rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs}
- assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath), Rhs: &ExpressionNode{Operation: rhsOp}}
+ assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath, preferences.TraversePrefs), Rhs: &ExpressionNode{Operation: rhsOp}}
_, err := d.GetMatchingNodes(nodeToMap(lhs), assignmentOpNode)
diff --git a/pkg/yqlib/operator_multiply_test.go b/pkg/yqlib/operator_multiply_test.go
index 40819c3a..ebca2f13 100644
--- a/pkg/yqlib/operator_multiply_test.go
+++ b/pkg/yqlib/operator_multiply_test.go
@@ -107,6 +107,22 @@ b:
"D0, P[a], (!!seq)::[1, 2]\n",
},
},
+ {
+ description: "Merge, only existing fields",
+ document: `{a: {thing: one, cat: frog}, b: {missing: two, thing: two}}`,
+ expression: `.a *? .b`,
+ expected: []string{
+ "D0, P[a], (!!map)::{thing: two, cat: frog}\n",
+ },
+ },
+ {
+ skipDoc: true,
+ document: `{a: [{thing: one}], b: [{missing: two, thing: two}]}`,
+ expression: `.a *? .b`,
+ expected: []string{
+ "D0, P[a], (!!seq)::[{thing: two}]\n",
+ },
+ },
{
description: "Merge, appending arrays",
document: `{a: {array: [1, 2, animal: dog], value: coconut}, b: {array: [3, 4, animal: cat], value: banana}}`,
@@ -115,6 +131,14 @@ b:
"D0, P[a], (!!map)::{array: [1, 2, {animal: dog}, 3, 4, {animal: cat}], value: banana}\n",
},
},
+ {
+ description: "Merge, only existing fields, appending arrays",
+ document: `{a: {thing: [1,2]}, b: {thing: [3,4], another: [1]}}`,
+ expression: `.a *?+ .b`,
+ expected: []string{
+ "D0, P[a], (!!map)::{thing: [1, 2, 3, 4]}\n",
+ },
+ },
{
description: "Merge to prefix an element",
document: `{a: cat, b: dog}`,
diff --git a/pkg/yqlib/operator_recursive_descent.go b/pkg/yqlib/operator_recursive_descent.go
index 52257312..962e8e41 100644
--- a/pkg/yqlib/operator_recursive_descent.go
+++ b/pkg/yqlib/operator_recursive_descent.go
@@ -7,7 +7,7 @@ import (
)
type recursiveDescentPreferences struct {
- TraversePreferences *traversePreferences
+ TraversePreferences traversePreferences
RecurseArray bool
}
diff --git a/pkg/yqlib/operator_traverse_path.go b/pkg/yqlib/operator_traverse_path.go
index b7825bab..d63b6cec 100644
--- a/pkg/yqlib/operator_traverse_path.go
+++ b/pkg/yqlib/operator_traverse_path.go
@@ -10,11 +10,12 @@ import (
)
type traversePreferences struct {
- FollowAlias bool
- IncludeMapKeys bool
+ DontFollowAlias bool
+ IncludeMapKeys bool
+ DontAutoCreate bool // by default, we automatically create entries on the fly.
}
-func splat(d *dataTreeNavigator, matches *list.List, prefs *traversePreferences) (*list.List, error) {
+func splat(d *dataTreeNavigator, matches *list.List, prefs traversePreferences) (*list.List, error) {
return traverseNodesWithArrayIndices(matches, make([]*yaml.Node, 0), prefs)
}
@@ -54,8 +55,7 @@ func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, operation *Oper
switch value.Kind {
case yaml.MappingNode:
log.Debug("its a map with %v entries", len(value.Content)/2)
- prefs := &traversePreferences{FollowAlias: true}
- return traverseMap(matchingNode, operation.StringValue, prefs, false)
+ return traverseMap(matchingNode, operation.StringValue, operation.Preferences.(traversePreferences), false)
case yaml.SequenceNode:
log.Debug("its a sequence of %v things!", len(value.Content))
@@ -83,11 +83,10 @@ func traverseArrayOperator(d *dataTreeNavigator, matchingNodes *list.List, expre
}
var indicesToTraverse = rhs.Front().Value.(*CandidateNode).Node.Content
- prefs := &traversePreferences{FollowAlias: true}
- return traverseNodesWithArrayIndices(matchingNodes, indicesToTraverse, prefs)
+ return traverseNodesWithArrayIndices(matchingNodes, indicesToTraverse, traversePreferences{})
}
-func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse []*yaml.Node, prefs *traversePreferences) (*list.List, error) {
+func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse []*yaml.Node, prefs traversePreferences) (*list.List, error) {
var matchingNodeMap = list.New()
for el := matchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
@@ -101,7 +100,7 @@ func traverseNodesWithArrayIndices(matchingNodes *list.List, indicesToTraverse [
return matchingNodeMap, nil
}
-func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml.Node, prefs *traversePreferences) (*list.List, error) { // call this if doc / alias like the other traverse
+func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml.Node, prefs traversePreferences) (*list.List, error) { // call this if doc / alias like the other traverse
node := matchingNode.Node
if node.Tag == "!!null" {
log.Debugf("OperatorArrayTraverse got a null - turning it into an empty array")
@@ -124,7 +123,7 @@ func traverseArrayIndices(matchingNode *CandidateNode, indicesToTraverse []*yaml
return list.New(), nil
}
-func traverseMapWithIndices(candidate *CandidateNode, indices []*yaml.Node, prefs *traversePreferences) (*list.List, error) {
+func traverseMapWithIndices(candidate *CandidateNode, indices []*yaml.Node, prefs traversePreferences) (*list.List, error) {
if len(indices) == 0 {
return traverseMap(candidate, "", prefs, true)
}
@@ -188,7 +187,7 @@ func keyMatches(key *yaml.Node, wantedKey string) bool {
return matchKey(key.Value, wantedKey)
}
-func traverseMap(matchingNode *CandidateNode, key string, prefs *traversePreferences, splat bool) (*list.List, error) {
+func traverseMap(matchingNode *CandidateNode, key string, prefs traversePreferences, splat bool) (*list.List, error) {
var newMatches = orderedmap.NewOrderedMap()
err := doTraverseMap(newMatches, matchingNode, key, prefs, splat)
@@ -196,7 +195,7 @@ func traverseMap(matchingNode *CandidateNode, key string, prefs *traversePrefere
return nil, err
}
- if newMatches.Len() == 0 {
+ if !prefs.DontAutoCreate && newMatches.Len() == 0 {
//no matches, create one automagically
valueNode := &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"}
node := matchingNode.Node
@@ -214,7 +213,7 @@ func traverseMap(matchingNode *CandidateNode, key string, prefs *traversePrefere
return results, nil
}
-func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, wantedKey string, prefs *traversePreferences, splat bool) error {
+func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, wantedKey string, prefs traversePreferences, splat bool) error {
// value.Content is a concatenated array of key, value,
// so keys are in the even indexes, values in odd.
// merge aliases are defined first, but we only want to traverse them
@@ -229,7 +228,7 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode,
log.Debug("checking %v (%v)", key.Value, key.Tag)
//skip the 'merge' tag, find a direct match first
- if key.Tag == "!!merge" && prefs.FollowAlias {
+ if key.Tag == "!!merge" && !prefs.DontFollowAlias {
log.Debug("Merge anchor")
err := traverseMergeAnchor(newMatches, candidate, value, wantedKey, prefs, splat)
if err != nil {
@@ -249,7 +248,7 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode,
return nil
}
-func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *yaml.Node, wantedKey string, prefs *traversePreferences, splat bool) error {
+func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *yaml.Node, wantedKey string, prefs traversePreferences, splat bool) error {
switch value.Kind {
case yaml.AliasNode:
candidateNode := originalCandidate.CreateChild(nil, value.Alias)
diff --git a/pkg/yqlib/operators.go b/pkg/yqlib/operators.go
index 01e7087f..37d4b1eb 100644
--- a/pkg/yqlib/operators.go
+++ b/pkg/yqlib/operators.go
@@ -35,14 +35,16 @@ func nodeToMap(candidate *CandidateNode) *list.List {
return elMap
}
-func createTraversalTree(path []interface{}) *ExpressionNode {
+func createTraversalTree(path []interface{}, traversePrefs traversePreferences) *ExpressionNode {
if len(path) == 0 {
return &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
} else if len(path) == 1 {
- return &ExpressionNode{Operation: &Operation{OperationType: traversePathOpType, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}}
+ return &ExpressionNode{Operation: &Operation{OperationType: traversePathOpType, Preferences: traversePrefs, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}}
}
+
return &ExpressionNode{
Operation: &Operation{OperationType: shortPipeOpType},
- Lhs: createTraversalTree(path[0:1]),
- Rhs: createTraversalTree(path[1:])}
+ Lhs: createTraversalTree(path[0:1], traversePrefs),
+ Rhs: createTraversalTree(path[1:], traversePrefs),
+ }
}