diff --git a/pkg/yqlib/doc/Multiply.md b/pkg/yqlib/doc/Multiply.md index 9f09fea5..42d11573 100644 --- a/pkg/yqlib/doc/Multiply.md +++ b/pkg/yqlib/doc/Multiply.md @@ -66,9 +66,9 @@ b: Given a sample.yml file of: ```yaml a: {things: great} -b: - also: "me" - + b: + also: "me" + ``` then ```bash @@ -76,9 +76,6 @@ yq eval '. * {"a":.b}' sample.yml ``` will output ```yaml -a: {things: great, also: "me"} -b: - also: "me" ``` ## Merge arrays @@ -109,6 +106,38 @@ b: - 5 ``` +## Merge, appending arrays +Given a sample.yml file of: +```yaml +a: + array: + - 1 + - 2 + - animal: dog + value: coconut +b: + array: + - 3 + - 4 + - animal: cat + value: banana +``` +then +```bash +yq eval '.a *+ .b' sample.yml +``` +will output +```yaml +array: + - 1 + - 2 + - animal: dog + - 3 + - 4 + - animal: cat +value: banana +``` + ## Merge to prefix an element Given a sample.yml file of: ```yaml diff --git a/pkg/yqlib/operator_add.go b/pkg/yqlib/operator_add.go index fad85ecd..9b294c53 100644 --- a/pkg/yqlib/operator_add.go +++ b/pkg/yqlib/operator_add.go @@ -8,6 +8,10 @@ import ( yaml "gopkg.in/yaml.v3" ) +type AddPreferences struct { + InPlace bool +} + func toNodes(candidates *list.List) []*yaml.Node { if candidates.Len() == 0 { @@ -41,26 +45,35 @@ func AddOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathT return nil, err } + preferences := pathNode.Operation.Preferences + inPlace := preferences != nil && preferences.(*AddPreferences).InPlace + for el := lhs.Front(); el != nil; el = el.Next() { lhsCandidate := el.Value.(*CandidateNode) lhsNode := UnwrapDoc(lhsCandidate.Node) - var newBlank = &CandidateNode{ - Path: lhsCandidate.Path, - Document: lhsCandidate.Document, - Filename: lhsCandidate.Filename, - Node: &yaml.Node{}, + var target *CandidateNode + + if inPlace { + target = lhsCandidate + } else { + target = &CandidateNode{ + Path: lhsCandidate.Path, + Document: lhsCandidate.Document, + Filename: lhsCandidate.Filename, + Node: &yaml.Node{}, + } } switch lhsNode.Kind { case yaml.MappingNode: return nil, fmt.Errorf("Maps not yet supported for addition") case yaml.SequenceNode: - newBlank.Node.Kind = yaml.SequenceNode - newBlank.Node.Style = lhsNode.Style - newBlank.Node.Tag = "!!seq" - newBlank.Node.Content = append(lhsNode.Content, toNodes(rhs)...) - results.PushBack(newBlank) + target.Node.Kind = yaml.SequenceNode + target.Node.Style = lhsNode.Style + target.Node.Tag = "!!seq" + target.Node.Content = append(lhsNode.Content, toNodes(rhs)...) + results.PushBack(target) case yaml.ScalarNode: return nil, fmt.Errorf("Scalars not yet supported for addition") } diff --git a/pkg/yqlib/operator_add_test.go b/pkg/yqlib/operator_add_test.go index 8f4459ed..14923be7 100644 --- a/pkg/yqlib/operator_add_test.go +++ b/pkg/yqlib/operator_add_test.go @@ -5,6 +5,9 @@ import ( ) var addOperatorScenarios = []expressionScenario{ + { + description: "+= test and doc", + }, { description: "Concatenate arrays", document: `{a: [1,2], b: [3,4]}`, diff --git a/pkg/yqlib/operator_collect_object.go b/pkg/yqlib/operator_collect_object.go index 298880c9..147fc633 100644 --- a/pkg/yqlib/operator_collect_object.go +++ b/pkg/yqlib/operator_collect_object.go @@ -94,7 +94,7 @@ func collect(d *dataTreeNavigator, aggregate *list.List, remainingMatches *list. newCandidate.Path = nil - newCandidate, err = multiply(d, newCandidate, splatCandidate) + newCandidate, err = multiply(&MultiplyPreferences{AppendArrays: false})(d, newCandidate, splatCandidate) if err != nil { return nil, err } diff --git a/pkg/yqlib/operator_multiply.go b/pkg/yqlib/operator_multiply.go index ac7b456e..a9657fbb 100644 --- a/pkg/yqlib/operator_multiply.go +++ b/pkg/yqlib/operator_multiply.go @@ -43,39 +43,49 @@ func crossFunction(d *dataTreeNavigator, matchingNodes *list.List, pathNode *Pat return results, nil } +type MultiplyPreferences struct { + AppendArrays bool +} + func MultiplyOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) { log.Debugf("-- MultiplyOperator") - return crossFunction(d, matchingNodes, pathNode, multiply) + return crossFunction(d, matchingNodes, pathNode, multiply(pathNode.Operation.Preferences.(*MultiplyPreferences))) } -func multiply(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { - lhs.Node = UnwrapDoc(lhs.Node) - rhs.Node = UnwrapDoc(rhs.Node) - log.Debugf("Multipling LHS: %v", lhs.Node.Tag) - log.Debugf("- RHS: %v", rhs.Node.Tag) +func multiply(preferences *MultiplyPreferences) func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { + return func(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { + lhs.Node = UnwrapDoc(lhs.Node) + rhs.Node = UnwrapDoc(rhs.Node) + log.Debugf("Multipling LHS: %v", lhs.Node.Tag) + log.Debugf("- RHS: %v", rhs.Node.Tag) - if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode || - (lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) { + shouldAppendArrays := preferences.AppendArrays + + if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode || + (lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) { + + var newBlank = &CandidateNode{ + Path: lhs.Path, + Document: lhs.Document, + Filename: lhs.Filename, + Node: &yaml.Node{}, + } + var newThing, err = mergeObjects(d, newBlank, lhs, false) + if err != nil { + return nil, err + } + return mergeObjects(d, newThing, rhs, shouldAppendArrays) - var newBlank = &CandidateNode{ - Path: lhs.Path, - Document: lhs.Document, - Filename: lhs.Filename, - Node: &yaml.Node{}, } - var newThing, err = mergeObjects(d, newBlank, lhs) - if err != nil { - return nil, err - } - return mergeObjects(d, newThing, rhs) - + return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag) } - return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag) } -func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { +func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode, shouldAppendArrays bool) (*CandidateNode, error) { var results = list.New() - err := recursiveDecent(d, results, nodeToMap(rhs)) + + // shouldn't recurse arrays if appending + err := recursiveDecent(d, results, nodeToMap(rhs), !shouldAppendArrays) if err != nil { return nil, err } @@ -86,7 +96,7 @@ func mergeObjects(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) } for el := results.Front(); el != nil; el = el.Next() { - err := applyAssignment(d, pathIndexToStartFrom, lhs, el.Value.(*CandidateNode)) + err := applyAssignment(d, pathIndexToStartFrom, lhs, el.Value.(*CandidateNode), shouldAppendArrays) if err != nil { return nil, err } @@ -107,7 +117,8 @@ func createTraversalTree(path []interface{}) *PathTreeNode { } -func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode) error { +func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode, shouldAppendArrays bool) error { + log.Debugf("merge - applyAssignment lhs %v, rhs: %v", NodeToString(lhs), NodeToString(rhs)) lhsPath := rhs.Path[pathIndexToStartFrom:] @@ -116,6 +127,10 @@ func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *Candid if rhs.Node.Kind == yaml.ScalarNode || rhs.Node.Kind == yaml.AliasNode { assignmentOp.OperationType = Assign assignmentOp.Preferences = &AssignOpPreferences{false} + } else if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode { + log.Debugf("append! lhs %v, rhs: %v", NodeToString(lhs), NodeToString(rhs)) + assignmentOp.OperationType = Add + assignmentOp.Preferences = &AddPreferences{InPlace: true} } rhsOp := &Operation{OperationType: ValueOp, CandidateNode: rhs} diff --git a/pkg/yqlib/operator_multiply_test.go b/pkg/yqlib/operator_multiply_test.go index d16de6fa..3007bdeb 100644 --- a/pkg/yqlib/operator_multiply_test.go +++ b/pkg/yqlib/operator_multiply_test.go @@ -5,6 +5,9 @@ import ( ) var multiplyOperatorScenarios = []expressionScenario{ + { + description: "*+ doc", + }, { skipDoc: true, document: `{a: {also: [1]}, b: {also: me}}`, @@ -80,15 +83,15 @@ var multiplyOperatorScenarios = []expressionScenario{ description: "Merge keeps style of LHS", dontFormatInputForDoc: true, document: `a: {things: great} -b: - also: "me" -`, + b: + also: "me" + `, expression: `. * {"a":.b}`, expected: []string{ `D0, P[], (!!map)::a: {things: great, also: "me"} -b: - also: "me" -`, + b: + also: "me" + `, }, }, { @@ -99,6 +102,22 @@ b: "D0, P[], (!!map)::{a: [3, 4, 5], b: [3, 4, 5]}\n", }, }, + { + skipDoc: true, + document: `{a: [1], b: [2]}`, + expression: `.a *+ .b`, + expected: []string{ + "D0, P[a], (!!seq)::[1, 2]\n", + }, + }, + { + description: "Merge, appending arrays", + document: `{a: {array: [1, 2, animal: dog], value: coconut}, b: {array: [3, 4, animal: cat], value: banana}}`, + expression: `.a *+ .b`, + expected: []string{ + "D0, P[a], (!!map)::{array: [1, 2, {animal: dog}, 3, 4, {animal: cat}], value: banana}\n", + }, + }, { description: "Merge to prefix an element", document: `{a: cat, b: dog}`, diff --git a/pkg/yqlib/operator_recursive_descent.go b/pkg/yqlib/operator_recursive_descent.go index 51e1439c..0cb8f898 100644 --- a/pkg/yqlib/operator_recursive_descent.go +++ b/pkg/yqlib/operator_recursive_descent.go @@ -3,13 +3,13 @@ package yqlib import ( "container/list" - "gopkg.in/yaml.v3" + yaml "gopkg.in/yaml.v3" ) func RecursiveDescentOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) { var results = list.New() - err := recursiveDecent(d, results, matchMap) + err := recursiveDecent(d, results, matchMap, true) if err != nil { return nil, err } @@ -17,7 +17,7 @@ func RecursiveDescentOperator(d *dataTreeNavigator, matchMap *list.List, pathNod return results, nil } -func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.List) error { +func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.List, recurseArray bool) error { for el := matchMap.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) @@ -26,14 +26,15 @@ func recursiveDecent(d *dataTreeNavigator, results *list.List, matchMap *list.Li log.Debugf("Recursive Decent, added %v", NodeToString(candidate)) results.PushBack(candidate) - if candidate.Node.Kind != yaml.AliasNode && len(candidate.Node.Content) > 0 { + if candidate.Node.Kind != yaml.AliasNode && len(candidate.Node.Content) > 0 && + (recurseArray || candidate.Node.Kind != yaml.SequenceNode) { children, err := Splat(d, nodeToMap(candidate)) if err != nil { return err } - err = recursiveDecent(d, results, children) + err = recursiveDecent(d, results, children, recurseArray) if err != nil { return err } diff --git a/pkg/yqlib/path_tokeniser.go b/pkg/yqlib/path_tokeniser.go index 9d0faba6..5946156b 100644 --- a/pkg/yqlib/path_tokeniser.go +++ b/pkg/yqlib/path_tokeniser.go @@ -251,8 +251,10 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`\]`), literalToken(CloseCollect, true)) lexer.Add([]byte(`\{`), literalToken(OpenCollectObject, false)) lexer.Add([]byte(`\}`), literalToken(CloseCollectObject, true)) - lexer.Add([]byte(`\*`), opToken(Multiply)) + lexer.Add([]byte(`\*`), opTokenWithPrefs(Multiply, nil, &MultiplyPreferences{AppendArrays: false})) + lexer.Add([]byte(`\*\+`), opTokenWithPrefs(Multiply, nil, &MultiplyPreferences{AppendArrays: true})) lexer.Add([]byte(`\+`), opToken(Add)) + lexer.Add([]byte(`\+=`), opTokenWithPrefs(Add, nil, &AddPreferences{InPlace: true})) err := lexer.Compile() if err != nil {