diff --git a/pkg/yqlib/doc/Add.md b/pkg/yqlib/doc/Add.md index 82550b40..bb994943 100644 --- a/pkg/yqlib/doc/Add.md +++ b/pkg/yqlib/doc/Add.md @@ -3,6 +3,31 @@ Add behaves differently according to the type of the LHS: - number scalars: arithmetic addition (soon) - string scalars: concatenate (soon) +Use `+=` as append assign for things like increment. `.a += .x` is equivalent to running `.a |= . + .x`. + +## Concatenate and assign arrays +Given a sample.yml file of: +```yaml +a: + val: thing + b: + - cat + - dog +``` +then +```bash +yq eval '.a.b += ["cow"]' sample.yml +``` +will output +```yaml +a: + val: thing + b: + - cat + - dog + - cow +``` + ## Concatenate arrays Given a sample.yml file of: ```yaml diff --git a/pkg/yqlib/doc/Assign.md b/pkg/yqlib/doc/Assign.md index c5f1ce61..11524986 100644 --- a/pkg/yqlib/doc/Assign.md +++ b/pkg/yqlib/doc/Assign.md @@ -148,7 +148,7 @@ Given a sample.yml file of: ``` then ```bash -yq eval '.a.b[0] |= "bogs"' sample.yml +yq eval '.a.b.[0] |= "bogs"' sample.yml ``` will output ```yaml diff --git a/pkg/yqlib/doc/Multiply.md b/pkg/yqlib/doc/Multiply.md index 42d11573..3fc98d53 100644 --- a/pkg/yqlib/doc/Multiply.md +++ b/pkg/yqlib/doc/Multiply.md @@ -2,6 +2,8 @@ Like the multiple operator in `jq`, depending on the operands, this multiply ope Upcoming versions of `yq` will add support for other types of multiplication (numbers, strings). +To concatenate when merging objects, use the `*+` form (see examples below). This will recursively merge objects, appending arrays when it encounters them. + Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below. ## Merging files @@ -66,9 +68,9 @@ b: Given a sample.yml file of: ```yaml a: {things: great} - b: - also: "me" - +b: + also: "me" + ``` then ```bash @@ -76,6 +78,9 @@ yq eval '. * {"a":.b}' sample.yml ``` will output ```yaml +a: {things: great, also: "me"} +b: + also: "me" ``` ## Merge arrays diff --git a/pkg/yqlib/doc/Traverse.md b/pkg/yqlib/doc/Traverse.md index 88ccdb10..20befb25 100644 --- a/pkg/yqlib/doc/Traverse.md +++ b/pkg/yqlib/doc/Traverse.md @@ -138,7 +138,7 @@ Given a sample.yml file of: ``` then ```bash -yq eval '[0]' sample.yml +yq eval '.[0]' sample.yml ``` will output ```yaml @@ -152,7 +152,7 @@ Given a sample.yml file of: ``` then ```bash -yq eval '[2]' sample.yml +yq eval '.[2]' sample.yml ``` will output ```yaml @@ -166,7 +166,7 @@ a: b ``` then ```bash -yq eval '[0]' sample.yml +yq eval '.[0]' sample.yml ``` will output ```yaml diff --git a/pkg/yqlib/doc/headers/Add.md b/pkg/yqlib/doc/headers/Add.md index f451aa95..96cf7246 100644 --- a/pkg/yqlib/doc/headers/Add.md +++ b/pkg/yqlib/doc/headers/Add.md @@ -2,3 +2,5 @@ Add behaves differently according to the type of the LHS: - arrays: concatenate - number scalars: arithmetic addition (soon) - string scalars: concatenate (soon) + +Use `+=` as append assign for things like increment. `.a += .x` is equivalent to running `.a |= . + .x`. diff --git a/pkg/yqlib/doc/headers/Multiply.md b/pkg/yqlib/doc/headers/Multiply.md index 62c73302..81cf758b 100644 --- a/pkg/yqlib/doc/headers/Multiply.md +++ b/pkg/yqlib/doc/headers/Multiply.md @@ -2,6 +2,8 @@ Like the multiple operator in `jq`, depending on the operands, this multiply ope Upcoming versions of `yq` will add support for other types of multiplication (numbers, strings). +To concatenate when merging objects, use the `*+` form (see examples below). This will recursively merge objects, appending arrays when it encounters them. + Note that when merging objects, this operator returns the merged object (not the parent). This will be clearer in the examples below. ## Merging files diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 8883d392..abc4e1a5 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -31,6 +31,7 @@ var And = &OperationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: AndOp var Union = &OperationType{Type: "UNION", NumArgs: 2, Precedence: 10, Handler: UnionOperator} var Assign = &OperationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: AssignUpdateOperator} +var AddAssign = &OperationType{Type: "ADD_ASSIGN", NumArgs: 2, Precedence: 40, Handler: AddAssignOperator} var AssignAttributes = &OperationType{Type: "ASSIGN_ATTRIBUTES", NumArgs: 2, Precedence: 40, Handler: AssignAttributesOperator} var AssignStyle = &OperationType{Type: "ASSIGN_STYLE", NumArgs: 2, Precedence: 40, Handler: AssignStyleOperator} diff --git a/pkg/yqlib/operator_add.go b/pkg/yqlib/operator_add.go index 9b294c53..4bab2f22 100644 --- a/pkg/yqlib/operator_add.go +++ b/pkg/yqlib/operator_add.go @@ -8,8 +8,18 @@ import ( yaml "gopkg.in/yaml.v3" ) -type AddPreferences struct { - InPlace bool +func createSelfAddOp(rhs *PathTreeNode) *PathTreeNode { + return &PathTreeNode{Operation: &Operation{OperationType: Add}, + Lhs: &PathTreeNode{Operation: &Operation{OperationType: SelfReference}}, + Rhs: rhs} +} + +func AddAssignOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) { + assignmentOp := &Operation{OperationType: Assign} + assignmentOp.Preferences = &AssignOpPreferences{true} + + assignmentOpNode := &PathTreeNode{Operation: assignmentOp, Lhs: pathNode.Lhs, Rhs: createSelfAddOp(pathNode.Rhs)} + return d.GetMatchingNodes(matchingNodes, assignmentOpNode) } func toNodes(candidates *list.List) []*yaml.Node { @@ -45,24 +55,15 @@ func AddOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathT return nil, err } - preferences := pathNode.Operation.Preferences - inPlace := preferences != nil && preferences.(*AddPreferences).InPlace - for el := lhs.Front(); el != nil; el = el.Next() { lhsCandidate := el.Value.(*CandidateNode) lhsNode := UnwrapDoc(lhsCandidate.Node) - var target *CandidateNode - - if inPlace { - target = lhsCandidate - } else { - target = &CandidateNode{ - Path: lhsCandidate.Path, - Document: lhsCandidate.Document, - Filename: lhsCandidate.Filename, - Node: &yaml.Node{}, - } + target := &CandidateNode{ + Path: lhsCandidate.Path, + Document: lhsCandidate.Document, + Filename: lhsCandidate.Filename, + Node: &yaml.Node{}, } switch lhsNode.Kind { diff --git a/pkg/yqlib/operator_add_test.go b/pkg/yqlib/operator_add_test.go index 5e16536b..04092635 100644 --- a/pkg/yqlib/operator_add_test.go +++ b/pkg/yqlib/operator_add_test.go @@ -6,9 +6,12 @@ import ( var addOperatorScenarios = []expressionScenario{ { - description: "+= test and doc", - expression: ".a.b+= .e.f" - expected: []string{"add .e.g to be, return top level node"} + description: "Concatenate and assign arrays", + document: `{a: {val: thing, b: [cat,dog]}}`, + expression: ".a.b += [\"cow\"]", + expected: []string{ + "D0, P[], (doc)::{a: {val: thing, b: [cat, dog, cow]}}\n", + }, }, { description: "Concatenate arrays", diff --git a/pkg/yqlib/operator_assign_test.go b/pkg/yqlib/operator_assign_test.go index 06251349..cf5fa788 100644 --- a/pkg/yqlib/operator_assign_test.go +++ b/pkg/yqlib/operator_assign_test.go @@ -99,7 +99,7 @@ var assignOperatorScenarios = []expressionScenario{ description: "Update empty object and array", dontFormatInputForDoc: true, document: `{}`, - expression: `.a.b[0] |= "bogs"`, + expression: `.a.b.[0] |= "bogs"`, expected: []string{ "D0, P[], (doc)::{a: {b: [bogs]}}\n", }, @@ -107,7 +107,7 @@ var assignOperatorScenarios = []expressionScenario{ { skipDoc: true, document: `{}`, - expression: `.a.b[1].c |= "bogs"`, + expression: `.a.b.[1].c |= "bogs"`, expected: []string{ "D0, P[], (doc)::{a: {b: [null, {c: bogs}]}}\n", }, diff --git a/pkg/yqlib/operator_collect_test.go b/pkg/yqlib/operator_collect_test.go index cadfef85..9441ae10 100644 --- a/pkg/yqlib/operator_collect_test.go +++ b/pkg/yqlib/operator_collect_test.go @@ -13,6 +13,14 @@ var collectOperatorScenarios = []expressionScenario{ "D0, P[], (!!seq)::[]\n", }, }, + { + skipDoc: true, + document: ``, + expression: `[3]`, + expected: []string{ + "D0, P[], (!!seq)::- 3\n", + }, + }, { description: "Collect single", document: ``, diff --git a/pkg/yqlib/operator_multiply.go b/pkg/yqlib/operator_multiply.go index a9657fbb..ad19fcf2 100644 --- a/pkg/yqlib/operator_multiply.go +++ b/pkg/yqlib/operator_multiply.go @@ -128,9 +128,7 @@ func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *Candid assignmentOp.OperationType = Assign assignmentOp.Preferences = &AssignOpPreferences{false} } else if shouldAppendArrays && rhs.Node.Kind == yaml.SequenceNode { - log.Debugf("append! lhs %v, rhs: %v", NodeToString(lhs), NodeToString(rhs)) - assignmentOp.OperationType = Add - assignmentOp.Preferences = &AddPreferences{InPlace: true} + assignmentOp.OperationType = AddAssign } rhsOp := &Operation{OperationType: ValueOp, CandidateNode: rhs} diff --git a/pkg/yqlib/operator_multiply_test.go b/pkg/yqlib/operator_multiply_test.go index 3007bdeb..d900c000 100644 --- a/pkg/yqlib/operator_multiply_test.go +++ b/pkg/yqlib/operator_multiply_test.go @@ -5,9 +5,6 @@ import ( ) var multiplyOperatorScenarios = []expressionScenario{ - { - description: "*+ doc", - }, { skipDoc: true, document: `{a: {also: [1]}, b: {also: me}}`, @@ -83,15 +80,15 @@ var multiplyOperatorScenarios = []expressionScenario{ description: "Merge keeps style of LHS", dontFormatInputForDoc: true, document: `a: {things: great} - b: - also: "me" - `, +b: + also: "me" +`, expression: `. * {"a":.b}`, expected: []string{ `D0, P[], (!!map)::a: {things: great, also: "me"} - b: - also: "me" - `, +b: + also: "me" +`, }, }, { diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go index 7c98f8aa..e39e5a62 100644 --- a/pkg/yqlib/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -159,7 +159,7 @@ var traversePathOperatorScenarios = []expressionScenario{ { description: "Traversing arrays by index", document: `[1,2,3]`, - expression: `[0]`, + expression: `.[0]`, expected: []string{ "D0, P[0], (!!int)::1\n", }, @@ -167,7 +167,7 @@ var traversePathOperatorScenarios = []expressionScenario{ { description: "Maps with numeric keys", document: `{2: cat}`, - expression: `[2]`, + expression: `.[2]`, expected: []string{ "D0, P[2], (!!str)::cat\n", }, @@ -175,7 +175,7 @@ var traversePathOperatorScenarios = []expressionScenario{ { description: "Maps with non existing numeric keys", document: `{a: b}`, - expression: `[0]`, + expression: `.[0]`, expected: []string{ "D0, P[0], (!!null)::null\n", }, diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index 15e52d74..083ce930 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -37,7 +37,7 @@ func testScenario(t *testing.T, s *expressionScenario) { if s.document != "" { inputs, err = readDocuments(strings.NewReader(s.document), "sample.yml", 0) if err != nil { - t.Error(err) + t.Error(err, s.document) return } } diff --git a/pkg/yqlib/path_parse_test.go b/pkg/yqlib/path_parse_test.go index f8c4decc..ab8ccfb5 100644 --- a/pkg/yqlib/path_parse_test.go +++ b/pkg/yqlib/path_parse_test.go @@ -49,6 +49,11 @@ var pathTests = []struct { append(make([]interface{}, 0), "[", "]"), append(make([]interface{}, 0), "EMPTY", "COLLECT", "PIPE"), }, + { + `[3]`, + append(make([]interface{}, 0), "[", "3 (int64)", "]"), + append(make([]interface{}, 0), "3 (int64)", "COLLECT", "PIPE"), + }, { `d0.a`, append(make([]interface{}, 0), "d0", "PIPE", "a"), diff --git a/pkg/yqlib/path_tokeniser.go b/pkg/yqlib/path_tokeniser.go index 5946156b..d712e191 100644 --- a/pkg/yqlib/path_tokeniser.go +++ b/pkg/yqlib/path_tokeniser.go @@ -223,7 +223,6 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`\s*\|=\s*`), opTokenWithPrefs(Assign, nil, &AssignOpPreferences{true})) - lexer.Add([]byte(`\[-?[0-9]+\]`), arrayIndextoken(false)) lexer.Add([]byte(`\.\[-?[0-9]+\]`), arrayIndextoken(true)) lexer.Add([]byte("( |\t|\n|\r)+"), skip) @@ -254,7 +253,7 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`\*`), opTokenWithPrefs(Multiply, nil, &MultiplyPreferences{AppendArrays: false})) lexer.Add([]byte(`\*\+`), opTokenWithPrefs(Multiply, nil, &MultiplyPreferences{AppendArrays: true})) lexer.Add([]byte(`\+`), opToken(Add)) - lexer.Add([]byte(`\+=`), opTokenWithPrefs(Add, nil, &AddPreferences{InPlace: true})) + lexer.Add([]byte(`\+=`), opToken(AddAssign)) err := lexer.Compile() if err != nil {