From ee8cc90307016db638ecb31ce0f0d49bcc0c2459 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Mon, 17 Apr 2023 16:20:00 +1000 Subject: [PATCH] getting closer --- pkg/yqlib/candidate_node.go | 10 + pkg/yqlib/doc/operators/add.md | 264 ++++++++ .../operators/create-collect-into-object.md | 92 +++ pkg/yqlib/operator_add.go | 11 +- pkg/yqlib/operator_add_test.go | 578 +++++++++--------- pkg/yqlib/operator_collect_object.go | 5 + pkg/yqlib/operator_collect_object_test.go | 54 +- pkg/yqlib/operator_create_map.go | 9 +- pkg/yqlib/operator_create_map_test.go | 16 + pkg/yqlib/operator_multiply.go | 13 +- 10 files changed, 724 insertions(+), 328 deletions(-) diff --git a/pkg/yqlib/candidate_node.go b/pkg/yqlib/candidate_node.go index 5bdb4bc0..82ceb5d2 100644 --- a/pkg/yqlib/candidate_node.go +++ b/pkg/yqlib/candidate_node.go @@ -120,6 +120,9 @@ func (n *CandidateNode) GetNiceTag() string { } func (n *CandidateNode) getParsedKey() interface{} { + if n.IsMapKey { + return n.Value + } if n.Key == nil { return nil } @@ -172,6 +175,13 @@ func (n *CandidateNode) AsList() *list.List { return elMap } +func (n *CandidateNode) AddKeyValueChild(rawKey *CandidateNode, rawValue *CandidateNode) { + value := rawValue.unwrapDocument() + value.Key = rawKey.unwrapDocument() + value.Key.IsMapKey = true + n.Content = append(n.Content, value.Key, value) +} + func (n *CandidateNode) GetValueRep() (interface{}, error) { log.Debugf("GetValueRep for %v value: %v", n.GetNicePath(), n.Value) realTag := n.guessTagFromCustomType() diff --git a/pkg/yqlib/doc/operators/add.md b/pkg/yqlib/doc/operators/add.md index 68c52e97..020c8ef6 100644 --- a/pkg/yqlib/doc/operators/add.md +++ b/pkg/yqlib/doc/operators/add.md @@ -9,3 +9,267 @@ Add behaves differently according to the type of the LHS: Use `+=` as a relative append assign for things like increment. Note that `.a += .x` is equivalent to running `.a = .a + .x`. +## Append to existing array +Note that the styling is copied from existing array elements + +Given a sample.yml file of: +```yaml +a: ['dog'] +``` +then +```bash +yq '.a += "cat"' sample.yml +``` +will output +```yaml +a: ['dog', 'cat'] +``` + +## Prepend to existing array +Given a sample.yml file of: +```yaml +a: + - dog +``` +then +```bash +yq '.a = ["cat"] + .a' sample.yml +``` +will output +```yaml +a: + - cat + - dog +``` + +## Add new object to array +Given a sample.yml file of: +```yaml +a: + - dog: woof +``` +then +```bash +yq '.a + {"cat": "meow"}' sample.yml +``` +will output +```yaml +- dog: woof +- cat: meow +``` + +## Relative append +Given a sample.yml file of: +```yaml +a: + a1: + b: + - cat + a2: + b: + - dog + a3: {} +``` +then +```bash +yq '.a[].b += ["mouse"]' sample.yml +``` +will output +```yaml +a: + a1: + b: + - cat + - mouse + a2: + b: + - dog + - mouse + a3: + b: + - mouse +``` + +## String concatenation +Given a sample.yml file of: +```yaml +a: cat +b: meow +``` +then +```bash +yq '.a += .b' sample.yml +``` +will output +```yaml +a: catmeow +b: meow +``` + +## Number addition - float +If the lhs or rhs are floats then the expression will be calculated with floats. + +Given a sample.yml file of: +```yaml +a: 3 +b: 4.9 +``` +then +```bash +yq '.a = .a + .b' sample.yml +``` +will output +```yaml +a: 7.9 +b: 4.9 +``` + +## Number addition - int +If both the lhs and rhs are ints then the expression will be calculated with ints. + +Given a sample.yml file of: +```yaml +a: 3 +b: 4 +``` +then +```bash +yq '.a = .a + .b' sample.yml +``` +will output +```yaml +a: 7 +b: 4 +``` + +## Increment numbers +Given a sample.yml file of: +```yaml +a: 3 +b: 5 +``` +then +```bash +yq '.[] += 1' sample.yml +``` +will output +```yaml +a: 4 +b: 6 +``` + +## Date addition +You can add durations to dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information. + +Given a sample.yml file of: +```yaml +a: 2021-01-01T00:00:00Z +``` +then +```bash +yq '.a += "3h10m"' sample.yml +``` +will output +```yaml +a: 2021-01-01T03:10:00Z +``` + +## Date addition - custom format +You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information. + +Given a sample.yml file of: +```yaml +a: Saturday, 15-Dec-01 at 2:59AM GMT +``` +then +```bash +yq 'with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")' sample.yml +``` +will output +```yaml +a: Saturday, 15-Dec-01 at 6:00AM GMT +``` + +## Add to null +Adding to null simply returns the rhs + +Running +```bash +yq --null-input 'null + "cat"' +``` +will output +```yaml +cat +``` + +## Add maps to shallow merge +Adding objects together shallow merges them. Use `*` to deeply merge. + +Given a sample.yml file of: +```yaml +a: + thing: + name: Astuff + value: x + a1: cool +b: + thing: + name: Bstuff + legs: 3 + b1: neat +``` +then +```bash +yq '.a += .b' sample.yml +``` +will output +```yaml +a: + thing: + name: Bstuff + legs: 3 + a1: cool + b1: neat +b: + thing: + name: Bstuff + legs: 3 + b1: neat +``` + +## Custom types: that are really strings +When custom tags are encountered, yq will try to decode the underlying type. + +Given a sample.yml file of: +```yaml +a: !horse cat +b: !goat _meow +``` +then +```bash +yq '.a += .b' sample.yml +``` +will output +```yaml +a: !horse cat_meow +b: !goat _meow +``` + +## Custom types: that are really numbers +When custom tags are encountered, yq will try to decode the underlying type. + +Given a sample.yml file of: +```yaml +a: !horse 1.2 +b: !goat 2.3 +``` +then +```bash +yq '.a += .b' sample.yml +``` +will output +```yaml +a: !horse 3.5 +b: !goat 2.3 +``` + diff --git a/pkg/yqlib/doc/operators/create-collect-into-object.md b/pkg/yqlib/doc/operators/create-collect-into-object.md index 3b24f3c7..ff0c9cbe 100644 --- a/pkg/yqlib/doc/operators/create-collect-into-object.md +++ b/pkg/yqlib/doc/operators/create-collect-into-object.md @@ -2,3 +2,95 @@ This is used to construct objects (or maps). This can be used against existing yaml, or to create fresh yaml documents. +## Collect empty object +Running +```bash +yq --null-input '{}' +``` +will output +```yaml +{} +``` + +## Wrap (prefix) existing object +Given a sample.yml file of: +```yaml +name: Mike +``` +then +```bash +yq '{"wrap": .}' sample.yml +``` +will output +```yaml +wrap: + name: Mike +``` + +## Using splat to create multiple objects +Given a sample.yml file of: +```yaml +name: Mike +pets: + - cat + - dog +``` +then +```bash +yq '{.name: .pets.[]}' sample.yml +``` +will output +```yaml +Mike: cat +Mike: dog +``` + +## Working with multiple documents +Given a sample.yml file of: +```yaml +name: Mike +pets: + - cat + - dog +--- +name: Rosey +pets: + - monkey + - sheep +``` +then +```bash +yq '{.name: .pets.[]}' sample.yml +``` +will output +```yaml +Mike: cat +Mike: dog +--- +Rosey: monkey +Rosey: sheep +``` + +## Creating yaml from scratch +Running +```bash +yq --null-input '{"wrap": "frog"}' +``` +will output +```yaml +wrap: frog +``` + +## Creating yaml from scratch with multiple objects +Running +```bash +yq --null-input '(.a.b = "foo") | (.d.e = "bar")' +``` +will output +```yaml +a: + b: foo +d: + e: bar +``` + diff --git a/pkg/yqlib/operator_add.go b/pkg/yqlib/operator_add.go index 2d119495..d19f9923 100644 --- a/pkg/yqlib/operator_add.go +++ b/pkg/yqlib/operator_add.go @@ -159,9 +159,9 @@ func addDateTimes(layout string, target *CandidateNode, lhs *CandidateNode, rhs func addSequences(target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error { log.Debugf("adding sequences! target: %v; lhs %v; rhs: %v", NodeToString(target), NodeToString(lhs), NodeToString(rhs)) target.Kind = SequenceNode - if len(lhs.Content) > 0 { - log.Debugf("copy lhs style") - target.Style = lhs.Style + if len(lhs.Content) == 0 { + log.Debugf("dont copy lhs style") + target.Style = 0 } target.Tag = lhs.Tag @@ -179,6 +179,11 @@ func addMaps(target *CandidateNode, lhsC *CandidateNode, rhsC *CandidateNode) { lhs := lhsC rhs := rhsC + if len(lhs.Content) == 0 { + log.Debugf("dont copy lhs style") + target.Style = 0 + } + target.Content = make([]*CandidateNode, len(lhs.Content)) copy(target.Content, lhs.Content) diff --git a/pkg/yqlib/operator_add_test.go b/pkg/yqlib/operator_add_test.go index 2cd5a66b..265de9e5 100644 --- a/pkg/yqlib/operator_add_test.go +++ b/pkg/yqlib/operator_add_test.go @@ -82,272 +82,272 @@ var addOperatorScenarios = []expressionScenario{ // "D0, P[a], (!!seq)::[1, 2]\n", // }, // }, - // { - // skipDoc: true, - // description: "Concatenate to empty array", - // document: `{a: []}`, - // expression: `.a + "cat"`, - // expected: []string{ - // "D0, P[a], (!!seq)::- cat\n", - // }, - // }, - // { - // description: "Append to existing array", - // subdescription: "Note that the styling is copied from existing array elements", - // dontFormatInputForDoc: true, - // document: `a: ['dog']`, - // expression: `.a += "cat"`, - // expected: []string{ - // "D0, P[], (doc)::a: ['dog', 'cat']\n", - // }, - // }, - // { - // description: "Prepend to existing array", - // document: `a: [dog]`, - // expression: `.a = ["cat"] + .a`, - // expected: []string{ - // "D0, P[], (doc)::a: [cat, dog]\n", - // }, - // }, - // { - // skipDoc: true, - // description: "Concatenate to existing array", - // subdescription: "does not modify original", - // document: `{a: ['dog'], b: cat}`, - // expression: `.a = .a + .b`, - // expected: []string{ - // "D0, P[], (doc)::{a: ['dog', 'cat'], b: cat}\n", - // }, - // }, - // { - // skipDoc: true, - // description: "Concatenate to empty array", - // document: `a: []`, - // expression: `.a += "cat"`, - // expected: []string{ - // "D0, P[], (doc)::a:\n - cat\n", - // }, - // }, - // { - // skipDoc: true, - // description: "Concatenate to existing array", - // document: `a: [dog]`, - // expression: `.a += "cat"`, - // expected: []string{ - // "D0, P[], (doc)::a: [dog, cat]\n", - // }, - // }, - // { - // skipDoc: true, - // description: "Concatenate to empty object", - // document: `{a: {}}`, - // expression: `.a + {"b": "cat"}`, - // expected: []string{ - // "D0, P[a], (!!map)::b: cat\n", - // }, - // }, - // { - // skipDoc: true, - // description: "Concatenate to existing object", - // document: `{a: {c: dog}}`, - // expression: `.a + {"b": "cat"}`, - // expected: []string{ - // "D0, P[a], (!!map)::{c: dog, b: cat}\n", - // }, - // }, - // { - // skipDoc: true, - // description: "Concatenate to existing object", - // subdescription: "matches stylig", - // document: "a:\n c: dog", - // expression: `.a + {"b": "cat"}`, - // expected: []string{ - // "D0, P[a], (!!map)::c: dog\nb: cat\n", - // }, - // }, - // { - // skipDoc: true, - // description: "Concatenate to empty object in place", - // document: `a: {}`, - // expression: `.a += {"b": "cat"}`, - // expected: []string{ - // "D0, P[], (doc)::a:\n b: cat\n", - // }, - // }, - // { - // skipDoc: true, - // description: "Concatenate to existing object in place", - // document: `a: {c: dog}`, - // expression: `.a += {"b": "cat"}`, - // expected: []string{ - // "D0, P[], (doc)::a: {c: dog, b: cat}\n", - // }, - // }, - // { - // description: "Add new object to array", - // document: `a: [{dog: woof}]`, - // expression: `.a + {"cat": "meow"}`, - // expected: []string{ - // "D0, P[a], (!!seq)::[{dog: woof}, {cat: meow}]\n", - // }, - // }, - // { - // description: "Relative append", - // document: `a: { a1: {b: [cat]}, a2: {b: [dog]}, a3: {} }`, - // expression: `.a[].b += ["mouse"]`, - // expected: []string{ - // "D0, P[], (doc)::a: {a1: {b: [cat, mouse]}, a2: {b: [dog, mouse]}, a3: {b: [mouse]}}\n", - // }, - // }, - // { - // description: "String concatenation", - // document: `{a: cat, b: meow}`, - // expression: `.a += .b`, - // expected: []string{ - // "D0, P[], (doc)::{a: catmeow, b: meow}\n", - // }, - // }, - // { - // description: "String concatenation - str + int", - // skipDoc: true, - // document: `{a: !cool cat, b: meow}`, - // expression: `.a + 3`, - // expected: []string{ - // "D0, P[a], (!cool)::cat3\n", - // }, - // }, - // { - // description: "String concatenation - int + str", - // skipDoc: true, - // document: `{a: !cool cat, b: meow}`, - // expression: `3 + .a`, - // expected: []string{ - // "D0, P[], (!cool)::3cat\n", - // }, - // }, - // { - // description: "Number addition - float", - // subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.", - // document: `{a: 3, b: 4.9}`, - // expression: `.a = .a + .b`, - // expected: []string{ - // "D0, P[], (doc)::{a: 7.9, b: 4.9}\n", - // }, - // }, - // { - // description: "Number addition - int", - // subdescription: "If both the lhs and rhs are ints then the expression will be calculated with ints.", - // document: `{a: 3, b: 4}`, - // expression: `.a = .a + .b`, - // expected: []string{ - // "D0, P[], (doc)::{a: 7, b: 4}\n", - // }, - // }, - // { - // description: "Increment numbers", - // document: `{a: 3, b: 5}`, - // expression: `.[] += 1`, - // expected: []string{ - // "D0, P[], (doc)::{a: 4, b: 6}\n", - // }, - // }, - // { - // description: "Date addition", - // subdescription: "You can add durations to dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", - // document: `a: 2021-01-01T00:00:00Z`, - // expression: `.a += "3h10m"`, - // expected: []string{ - // "D0, P[], (doc)::a: 2021-01-01T03:10:00Z\n", - // }, - // }, - // { - // description: "Date addition -date only", - // skipDoc: true, - // document: `a: 2021-01-01`, - // expression: `.a += "24h"`, - // expected: []string{ - // "D0, P[], (doc)::a: 2021-01-02T00:00:00Z\n", - // }, - // }, - // { - // description: "Date addition - custom format", - // subdescription: "You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", - // document: `a: Saturday, 15-Dec-01 at 2:59AM GMT`, - // expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")`, - // expected: []string{ - // "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 6:00AM GMT\n", - // }, - // }, - // { - // skipDoc: true, - // description: "Date addition - custom format", - // subdescription: "You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", - // document: `a: !cat Saturday, 15-Dec-01 at 2:59AM GMT`, - // expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")`, - // expected: []string{ - // "D0, P[], (doc)::a: !cat Saturday, 15-Dec-01 at 6:00AM GMT\n", - // }, - // }, - // { - // description: "Add to null", - // subdescription: "Adding to null simply returns the rhs", - // expression: `null + "cat"`, - // expected: []string{ - // "D0, P[], (!!str)::cat\n", - // }, - // }, - // { - // description: "Add maps to shallow merge", - // subdescription: "Adding objects together shallow merges them. Use `*` to deeply merge.", - // document: "a: {thing: {name: Astuff, value: x}, a1: cool}\nb: {thing: {name: Bstuff, legs: 3}, b1: neat}", - // expression: `.a += .b`, - // expected: []string{ - // "D0, P[], (doc)::a: {thing: {name: Bstuff, legs: 3}, a1: cool, b1: neat}\nb: {thing: {name: Bstuff, legs: 3}, b1: neat}\n", - // }, - // }, - // { - // description: "Custom types: that are really strings", - // subdescription: "When custom tags are encountered, yq will try to decode the underlying type.", - // document: "a: !horse cat\nb: !goat _meow", - // expression: `.a += .b`, - // expected: []string{ - // "D0, P[], (doc)::a: !horse cat_meow\nb: !goat _meow\n", - // }, - // }, - // { - // description: "Custom types: that are really numbers", - // subdescription: "When custom tags are encountered, yq will try to decode the underlying type.", - // document: "a: !horse 1.2\nb: !goat 2.3", - // expression: `.a += .b`, - // expected: []string{ - // "D0, P[], (doc)::a: !horse 3.5\nb: !goat 2.3\n", - // }, - // }, - // { - // skipDoc: true, - // document: "a: !horse 2\nb: !goat 2.3", - // expression: `.a += .b`, - // expected: []string{ - // "D0, P[], (doc)::a: !horse 4.3\nb: !goat 2.3\n", - // }, - // }, - // { - // skipDoc: true, - // document: "a: 2\nb: !goat 2.3", - // expression: `.a += .b`, - // expected: []string{ - // "D0, P[], (doc)::a: 4.3\nb: !goat 2.3\n", - // }, - // }, - // { - // skipDoc: true, - // description: "Custom types: that are really ints", - // document: "a: !horse 2\nb: !goat 3", - // expression: `.a += .b`, - // expected: []string{ - // "D0, P[], (doc)::a: !horse 5\nb: !goat 3\n", - // }, - // }, + { + skipDoc: true, + description: "Concatenate to empty array", + document: `{a: []}`, + expression: `.a + "cat"`, + expected: []string{ + "D0, P[a], (!!seq)::- cat\n", + }, + }, + { + description: "Append to existing array", + subdescription: "Note that the styling is copied from existing array elements", + dontFormatInputForDoc: true, + document: `a: ['dog']`, + expression: `.a += "cat"`, + expected: []string{ + "D0, P[], (doc)::a: ['dog', 'cat']\n", + }, + }, + { + description: "Prepend to existing array", + document: `a: [dog]`, + expression: `.a = ["cat"] + .a`, + expected: []string{ + "D0, P[], (doc)::a: [cat, dog]\n", + }, + }, + { + skipDoc: true, + description: "Concatenate to existing array", + subdescription: "does not modify original", + document: `{a: ['dog'], b: cat}`, + expression: `.a = .a + .b`, + expected: []string{ + "D0, P[], (doc)::{a: ['dog', 'cat'], b: cat}\n", + }, + }, + { + skipDoc: true, + description: "Concatenate to empty array", + document: `a: []`, + expression: `.a += "cat"`, + expected: []string{ + "D0, P[], (doc)::a:\n - cat\n", + }, + }, + { + skipDoc: true, + description: "Concatenate to existing array", + document: `a: [dog]`, + expression: `.a += "cat"`, + expected: []string{ + "D0, P[], (doc)::a: [dog, cat]\n", + }, + }, + { + skipDoc: true, + description: "Concatenate to empty object", + document: `{a: {}}`, + expression: `.a + {"b": "cat"}`, + expected: []string{ + "D0, P[a], (!!map)::b: cat\n", + }, + }, + { + skipDoc: true, + description: "Concatenate to existing object", + document: `{a: {c: dog}}`, + expression: `.a + {"b": "cat"}`, + expected: []string{ + "D0, P[a], (!!map)::{c: dog, b: cat}\n", + }, + }, + { + skipDoc: true, + description: "Concatenate to existing object", + subdescription: "matches stylig", + document: "a:\n c: dog", + expression: `.a + {"b": "cat"}`, + expected: []string{ + "D0, P[a], (!!map)::c: dog\nb: cat\n", + }, + }, + { + skipDoc: true, + description: "Concatenate to empty object in place", + document: `a: {}`, + expression: `.a += {"b": "cat"}`, + expected: []string{ + "D0, P[], (doc)::a:\n b: cat\n", + }, + }, + { + skipDoc: true, + description: "Concatenate to existing object in place", + document: `a: {c: dog}`, + expression: `.a += {"b": "cat"}`, + expected: []string{ + "D0, P[], (doc)::a: {c: dog, b: cat}\n", + }, + }, + { + description: "Add new object to array", + document: `a: [{dog: woof}]`, + expression: `.a + {"cat": "meow"}`, + expected: []string{ + "D0, P[a], (!!seq)::[{dog: woof}, {cat: meow}]\n", + }, + }, + { + description: "Relative append", + document: `a: { a1: {b: [cat]}, a2: {b: [dog]}, a3: {} }`, + expression: `.a[].b += ["mouse"]`, + expected: []string{ + "D0, P[], (doc)::a: {a1: {b: [cat, mouse]}, a2: {b: [dog, mouse]}, a3: {b: [mouse]}}\n", + }, + }, + { + description: "String concatenation", + document: `{a: cat, b: meow}`, + expression: `.a += .b`, + expected: []string{ + "D0, P[], (doc)::{a: catmeow, b: meow}\n", + }, + }, + { + description: "String concatenation - str + int", + skipDoc: true, + document: `{a: !cool cat, b: meow}`, + expression: `.a + 3`, + expected: []string{ + "D0, P[a], (!cool)::cat3\n", + }, + }, + { + description: "String concatenation - int + str", + skipDoc: true, + document: `{a: !cool cat, b: meow}`, + expression: `3 + .a`, + expected: []string{ + "D0, P[], (!cool)::3cat\n", + }, + }, + { + description: "Number addition - float", + subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.", + document: `{a: 3, b: 4.9}`, + expression: `.a = .a + .b`, + expected: []string{ + "D0, P[], (doc)::{a: 7.9, b: 4.9}\n", + }, + }, + { + description: "Number addition - int", + subdescription: "If both the lhs and rhs are ints then the expression will be calculated with ints.", + document: `{a: 3, b: 4}`, + expression: `.a = .a + .b`, + expected: []string{ + "D0, P[], (doc)::{a: 7, b: 4}\n", + }, + }, + { + description: "Increment numbers", + document: `{a: 3, b: 5}`, + expression: `.[] += 1`, + expected: []string{ + "D0, P[], (doc)::{a: 4, b: 6}\n", + }, + }, + { + description: "Date addition", + subdescription: "You can add durations to dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", + document: `a: 2021-01-01T00:00:00Z`, + expression: `.a += "3h10m"`, + expected: []string{ + "D0, P[], (doc)::a: 2021-01-01T03:10:00Z\n", + }, + }, + { + description: "Date addition -date only", + skipDoc: true, + document: `a: 2021-01-01`, + expression: `.a += "24h"`, + expected: []string{ + "D0, P[], (doc)::a: 2021-01-02T00:00:00Z\n", + }, + }, + { + description: "Date addition - custom format", + subdescription: "You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", + document: `a: Saturday, 15-Dec-01 at 2:59AM GMT`, + expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")`, + expected: []string{ + "D0, P[], (doc)::a: Saturday, 15-Dec-01 at 6:00AM GMT\n", + }, + }, + { + skipDoc: true, + description: "Date addition - custom format", + subdescription: "You can add durations to dates. See [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.", + document: `a: !cat Saturday, 15-Dec-01 at 2:59AM GMT`, + expression: `with_dtf("Monday, 02-Jan-06 at 3:04PM MST", .a += "3h1m")`, + expected: []string{ + "D0, P[], (doc)::a: !cat Saturday, 15-Dec-01 at 6:00AM GMT\n", + }, + }, + { + description: "Add to null", + subdescription: "Adding to null simply returns the rhs", + expression: `null + "cat"`, + expected: []string{ + "D0, P[], (!!str)::cat\n", + }, + }, + { + description: "Add maps to shallow merge", + subdescription: "Adding objects together shallow merges them. Use `*` to deeply merge.", + document: "a: {thing: {name: Astuff, value: x}, a1: cool}\nb: {thing: {name: Bstuff, legs: 3}, b1: neat}", + expression: `.a += .b`, + expected: []string{ + "D0, P[], (doc)::a: {thing: {name: Bstuff, legs: 3}, a1: cool, b1: neat}\nb: {thing: {name: Bstuff, legs: 3}, b1: neat}\n", + }, + }, + { + description: "Custom types: that are really strings", + subdescription: "When custom tags are encountered, yq will try to decode the underlying type.", + document: "a: !horse cat\nb: !goat _meow", + expression: `.a += .b`, + expected: []string{ + "D0, P[], (doc)::a: !horse cat_meow\nb: !goat _meow\n", + }, + }, + { + description: "Custom types: that are really numbers", + subdescription: "When custom tags are encountered, yq will try to decode the underlying type.", + document: "a: !horse 1.2\nb: !goat 2.3", + expression: `.a += .b`, + expected: []string{ + "D0, P[], (doc)::a: !horse 3.5\nb: !goat 2.3\n", + }, + }, + { + skipDoc: true, + document: "a: !horse 2\nb: !goat 2.3", + expression: `.a += .b`, + expected: []string{ + "D0, P[], (doc)::a: !horse 4.3\nb: !goat 2.3\n", + }, + }, + { + skipDoc: true, + document: "a: 2\nb: !goat 2.3", + expression: `.a += .b`, + expected: []string{ + "D0, P[], (doc)::a: 4.3\nb: !goat 2.3\n", + }, + }, + { + skipDoc: true, + description: "Custom types: that are really ints", + document: "a: !horse 2\nb: !goat 3", + expression: `.a += .b`, + expected: []string{ + "D0, P[], (doc)::a: !horse 5\nb: !goat 3\n", + }, + }, { description: "Custom types: that are really arrays", skipDoc: true, @@ -358,29 +358,29 @@ var addOperatorScenarios = []expressionScenario{ "D0, P[], (doc)::a: !horse [a, b]\nb: !goat [b]\n", }, }, - // { - // skipDoc: true, - // description: "Keep anchors", - // document: "a: &horse [1]", - // expression: `.a += 2`, - // expected: []string{ - // "D0, P[], (doc)::a: &horse [1, 2]\n", - // }, - // }, - // { - // skipDoc: true, - // description: "Add sequence to map", - // document: "a: {x: cool}", - // expression: `.a += [2]`, - // expectedError: "!!seq () cannot be added to a !!map (a)", - // }, - // { - // skipDoc: true, - // description: "Add sequence to scalar", - // document: "a: cool", - // expression: `.a += [2]`, - // expectedError: "!!seq () cannot be added to a !!str (a)", - // }, + { + skipDoc: true, + description: "Keep anchors", + document: "a: &horse [1]", + expression: `.a += 2`, + expected: []string{ + "D0, P[], (doc)::a: &horse [1, 2]\n", + }, + }, + { + skipDoc: true, + description: "Add sequence to map", + document: "a: {x: cool}", + expression: `.a += [2]`, + expectedError: "!!seq () cannot be added to a !!map (a)", + }, + { + skipDoc: true, + description: "Add sequence to scalar", + document: "a: cool", + expression: `.a += [2]`, + expectedError: "!!seq () cannot be added to a !!str (a)", + }, } func TestAddOperatorScenarios(t *testing.T) { diff --git a/pkg/yqlib/operator_collect_object.go b/pkg/yqlib/operator_collect_object.go index d767f7d7..6f8f5922 100644 --- a/pkg/yqlib/operator_collect_object.go +++ b/pkg/yqlib/operator_collect_object.go @@ -22,6 +22,7 @@ func collectObjectOperator(d *dataTreeNavigator, originalContext Context, expres if context.MatchingNodes.Len() == 0 { candidate := &CandidateNode{Kind: MappingNode, Tag: "!!map", Value: "{}"} + log.Debugf("-- collectObjectOperation - starting with empty map") return context.SingleChildContext(candidate), nil } first := context.MatchingNodes.Front().Value.(*CandidateNode) @@ -38,6 +39,7 @@ func collectObjectOperator(d *dataTreeNavigator, originalContext Context, expres rotated[i].PushBack(candidateNode.Content[i]) } } + log.Debugf("-- collectObjectOperation, lenght of rotated is %v", len(rotated)) newObject := list.New() for i := 0; i < len(first.Content); i++ { @@ -58,6 +60,7 @@ func collect(d *dataTreeNavigator, context Context, remainingMatches *list.List) } candidate := remainingMatches.Remove(remainingMatches.Front()).(*CandidateNode) + log.Debugf("-- collectObjectOperation - collect %v", NodeToString(candidate)) splatted, err := splat(context.SingleChildContext(candidate), traversePreferences{DontFollowAlias: true, IncludeMapKeys: false}) @@ -67,6 +70,7 @@ func collect(d *dataTreeNavigator, context Context, remainingMatches *list.List) } if context.MatchingNodes.Len() == 0 { + log.Debugf("-- collectObjectOperation - collect context is empty, next") return collect(d, splatted, remainingMatches) } @@ -79,6 +83,7 @@ func collect(d *dataTreeNavigator, context Context, remainingMatches *list.List) newCandidate := aggCandidate.Copy() newCandidate, err = multiply(multiplyPreferences{AppendArrays: false})(d, context, newCandidate, splatCandidate) + if err != nil { return Context{}, err } diff --git a/pkg/yqlib/operator_collect_object_test.go b/pkg/yqlib/operator_collect_object_test.go index fb6a3edd..0dd88044 100644 --- a/pkg/yqlib/operator_collect_object_test.go +++ b/pkg/yqlib/operator_collect_object_test.go @@ -5,22 +5,22 @@ import ( ) var collectObjectOperatorScenarios = []expressionScenario{ - // { - // skipDoc: true, - // document: `[{name: cat}, {name: dog}]`, - // expression: `.[] | {.name: "great"}`, - // expected: []string{ - // "D0, P[], (!!map)::cat: great\n", - // "D0, P[], (!!map)::dog: great\n", - // }, - // }, - // { - // skipDoc: true, - // expression: `({} + {}) | (.b = 3)`, - // expected: []string{ - // "D0, P[], (!!map)::b: 3\n", - // }, - // }, + { + skipDoc: true, + document: `[{name: cat}, {name: dog}]`, + expression: `.[] | {.name: "great"}`, + expected: []string{ + "D0, P[], (!!map)::cat: great\n", + "D0, P[], (!!map)::dog: great\n", + }, + }, + { + skipDoc: true, + expression: `({} + {}) | (.b = 3)`, + expected: []string{ + "D0, P[], (!!map)::b: 3\n", + }, + }, { skipDoc: true, document: "a: []", @@ -62,22 +62,24 @@ var collectObjectOperatorScenarios = []expressionScenario{ }, }, { - skipDoc: true, - document: "{name: Mike}\n", - document2: "{name: Bob}\n", - expression: `{"wrap": .}`, + skipDoc: true, + description: "Two documents", + document: "{name: Mike}\n", + document2: "{name: Bob}\n", + expression: `{"wrap": .}`, expected: []string{ "D0, P[], (!!map)::wrap: {name: Mike}\n", "D0, P[], (!!map)::wrap: {name: Bob}\n", }, }, { - skipDoc: true, - document: "{name: Mike}\n---\n{name: Bob}", - expression: `{"wrap": .}`, + skipDoc: true, + description: "two embedded documents", + document: "{name: Mike}\n---\n{name: Bob}", + expression: `{"wrap": .}`, expected: []string{ "D0, P[], (!!map)::wrap: {name: Mike}\n", - "D0, P[], (!!map)::wrap: {name: Bob}\n", + "D1, P[], (!!map)::wrap: {name: Bob}\n", }, }, { @@ -105,8 +107,8 @@ var collectObjectOperatorScenarios = []expressionScenario{ expected: []string{ "D0, P[], (!!map)::Mike: cat\n", "D0, P[], (!!map)::Mike: dog\n", - "D0, P[], (!!map)::Rosey: monkey\n", - "D0, P[], (!!map)::Rosey: sheep\n", + "D1, P[], (!!map)::Rosey: monkey\n", + "D1, P[], (!!map)::Rosey: sheep\n", }, }, { diff --git a/pkg/yqlib/operator_create_map.go b/pkg/yqlib/operator_create_map.go index 4390ce60..ac9b17a3 100644 --- a/pkg/yqlib/operator_create_map.go +++ b/pkg/yqlib/operator_create_map.go @@ -51,12 +51,9 @@ func sequenceFor(d *dataTreeNavigator, context Context, matchingNode *CandidateN mapPairs, err := crossFunction(d, context.ChildContext(matches), expressionNode, func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { node := CandidateNode{Kind: MappingNode, Tag: "!!map"} - log.Debugf("LHS:", NodeToString(lhs)) - log.Debugf("RHS:", NodeToString(rhs)) - node.Content = []*CandidateNode{ - lhs.unwrapDocument(), - rhs.unwrapDocument(), - } + + node.AddKeyValueChild(lhs, rhs) + node.Document = document return &node, nil diff --git a/pkg/yqlib/operator_create_map_test.go b/pkg/yqlib/operator_create_map_test.go index 6a972467..e01bf778 100644 --- a/pkg/yqlib/operator_create_map_test.go +++ b/pkg/yqlib/operator_create_map_test.go @@ -12,6 +12,22 @@ var createMapOperatorScenarios = []expressionScenario{ "D0, P[], (!!seq)::- [{frog: jumps}]\n", }, }, + { + skipDoc: true, + description: "sets key properly", + expression: `("frog": "jumps") | .[0][0] | .frog`, + expected: []string{ + "D0, P[frog], (!!str)::jumps\n", + }, + }, + { + skipDoc: true, + description: "sets key properly on map", + expression: `{"frog": "jumps"} | .frog`, + expected: []string{ + "D0, P[frog], (!!str)::jumps\n", + }, + }, { document: `{name: Mike, age: 32}`, expression: `.name: .age`, diff --git a/pkg/yqlib/operator_multiply.go b/pkg/yqlib/operator_multiply.go index 181696fc..487837c4 100644 --- a/pkg/yqlib/operator_multiply.go +++ b/pkg/yqlib/operator_multiply.go @@ -54,8 +54,8 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex leadingContent, headComment, footComment := getComments(lhs, rhs) lhs = lhs.unwrapDocument() rhs = rhs.unwrapDocument() - log.Debugf("Multiplying LHS: %v", lhs.Tag) - log.Debugf("- RHS: %v", rhs.Tag) + log.Debugf("Multiplying LHS: %v", NodeToString(lhs)) + log.Debugf("- RHS: %v", NodeToString(rhs)) if rhs.Tag == "!!null" { return lhs.Copy(), nil @@ -93,7 +93,7 @@ func multiplyScalars(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, er } else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") { return multiplyFloats(lhs, rhs, lhsIsCustom) } - return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Tag, rhs.Tag) + return nil, fmt.Errorf("cannot multiply %v with %v", lhs.Tag, rhs.Tag) } func multiplyFloats(lhs *CandidateNode, rhs *CandidateNode, lhsIsCustom bool) (*CandidateNode, error) { @@ -152,10 +152,14 @@ func mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs var pathIndexToStartFrom int if results.Front() != nil { pathIndexToStartFrom = len(results.Front().Value.(*CandidateNode).GetPath()) + log.Debugf("pathIndexToStartFrom: %v", pathIndexToStartFrom) } for el := results.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) + + log.Debugf("*** going to applied assignment to LHS: %v with RHS: %v", NodeToString(lhs), NodeToString(candidate)) + if candidate.Tag == "!!merge" { continue } @@ -164,13 +168,14 @@ func mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs if err != nil { return nil, err } + + log.Debugf("*** applied assignment to LHS: %v", NodeToString(lhs)) } return lhs, nil } func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode, preferences multiplyPreferences) error { shouldAppendArrays := preferences.AppendArrays - log.Debugf("merge - applyAssignment lhs %v, rhs: %v", lhs.GetKey(), rhs.GetKey()) lhsPath := rhs.GetPath()[pathIndexToStartFrom:] log.Debugf("merge - lhsPath %v", lhsPath)