From a6c79f3410075f51da84aea5df1487434337feaf Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sat, 22 Jan 2022 13:47:22 +1100 Subject: [PATCH] Multiply, substract with custom types --- pkg/yqlib/candidate_node.go | 7 ++- pkg/yqlib/doc/operators/add.md | 6 ++- pkg/yqlib/doc/operators/headers/add.md | 2 + pkg/yqlib/doc/operators/multiply-merge.md | 41 +++++++++++++++++ pkg/yqlib/doc/operators/subtract.md | 18 ++++++++ pkg/yqlib/lib.go | 27 +++++++++++- pkg/yqlib/operator_add.go | 7 ++- pkg/yqlib/operator_add_test.go | 29 +++++++++++- pkg/yqlib/operator_multiply.go | 40 +++++++++++++---- pkg/yqlib/operator_multiply_test.go | 54 +++++++++++++++++++++++ pkg/yqlib/operator_subtract.go | 28 +++++++++--- pkg/yqlib/operator_subtract_test.go | 28 ++++++++++++ 12 files changed, 266 insertions(+), 21 deletions(-) diff --git a/pkg/yqlib/candidate_node.go b/pkg/yqlib/candidate_node.go index 2bfbc586..298c4efd 100644 --- a/pkg/yqlib/candidate_node.go +++ b/pkg/yqlib/candidate_node.go @@ -136,7 +136,12 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignP n.Node.Value = "" } n.Node.Kind = other.Node.Kind - n.Node.Tag = other.Node.Tag + + // don't clobber custom tags... + if strings.HasPrefix(n.Node.Tag, "!!") || n.Node.Tag == "" { + n.Node.Tag = other.Node.Tag + } + n.Node.Alias = other.Node.Alias if !prefs.DontOverWriteAnchor { diff --git a/pkg/yqlib/doc/operators/add.md b/pkg/yqlib/doc/operators/add.md index 5a14444f..d7da2453 100644 --- a/pkg/yqlib/doc/operators/add.md +++ b/pkg/yqlib/doc/operators/add.md @@ -7,6 +7,8 @@ Add behaves differently according to the type of the LHS: Use `+=` as append assign for things like increment. Note that `.a += .x` is equivalent to running `.a = .a + .x`. +Add is not (yet) supported for maps - however you can use merge `*` which will have a similar effect... + ## Concatenate and assign arrays Given a sample.yml file of: ```yaml @@ -266,7 +268,7 @@ cat ``` ## Custom types: that are really strings -when custom tags are encountered, yq will try to decode the underlying type. +When custom tags are encountered, yq will try to decode the underlying type. Given a sample.yml file of: ```yaml @@ -284,7 +286,7 @@ b: !goat _meow ``` ## Custom types: that are really numbers -when custom tags are encountered, yq will try to decode the underlying type. +When custom tags are encountered, yq will try to decode the underlying type. Given a sample.yml file of: ```yaml diff --git a/pkg/yqlib/doc/operators/headers/add.md b/pkg/yqlib/doc/operators/headers/add.md index de4caf8c..5a3b2040 100644 --- a/pkg/yqlib/doc/operators/headers/add.md +++ b/pkg/yqlib/doc/operators/headers/add.md @@ -6,3 +6,5 @@ Add behaves differently according to the type of the LHS: * string scalars: concatenate Use `+=` as append assign for things like increment. Note that `.a += .x` is equivalent to running `.a = .a + .x`. + +Add is not (yet) supported for maps - however you can use merge `*` which will have a similar effect... diff --git a/pkg/yqlib/doc/operators/multiply-merge.md b/pkg/yqlib/doc/operators/multiply-merge.md index 1fea5319..e83936bc 100644 --- a/pkg/yqlib/doc/operators/multiply-merge.md +++ b/pkg/yqlib/doc/operators/multiply-merge.md @@ -410,3 +410,44 @@ thing: foobar_thing b: foobarList_b ``` +## Custom types: that are really numbers +When custom tags are encountered, yq will try to decode the underlying type. + +Given a sample.yml file of: +```yaml +a: !horse 2 +b: !goat 3 +``` +then +```bash +yq eval '.a = .a * .b' sample.yml +``` +will output +```yaml +a: !horse 6 +b: !goat 3 +``` + +## Custom types: that are really maps +Custom tags will be maintained. + +Given a sample.yml file of: +```yaml +a: !horse + cat: meow +b: !goat + dog: woof +``` +then +```bash +yq eval '.a = .a * .b' sample.yml +``` +will output +```yaml +a: !horse + cat: meow + dog: woof +b: !goat + dog: woof +``` + diff --git a/pkg/yqlib/doc/operators/subtract.md b/pkg/yqlib/doc/operators/subtract.md index bcc93615..3c2a46c2 100644 --- a/pkg/yqlib/doc/operators/subtract.md +++ b/pkg/yqlib/doc/operators/subtract.md @@ -111,3 +111,21 @@ a: 2 b: 4 ``` +## Custom types: that are really numbers +When custom tags are encountered, yq will try to decode the underlying type. + +Given a sample.yml file of: +```yaml +a: !horse 2 +b: !goat 1 +``` +then +```bash +yq eval '.a -= .b' sample.yml +``` +will output +```yaml +a: !horse 1 +b: !goat 1 +``` + diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 54ce7e0e..36d76798 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -190,9 +190,32 @@ func recurseNodeObjectEqual(lhs *yaml.Node, rhs *yaml.Node) bool { } func recursiveNodeEqual(lhs *yaml.Node, rhs *yaml.Node) bool { - if lhs.Kind != rhs.Kind || lhs.Tag != rhs.Tag { + if lhs.Kind != rhs.Kind { return false - } else if lhs.Tag == "!!null" { + } + + if lhs.Kind == yaml.ScalarNode { + //process custom tags of scalar nodes. + //dont worry about matching tags of maps or arrays. + + lhsTag := lhs.Tag + rhsTag := rhs.Tag + if !strings.HasPrefix(lhsTag, "!!") { + // custom tag - we have to have a guess + lhsTag = guessTagFromCustomType(lhs) + } + + if !strings.HasPrefix(rhsTag, "!!") { + // custom tag - we have to have a guess + rhsTag = guessTagFromCustomType(rhs) + } + + if lhsTag != rhsTag { + return false + } + } + + if lhs.Tag == "!!null" { return true } else if lhs.Kind == yaml.ScalarNode { diff --git a/pkg/yqlib/operator_add.go b/pkg/yqlib/operator_add.go index d09defac..8470c61c 100644 --- a/pkg/yqlib/operator_add.go +++ b/pkg/yqlib/operator_add.go @@ -71,12 +71,17 @@ func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Candida } func guessTagFromCustomType(node *yaml.Node) string { + if node.Value == "" { + log.Warning("node has no value to guess the type with") + return node.Tag + } + decoder := NewYamlDecoder() decoder.Init(strings.NewReader(node.Value)) var dataBucket yaml.Node errorReading := decoder.Decode(&dataBucket) if errorReading != nil { - log.Warning("could not guess underlying tag type %w", errorReading) + log.Warning("could not guess underlying tag type %v", errorReading) return node.Tag } guessedTag := unwrapDoc(&dataBucket).Tag diff --git a/pkg/yqlib/operator_add_test.go b/pkg/yqlib/operator_add_test.go index 8a42820f..df1a605f 100644 --- a/pkg/yqlib/operator_add_test.go +++ b/pkg/yqlib/operator_add_test.go @@ -146,7 +146,7 @@ var addOperatorScenarios = []expressionScenario{ }, { description: "Custom types: that are really strings", - subdescription: "when custom tags are encountered, yq will try to decode the underlying type.", + subdescription: "When custom tags are encountered, yq will try to decode the underlying type.", document: "a: !horse cat\nb: !goat _meow", expression: `.a += .b`, expected: []string{ @@ -155,13 +155,38 @@ var addOperatorScenarios = []expressionScenario{ }, { description: "Custom types: that are really numbers", - subdescription: "when custom tags are encountered, yq will try to decode the underlying type.", + subdescription: "When custom tags are encountered, yq will try to decode the underlying type.", document: "a: !horse 1.2\nb: !goat 2.3", expression: `.a += .b`, expected: []string{ "D0, P[], (doc)::a: !horse 3.5\nb: !goat 2.3\n", }, }, + { + skipDoc: true, + document: "a: !horse 2\nb: !goat 2.3", + expression: `.a += .b`, + expected: []string{ + "D0, P[], (doc)::a: !horse 4.3\nb: !goat 2.3\n", + }, + }, + { + skipDoc: true, + document: "a: 2\nb: !goat 2.3", + expression: `.a += .b`, + expected: []string{ + "D0, P[], (doc)::a: 4.3\nb: !goat 2.3\n", + }, + }, + { + skipDoc: true, + description: "Custom types: that are really ints", + document: "a: !horse 2\nb: !goat 3", + expression: `.a += .b`, + expected: []string{ + "D0, P[], (doc)::a: !horse 5\nb: !goat 3\n", + }, + }, { description: "Custom types: that are really arrays", skipDoc: true, diff --git a/pkg/yqlib/operator_multiply.go b/pkg/yqlib/operator_multiply.go index 8575c9ea..040cb7c0 100644 --- a/pkg/yqlib/operator_multiply.go +++ b/pkg/yqlib/operator_multiply.go @@ -4,6 +4,7 @@ import ( "container/list" "fmt" "strconv" + "strings" "github.com/jinzhu/copier" yaml "gopkg.in/yaml.v3" @@ -57,20 +58,43 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex newBlank.Node.FootComment = footComment return mergeObjects(d, context.WritableClone(), &newBlank, rhs, preferences) - } else if lhs.Node.Tag == "!!int" && rhs.Node.Tag == "!!int" { - return multiplyIntegers(lhs, rhs) - } else if (lhs.Node.Tag == "!!int" || lhs.Node.Tag == "!!float") && (rhs.Node.Tag == "!!int" || rhs.Node.Tag == "!!float") { - return multiplyFloats(lhs, rhs) } - return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag) + return multiplyScalars(lhs, rhs) } } -func multiplyFloats(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { +func multiplyScalars(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { + lhsTag := lhs.Node.Tag + rhsTag := rhs.Node.Tag + lhsIsCustom := false + if !strings.HasPrefix(lhsTag, "!!") { + // custom tag - we have to have a guess + lhsTag = guessTagFromCustomType(lhs.Node) + lhsIsCustom = true + } + + if !strings.HasPrefix(rhsTag, "!!") { + // custom tag - we have to have a guess + rhsTag = guessTagFromCustomType(rhs.Node) + } + + if lhsTag == "!!int" && rhsTag == "!!int" { + return multiplyIntegers(lhs, rhs) + } else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") { + return multiplyFloats(lhs, rhs, lhsIsCustom) + } + return nil, fmt.Errorf("Cannot multiply %v with %v", lhs.Node.Tag, rhs.Node.Tag) +} + +func multiplyFloats(lhs *CandidateNode, rhs *CandidateNode, lhsIsCustom bool) (*CandidateNode, error) { target := lhs.CreateReplacement(&yaml.Node{}) target.Node.Kind = yaml.ScalarNode target.Node.Style = lhs.Node.Style - target.Node.Tag = "!!float" + if lhsIsCustom { + target.Node.Tag = lhs.Node.Tag + } else { + target.Node.Tag = "!!float" + } lhsNum, err := strconv.ParseFloat(lhs.Node.Value, 64) if err != nil { @@ -88,7 +112,7 @@ func multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, e target := lhs.CreateReplacement(&yaml.Node{}) target.Node.Kind = yaml.ScalarNode target.Node.Style = lhs.Node.Style - target.Node.Tag = "!!int" + target.Node.Tag = lhs.Node.Tag format, lhsNum, err := parseInt(lhs.Node.Value) if err != nil { diff --git a/pkg/yqlib/operator_multiply_test.go b/pkg/yqlib/operator_multiply_test.go index b3711048..4f83bcd6 100644 --- a/pkg/yqlib/operator_multiply_test.go +++ b/pkg/yqlib/operator_multiply_test.go @@ -461,6 +461,60 @@ var multiplyOperatorScenarios = []expressionScenario{ "D0, P[b], (!!map)::{name: dog, <<: *cat}\n", }, }, + { + description: "Custom types: that are really numbers", + subdescription: "When custom tags are encountered, yq will try to decode the underlying type.", + document: "a: !horse 2\nb: !goat 3", + expression: ".a = .a * .b", + expected: []string{ + "D0, P[], (doc)::a: !horse 6\nb: !goat 3\n", + }, + }, + { + skipDoc: true, + description: "Custom types: that are really numbers", + document: "a: !horse 2.5\nb: !goat 3.5", + expression: ".a = .a * .b", + expected: []string{ + "D0, P[], (doc)::a: !horse 8.75\nb: !goat 3.5\n", + }, + }, + { + skipDoc: true, + description: "Custom types: that are really numbers", + document: "a: 2\nb: !goat 3.5", + expression: ".a = .a * .b", + expected: []string{ + "D0, P[], (doc)::a: !!float 7\nb: !goat 3.5\n", + }, + }, + { + skipDoc: true, + description: "Custom types: that are really arrays", + document: "a: !horse [1,2]\nb: !goat [3]", + expression: ".a = .a * .b", + expected: []string{ + "D0, P[], (doc)::a: !horse [3]\nb: !goat [3]\n", + }, + }, + { + description: "Custom types: that are really maps", + subdescription: "Custom tags will be maintained.", + document: "a: !horse {cat: meow}\nb: !goat {dog: woof}", + expression: ".a = .a * .b", + expected: []string{ + "D0, P[], (doc)::a: !horse {cat: meow, dog: woof}\nb: !goat {dog: woof}\n", + }, + }, + { + skipDoc: true, + description: "Custom types: that are really maps", + document: "a: {cat: !horse meow}\nb: {cat: 5}", + expression: ".a = .a * .b", + expected: []string{ + "D0, P[], (doc)::a: {cat: !horse 5}\nb: {cat: 5}\n", + }, + }, } func TestMultiplyOperatorScenarios(t *testing.T) { diff --git a/pkg/yqlib/operator_subtract.go b/pkg/yqlib/operator_subtract.go index 4c267e8c..89f7346f 100644 --- a/pkg/yqlib/operator_subtract.go +++ b/pkg/yqlib/operator_subtract.go @@ -3,6 +3,7 @@ package yqlib import ( "fmt" "strconv" + "strings" "gopkg.in/yaml.v3" ) @@ -74,10 +75,23 @@ func subtract(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Ca } func subtractScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*CandidateNode, error) { + lhsTag := lhs.Tag + rhsTag := rhs.Tag + lhsIsCustom := false + if !strings.HasPrefix(lhsTag, "!!") { + // custom tag - we have to have a guess + lhsTag = guessTagFromCustomType(lhs) + lhsIsCustom = true + } - if lhs.Tag == "!!str" { + if !strings.HasPrefix(rhsTag, "!!") { + // custom tag - we have to have a guess + rhsTag = guessTagFromCustomType(rhs) + } + + if lhsTag == "!!str" { return nil, fmt.Errorf("strings cannot be subtracted") - } else if lhs.Tag == "!!int" && rhs.Tag == "!!int" { + } else if lhsTag == "!!int" && rhsTag == "!!int" { format, lhsNum, err := parseInt(lhs.Value) if err != nil { return nil, err @@ -87,9 +101,9 @@ func subtractScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*Ca return nil, err } result := lhsNum - rhsNum - target.Node.Tag = "!!int" + target.Node.Tag = lhs.Tag target.Node.Value = fmt.Sprintf(format, result) - } else if (lhs.Tag == "!!int" || lhs.Tag == "!!float") && (rhs.Tag == "!!int" || rhs.Tag == "!!float") { + } else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") { lhsNum, err := strconv.ParseFloat(lhs.Value, 64) if err != nil { return nil, err @@ -99,7 +113,11 @@ func subtractScalars(target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) (*Ca return nil, err } result := lhsNum - rhsNum - target.Node.Tag = "!!float" + if lhsIsCustom { + target.Node.Tag = lhs.Tag + } else { + target.Node.Tag = "!!float" + } target.Node.Value = fmt.Sprintf("%v", result) } else { return nil, fmt.Errorf("%v cannot be added to %v", lhs.Tag, rhs.Tag) diff --git a/pkg/yqlib/operator_subtract_test.go b/pkg/yqlib/operator_subtract_test.go index e43e0bfe..5999b1e0 100644 --- a/pkg/yqlib/operator_subtract_test.go +++ b/pkg/yqlib/operator_subtract_test.go @@ -93,6 +93,34 @@ var subtractOperatorScenarios = []expressionScenario{ "D0, P[], (doc)::{a: 2, b: 4}\n", }, }, + { + description: "Custom types: that are really numbers", + subdescription: "When custom tags are encountered, yq will try to decode the underlying type.", + document: "a: !horse 2\nb: !goat 1", + expression: `.a -= .b`, + expected: []string{ + "D0, P[], (doc)::a: !horse 1\nb: !goat 1\n", + }, + }, + { + skipDoc: true, + description: "Custom types: that are really floats", + subdescription: "When custom tags are encountered, yq will try to decode the underlying type.", + document: "a: !horse 2.5\nb: !goat 1.5", + expression: `.a - .b`, + expected: []string{ + "D0, P[a], (!horse)::1\n", + }, + }, + { + skipDoc: true, + description: "Custom types: that are really maps", + document: `[!horse {a: b, c: d}, !goat {a: b}]`, + expression: `. - [{"c": "d", "a": "b"}]`, + expected: []string{ + "D0, P[], (!!seq)::[!goat {a: b}]\n", + }, + }, } func TestSubtractOperatorScenarios(t *testing.T) {