From 71706af3d4562501fb47959c69c2d682b32c5c6e Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Tue, 22 Feb 2022 14:50:45 +1100 Subject: [PATCH] String op can now run on custom types --- pkg/yqlib/doc/operators/string-operators.md | 18 +++++++ pkg/yqlib/lib.go | 34 ++++++++---- pkg/yqlib/operator_add.go | 26 +-------- pkg/yqlib/operator_multiply.go | 7 +-- pkg/yqlib/operator_strings.go | 15 ++++-- pkg/yqlib/operator_strings_test.go | 58 +++++++++++++++++++++ 6 files changed, 111 insertions(+), 47 deletions(-) diff --git a/pkg/yqlib/doc/operators/string-operators.md b/pkg/yqlib/doc/operators/string-operators.md index 07c04a79..c9447a99 100644 --- a/pkg/yqlib/doc/operators/string-operators.md +++ b/pkg/yqlib/doc/operators/string-operators.md @@ -274,6 +274,24 @@ a: cart b: heart ``` +## Custom types: that are really strings +When custom tags are encountered, yq will try to decode the underlying type. + +Given a sample.yml file of: +```yaml +a: !horse cat +b: !goat heat +``` +then +```bash +yq '.[] |= sub("(a)", "${1}r")' sample.yml +``` +will output +```yaml +a: !horse cart +b: !goat heart +``` + ## Split strings Given a sample.yml file of: ```yaml diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index ed2d9c6c..dd07d5c1 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -209,6 +209,27 @@ func recurseNodeObjectEqual(lhs *yaml.Node, rhs *yaml.Node) bool { return true } +func guessTagFromCustomType(node *yaml.Node) string { + if strings.HasPrefix(node.Tag, "!!") { + return node.Tag + } else if node.Value == "" { + log.Warning("node has no value to guess the type with") + return node.Tag + } + + decoder := NewYamlDecoder() + decoder.Init(strings.NewReader(node.Value)) + var dataBucket yaml.Node + errorReading := decoder.Decode(&dataBucket) + if errorReading != nil { + log.Warning("could not guess underlying tag type %v", errorReading) + return node.Tag + } + guessedTag := unwrapDoc(&dataBucket).Tag + log.Info("im guessing the tag %v is a %v", node.Tag, guessedTag) + return guessedTag +} + func recursiveNodeEqual(lhs *yaml.Node, rhs *yaml.Node) bool { if lhs.Kind != rhs.Kind { return false @@ -218,17 +239,8 @@ func recursiveNodeEqual(lhs *yaml.Node, rhs *yaml.Node) bool { //process custom tags of scalar nodes. //dont worry about matching tags of maps or arrays. - lhsTag := lhs.Tag - rhsTag := rhs.Tag - if !strings.HasPrefix(lhsTag, "!!") { - // custom tag - we have to have a guess - lhsTag = guessTagFromCustomType(lhs) - } - - if !strings.HasPrefix(rhsTag, "!!") { - // custom tag - we have to have a guess - rhsTag = guessTagFromCustomType(rhs) - } + lhsTag := guessTagFromCustomType(lhs) + rhsTag := guessTagFromCustomType(rhs) if lhsTag != rhsTag { return false diff --git a/pkg/yqlib/operator_add.go b/pkg/yqlib/operator_add.go index efcefdfd..db9cd298 100644 --- a/pkg/yqlib/operator_add.go +++ b/pkg/yqlib/operator_add.go @@ -79,28 +79,9 @@ func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Candida return target, nil } -func guessTagFromCustomType(node *yaml.Node) string { - if node.Value == "" { - log.Warning("node has no value to guess the type with") - return node.Tag - } - - decoder := NewYamlDecoder() - decoder.Init(strings.NewReader(node.Value)) - var dataBucket yaml.Node - errorReading := decoder.Decode(&dataBucket) - if errorReading != nil { - log.Warning("could not guess underlying tag type %v", errorReading) - return node.Tag - } - guessedTag := unwrapDoc(&dataBucket).Tag - log.Info("im guessing the tag %v is a %v", node.Tag, guessedTag) - return guessedTag -} - func addScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error { lhsTag := lhs.Tag - rhsTag := rhs.Tag + rhsTag := guessTagFromCustomType(rhs) lhsIsCustom := false if !strings.HasPrefix(lhsTag, "!!") { // custom tag - we have to have a guess @@ -108,11 +89,6 @@ func addScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yam lhsIsCustom = true } - if !strings.HasPrefix(rhsTag, "!!") { - // custom tag - we have to have a guess - rhsTag = guessTagFromCustomType(rhs) - } - isDateTime := lhs.Tag == "!!timestamp" // if the lhs is a string, it might be a timestamp in a custom format. diff --git a/pkg/yqlib/operator_multiply.go b/pkg/yqlib/operator_multiply.go index b8efb020..c64e1c81 100644 --- a/pkg/yqlib/operator_multiply.go +++ b/pkg/yqlib/operator_multiply.go @@ -80,7 +80,7 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex func multiplyScalars(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { lhsTag := lhs.Node.Tag - rhsTag := rhs.Node.Tag + rhsTag := guessTagFromCustomType(rhs.Node) lhsIsCustom := false if !strings.HasPrefix(lhsTag, "!!") { // custom tag - we have to have a guess @@ -88,11 +88,6 @@ func multiplyScalars(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, er lhsIsCustom = true } - if !strings.HasPrefix(rhsTag, "!!") { - // custom tag - we have to have a guess - rhsTag = guessTagFromCustomType(rhs.Node) - } - if lhsTag == "!!int" && rhsTag == "!!int" { return multiplyIntegers(lhs, rhs) } else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") { diff --git a/pkg/yqlib/operator_strings.go b/pkg/yqlib/operator_strings.go index d57fbeb6..2ab60fd3 100644 --- a/pkg/yqlib/operator_strings.go +++ b/pkg/yqlib/operator_strings.go @@ -61,7 +61,8 @@ func substituteStringOperator(d *dataTreeNavigator, context Context, expressionN for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) node := unwrapDoc(candidate.Node) - if node.Tag != "!!str" { + + if guessTagFromCustomType(node) != "!!str" { return Context{}, fmt.Errorf("cannot substitute with %v, can only substitute strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag) } @@ -247,7 +248,8 @@ func matchOperator(d *dataTreeNavigator, context Context, expressionNode *Expres for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) node := unwrapDoc(candidate.Node) - if node.Tag != "!!str" { + + if guessTagFromCustomType(node) != "!!str" { return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag) } @@ -268,7 +270,8 @@ func captureOperator(d *dataTreeNavigator, context Context, expressionNode *Expr for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) node := unwrapDoc(candidate.Node) - if node.Tag != "!!str" { + + if guessTagFromCustomType(node) != "!!str" { return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag) } capture(matchPrefs, regEx, candidate, node.Value, results) @@ -289,7 +292,8 @@ func testOperator(d *dataTreeNavigator, context Context, expressionNode *Express for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) node := unwrapDoc(candidate.Node) - if node.Tag != "!!str" { + + if guessTagFromCustomType(node) != "!!str" { return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag) } matches := regEx.FindStringSubmatch(node.Value) @@ -361,7 +365,8 @@ func splitStringOperator(d *dataTreeNavigator, context Context, expressionNode * if node.Tag == "!!null" { continue } - if node.Tag != "!!str" { + + if guessTagFromCustomType(node) != "!!str" { return Context{}, fmt.Errorf("Cannot split %v, can only split strings", node.Tag) } targetNode := split(node.Value, splitStr) diff --git a/pkg/yqlib/operator_strings_test.go b/pkg/yqlib/operator_strings_test.go index 8a510993..cf65f8a6 100644 --- a/pkg/yqlib/operator_strings_test.go +++ b/pkg/yqlib/operator_strings_test.go @@ -13,6 +13,14 @@ var stringsOperatorScenarios = []expressionScenario{ "D0, P[], (!!str)::cat; meow; 1; ; true\n", }, }, + { + skipDoc: true, + document: `[!horse cat, !goat meow, !frog 1, null, true]`, + expression: `join("; ")`, + expected: []string{ + "D0, P[], (!!str)::cat; meow; 1; ; true\n", + }, + }, { description: "Match string", document: `foo bar foo`, @@ -21,6 +29,14 @@ var stringsOperatorScenarios = []expressionScenario{ "D0, P[], ()::string: foo\noffset: 0\nlength: 3\ncaptures: []\n", }, }, + { + skipDoc: true, + document: `!horse foo bar foo`, + expression: `match("foo")`, + expected: []string{ + "D0, P[], ()::string: foo\noffset: 0\nlength: 3\ncaptures: []\n", + }, + }, { description: "Match string, case insensitive", document: `foo bar FOO`, @@ -53,6 +69,14 @@ var stringsOperatorScenarios = []expressionScenario{ "D0, P[], ()::a: xyzzy\nn: \"14\"\n", }, }, + { + skipDoc: true, + document: `!horse xyzzy-14`, + expression: `capture("(?P[a-z]+)-(?P[0-9]+)")`, + expected: []string{ + "D0, P[], ()::a: xyzzy\nn: \"14\"\n", + }, + }, { skipDoc: true, description: "Capture named groups into a map, with null", @@ -78,6 +102,14 @@ var stringsOperatorScenarios = []expressionScenario{ "D0, P[], (!!seq)::- string: cat\n offset: 0\n length: 3\n captures: []\n- string: cat\n offset: 4\n length: 3\n captures: []\n", }, }, + { + skipDoc: true, + document: `!horse cat cat`, + expression: `[match("cat"; "g")]`, + expected: []string{ + "D0, P[], (!!seq)::- string: cat\n offset: 0\n length: 3\n captures: []\n- string: cat\n offset: 4\n length: 3\n captures: []\n", + }, + }, { skipDoc: true, description: "No match", @@ -107,6 +139,15 @@ var stringsOperatorScenarios = []expressionScenario{ "D0, P[1], (!!bool)::false\n", }, }, + { + skipDoc: true, + document: `[!horse "cat", !cat "dog"]`, + expression: `.[] | test("at")`, + expected: []string{ + "D0, P[0], (!!bool)::true\n", + "D0, P[1], (!!bool)::false\n", + }, + }, { skipDoc: true, document: `["cat*", "cat*", "cat"]`, @@ -135,6 +176,15 @@ var stringsOperatorScenarios = []expressionScenario{ "D0, P[], (doc)::a: cart\nb: heart\n", }, }, + { + description: "Custom types: that are really strings", + subdescription: "When custom tags are encountered, yq will try to decode the underlying type.", + document: "a: !horse cat\nb: !goat heat", + expression: `.[] |= sub("(a)", "${1}r")`, + expected: []string{ + "D0, P[], (doc)::a: !horse cart\nb: !goat heart\n", + }, + }, { description: "Split strings", document: `"cat; meow; 1; ; true"`, @@ -151,6 +201,14 @@ var stringsOperatorScenarios = []expressionScenario{ "D0, P[], (!!seq)::- word\n", }, }, + { + skipDoc: true, + document: `!horse "word"`, + expression: `split("; ")`, + expected: []string{ + "D0, P[], (!!seq)::- word\n", + }, + }, { skipDoc: true, document: `""`,