String op can now run on custom types

pkg/yqlib/doc/operators/string-operators.md

@@ -274,6 +274,24 @@ a: cart
 b: heart
 ```
 
+## Custom types: that are really strings
+When custom tags are encountered, yq will try to decode the underlying type.
+
+Given a sample.yml file of:
+```yaml
+a: !horse cat
+b: !goat heat
+```
+then
+```bash
+yq '.[] |= sub("(a)", "${1}r")' sample.yml
+```
+will output
+```yaml
+a: !horse cart
+b: !goat heart
+```
+
 ## Split strings
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/lib.go

@@ -209,6 +209,27 @@ func recurseNodeObjectEqual(lhs *yaml.Node, rhs *yaml.Node) bool {
 	return true
 }
 
+func guessTagFromCustomType(node *yaml.Node) string {
+	if strings.HasPrefix(node.Tag, "!!") {
+		return node.Tag
+	} else if node.Value == "" {
+		log.Warning("node has no value to guess the type with")
+		return node.Tag
+	}
+
+	decoder := NewYamlDecoder()
+	decoder.Init(strings.NewReader(node.Value))
+	var dataBucket yaml.Node
+	errorReading := decoder.Decode(&dataBucket)
+	if errorReading != nil {
+		log.Warning("could not guess underlying tag type %v", errorReading)
+		return node.Tag
+	}
+	guessedTag := unwrapDoc(&dataBucket).Tag
+	log.Info("im guessing the tag %v is a %v", node.Tag, guessedTag)
+	return guessedTag
+}
+
 func recursiveNodeEqual(lhs *yaml.Node, rhs *yaml.Node) bool {
 	if lhs.Kind != rhs.Kind {
 		return false
@@ -218,17 +239,8 @@ func recursiveNodeEqual(lhs *yaml.Node, rhs *yaml.Node) bool {
 		//process custom tags of scalar nodes.
 		//dont worry about matching tags of maps or arrays.
 
-		lhsTag := lhs.Tag
+		lhsTag := guessTagFromCustomType(lhs)
-		rhsTag := rhs.Tag
+		rhsTag := guessTagFromCustomType(rhs)
-		if !strings.HasPrefix(lhsTag, "!!") {
-			// custom tag - we have to have a guess
-			lhsTag = guessTagFromCustomType(lhs)
-		}
-
-		if !strings.HasPrefix(rhsTag, "!!") {
-			// custom tag - we have to have a guess
-			rhsTag = guessTagFromCustomType(rhs)
-		}
 
 		if lhsTag != rhsTag {
 			return false

pkg/yqlib/operator_add.go

@@ -79,28 +79,9 @@ func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Candida
 	return target, nil
 }
 
-func guessTagFromCustomType(node *yaml.Node) string {
-	if node.Value == "" {
-		log.Warning("node has no value to guess the type with")
-		return node.Tag
-	}
-
-	decoder := NewYamlDecoder()
-	decoder.Init(strings.NewReader(node.Value))
-	var dataBucket yaml.Node
-	errorReading := decoder.Decode(&dataBucket)
-	if errorReading != nil {
-		log.Warning("could not guess underlying tag type %v", errorReading)
-		return node.Tag
-	}
-	guessedTag := unwrapDoc(&dataBucket).Tag
-	log.Info("im guessing the tag %v is a %v", node.Tag, guessedTag)
-	return guessedTag
-}
-
 func addScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error {
 	lhsTag := lhs.Tag
-	rhsTag := rhs.Tag
+	rhsTag := guessTagFromCustomType(rhs)
 	lhsIsCustom := false
 	if !strings.HasPrefix(lhsTag, "!!") {
 		// custom tag - we have to have a guess
@@ -108,11 +89,6 @@ func addScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yam
 		lhsIsCustom = true
 	}
 
-	if !strings.HasPrefix(rhsTag, "!!") {
-		// custom tag - we have to have a guess
-		rhsTag = guessTagFromCustomType(rhs)
-	}
-
 	isDateTime := lhs.Tag == "!!timestamp"
 
 	// if the lhs is a string, it might be a timestamp in a custom format.

pkg/yqlib/operator_multiply.go

@@ -80,7 +80,7 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
 
 func multiplyScalars(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	lhsTag := lhs.Node.Tag
-	rhsTag := rhs.Node.Tag
+	rhsTag := guessTagFromCustomType(rhs.Node)
 	lhsIsCustom := false
 	if !strings.HasPrefix(lhsTag, "!!") {
 		// custom tag - we have to have a guess
@@ -88,11 +88,6 @@ func multiplyScalars(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, er
 		lhsIsCustom = true
 	}
 
-	if !strings.HasPrefix(rhsTag, "!!") {
-		// custom tag - we have to have a guess
-		rhsTag = guessTagFromCustomType(rhs.Node)
-	}
-
 	if lhsTag == "!!int" && rhsTag == "!!int" {
 		return multiplyIntegers(lhs, rhs)
 	} else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") {

pkg/yqlib/operator_strings.go

@@ -61,7 +61,8 @@ func substituteStringOperator(d *dataTreeNavigator, context Context, expressionN
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 		node := unwrapDoc(candidate.Node)
-		if node.Tag != "!!str" {
+
+		if guessTagFromCustomType(node) != "!!str" {
 			return Context{}, fmt.Errorf("cannot substitute with %v, can only substitute strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
 		}
 
@@ -247,7 +248,8 @@ func matchOperator(d *dataTreeNavigator, context Context, expressionNode *Expres
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 		node := unwrapDoc(candidate.Node)
-		if node.Tag != "!!str" {
+
+		if guessTagFromCustomType(node) != "!!str" {
 			return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
 		}
 
@@ -268,7 +270,8 @@ func captureOperator(d *dataTreeNavigator, context Context, expressionNode *Expr
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 		node := unwrapDoc(candidate.Node)
-		if node.Tag != "!!str" {
+
+		if guessTagFromCustomType(node) != "!!str" {
 			return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
 		}
 		capture(matchPrefs, regEx, candidate, node.Value, results)
@@ -289,7 +292,8 @@ func testOperator(d *dataTreeNavigator, context Context, expressionNode *Express
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 		node := unwrapDoc(candidate.Node)
-		if node.Tag != "!!str" {
+
+		if guessTagFromCustomType(node) != "!!str" {
 			return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
 		}
 		matches := regEx.FindStringSubmatch(node.Value)
@@ -361,7 +365,8 @@ func splitStringOperator(d *dataTreeNavigator, context Context, expressionNode *
 		if node.Tag == "!!null" {
 			continue
 		}
-		if node.Tag != "!!str" {
+
+		if guessTagFromCustomType(node) != "!!str" {
 			return Context{}, fmt.Errorf("Cannot split %v, can only split strings", node.Tag)
 		}
 		targetNode := split(node.Value, splitStr)

pkg/yqlib/operator_strings_test.go

@@ -13,6 +13,14 @@ var stringsOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!str)::cat; meow; 1; ; true\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `[!horse cat, !goat meow, !frog 1, null, true]`,
+		expression: `join("; ")`,
+		expected: []string{
+			"D0, P[], (!!str)::cat; meow; 1; ; true\n",
+		},
+	},
 	{
 		description: "Match string",
 		document:    `foo bar foo`,
@@ -21,6 +29,14 @@ var stringsOperatorScenarios = []expressionScenario{
 			"D0, P[], ()::string: foo\noffset: 0\nlength: 3\ncaptures: []\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `!horse foo bar foo`,
+		expression: `match("foo")`,
+		expected: []string{
+			"D0, P[], ()::string: foo\noffset: 0\nlength: 3\ncaptures: []\n",
+		},
+	},
 	{
 		description: "Match string, case insensitive",
 		document:    `foo bar FOO`,
@@ -53,6 +69,14 @@ var stringsOperatorScenarios = []expressionScenario{
 			"D0, P[], ()::a: xyzzy\nn: \"14\"\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `!horse xyzzy-14`,
+		expression: `capture("(?P<a>[a-z]+)-(?P<n>[0-9]+)")`,
+		expected: []string{
+			"D0, P[], ()::a: xyzzy\nn: \"14\"\n",
+		},
+	},
 	{
 		skipDoc:     true,
 		description: "Capture named groups into a map, with null",
@@ -78,6 +102,14 @@ var stringsOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::- string: cat\n  offset: 0\n  length: 3\n  captures: []\n- string: cat\n  offset: 4\n  length: 3\n  captures: []\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `!horse cat cat`,
+		expression: `[match("cat"; "g")]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- string: cat\n  offset: 0\n  length: 3\n  captures: []\n- string: cat\n  offset: 4\n  length: 3\n  captures: []\n",
+		},
+	},
 	{
 		skipDoc:     true,
 		description: "No match",
@@ -107,6 +139,15 @@ var stringsOperatorScenarios = []expressionScenario{
 			"D0, P[1], (!!bool)::false\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `[!horse "cat", !cat "dog"]`,
+		expression: `.[] | test("at")`,
+		expected: []string{
+			"D0, P[0], (!!bool)::true\n",
+			"D0, P[1], (!!bool)::false\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `["cat*", "cat*", "cat"]`,
@@ -135,6 +176,15 @@ var stringsOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: cart\nb: heart\n",
 		},
 	},
+	{
+		description:    "Custom types: that are really strings",
+		subdescription: "When custom tags are encountered, yq will try to decode the underlying type.",
+		document:       "a: !horse cat\nb: !goat heat",
+		expression:     `.[] |= sub("(a)", "${1}r")`,
+		expected: []string{
+			"D0, P[], (doc)::a: !horse cart\nb: !goat heart\n",
+		},
+	},
 	{
 		description: "Split strings",
 		document:    `"cat; meow; 1; ; true"`,
@@ -151,6 +201,14 @@ var stringsOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::- word\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `!horse "word"`,
+		expression: `split("; ")`,
+		expected: []string{
+			"D0, P[], (!!seq)::- word\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `""`,