suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2025-02-03 20:50:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

String op can now run on custom types

Browse Source
This commit is contained in:
Mike Farah 2022-02-22 14:50:45 +11:00
parent 8142e94349
commit 71706af3d4
6 changed files with 111 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
pkg/yqlib/doc/operators/string-operators.md
View File

@ -274,6 +274,24 @@ a: cart
b: heart
```
## Custom types: that are really strings
When custom tags are encountered, yq will try to decode the underlying type.
Given a sample.yml file of:
```yaml
a: !horse cat
b: !goat heat
```
then
```bash
yq '.[] |= sub("(a)", "${1}r")' sample.yml
```
will output
```yaml
a: !horse cart
b: !goat heart
```
## Split strings
Given a sample.yml file of:
```yaml

34
pkg/yqlib/lib.go
View File

@ -209,6 +209,27 @@ func recurseNodeObjectEqual(lhs *yaml.Node, rhs *yaml.Node) bool {
return true
}
func guessTagFromCustomType(node *yaml.Node) string {
if strings.HasPrefix(node.Tag, "!!") {
return node.Tag
} else if node.Value == "" {
log.Warning("node has no value to guess the type with")
return node.Tag
}
decoder := NewYamlDecoder()
decoder.Init(strings.NewReader(node.Value))
var dataBucket yaml.Node
errorReading := decoder.Decode(&dataBucket)
if errorReading != nil {
log.Warning("could not guess underlying tag type %v", errorReading)
return node.Tag
}
guessedTag := unwrapDoc(&dataBucket).Tag
log.Info("im guessing the tag %v is a %v", node.Tag, guessedTag)
return guessedTag
}
func recursiveNodeEqual(lhs *yaml.Node, rhs *yaml.Node) bool {
if lhs.Kind != rhs.Kind {
return false
@ -218,17 +239,8 @@ func recursiveNodeEqual(lhs *yaml.Node, rhs *yaml.Node) bool {
//process custom tags of scalar nodes.
//dont worry about matching tags of maps or arrays.
lhsTag := lhs.Tag
rhsTag := rhs.Tag
if !strings.HasPrefix(lhsTag, "!!") {
// custom tag - we have to have a guess
lhsTag = guessTagFromCustomType(lhs)
}
if !strings.HasPrefix(rhsTag, "!!") {
// custom tag - we have to have a guess
rhsTag = guessTagFromCustomType(rhs)
}
lhsTag := guessTagFromCustomType(lhs)
rhsTag := guessTagFromCustomType(rhs)
if lhsTag != rhsTag {
return false

26
pkg/yqlib/operator_add.go
View File

@ -79,28 +79,9 @@ func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Candida
return target, nil
}
func guessTagFromCustomType(node *yaml.Node) string {
if node.Value == "" {
log.Warning("node has no value to guess the type with")
return node.Tag
}
decoder := NewYamlDecoder()
decoder.Init(strings.NewReader(node.Value))
var dataBucket yaml.Node
errorReading := decoder.Decode(&dataBucket)
if errorReading != nil {
log.Warning("could not guess underlying tag type %v", errorReading)
return node.Tag
}
guessedTag := unwrapDoc(&dataBucket).Tag
log.Info("im guessing the tag %v is a %v", node.Tag, guessedTag)
return guessedTag
}
func addScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yaml.Node) error {
lhsTag := lhs.Tag
rhsTag := rhs.Tag
rhsTag := guessTagFromCustomType(rhs)
lhsIsCustom := false
if !strings.HasPrefix(lhsTag, "!!") {
// custom tag - we have to have a guess
@ -108,11 +89,6 @@ func addScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yam
lhsIsCustom = true
}
if !strings.HasPrefix(rhsTag, "!!") {
// custom tag - we have to have a guess
rhsTag = guessTagFromCustomType(rhs)
}
isDateTime := lhs.Tag == "!!timestamp"
// if the lhs is a string, it might be a timestamp in a custom format.

7
pkg/yqlib/operator_multiply.go
View File

@ -80,7 +80,7 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
func multiplyScalars(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
lhsTag := lhs.Node.Tag
rhsTag := rhs.Node.Tag
rhsTag := guessTagFromCustomType(rhs.Node)
lhsIsCustom := false
if !strings.HasPrefix(lhsTag, "!!") {
// custom tag - we have to have a guess
@ -88,11 +88,6 @@ func multiplyScalars(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, er
lhsIsCustom = true
}
if !strings.HasPrefix(rhsTag, "!!") {
// custom tag - we have to have a guess
rhsTag = guessTagFromCustomType(rhs.Node)
}
if lhsTag == "!!int" && rhsTag == "!!int" {
return multiplyIntegers(lhs, rhs)
} else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") {

15
pkg/yqlib/operator_strings.go
View File

@ -61,7 +61,8 @@ func substituteStringOperator(d *dataTreeNavigator, context Context, expressionN
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
node := unwrapDoc(candidate.Node)
if node.Tag != "!!str" {
if guessTagFromCustomType(node) != "!!str" {
return Context{}, fmt.Errorf("cannot substitute with %v, can only substitute strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
}
@ -247,7 +248,8 @@ func matchOperator(d *dataTreeNavigator, context Context, expressionNode *Expres
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
node := unwrapDoc(candidate.Node)
if node.Tag != "!!str" {
if guessTagFromCustomType(node) != "!!str" {
return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
}
@ -268,7 +270,8 @@ func captureOperator(d *dataTreeNavigator, context Context, expressionNode *Expr
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
node := unwrapDoc(candidate.Node)
if node.Tag != "!!str" {
if guessTagFromCustomType(node) != "!!str" {
return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
}
capture(matchPrefs, regEx, candidate, node.Value, results)
@ -289,7 +292,8 @@ func testOperator(d *dataTreeNavigator, context Context, expressionNode *Express
for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
node := unwrapDoc(candidate.Node)
if node.Tag != "!!str" {
if guessTagFromCustomType(node) != "!!str" {
return Context{}, fmt.Errorf("cannot match with %v, can only match strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
}
matches := regEx.FindStringSubmatch(node.Value)
@ -361,7 +365,8 @@ func splitStringOperator(d *dataTreeNavigator, context Context, expressionNode *
if node.Tag == "!!null" {
continue
}
if node.Tag != "!!str" {
if guessTagFromCustomType(node) != "!!str" {
return Context{}, fmt.Errorf("Cannot split %v, can only split strings", node.Tag)
}
targetNode := split(node.Value, splitStr)

58
pkg/yqlib/operator_strings_test.go
View File

@ -13,6 +13,14 @@ var stringsOperatorScenarios = []expressionScenario{
"D0, P[], (!!str)::cat; meow; 1; ; true\n",
},
},
{
skipDoc: true,
document: `[!horse cat, !goat meow, !frog 1, null, true]`,
expression: `join("; ")`,
expected: []string{
"D0, P[], (!!str)::cat; meow; 1; ; true\n",
},
},
{
description: "Match string",
document: `foo bar foo`,
@ -21,6 +29,14 @@ var stringsOperatorScenarios = []expressionScenario{
"D0, P[], ()::string: foo\noffset: 0\nlength: 3\ncaptures: []\n",
},
},
{
skipDoc: true,
document: `!horse foo bar foo`,
expression: `match("foo")`,
expected: []string{
"D0, P[], ()::string: foo\noffset: 0\nlength: 3\ncaptures: []\n",
},
},
{
description: "Match string, case insensitive",
document: `foo bar FOO`,
@ -53,6 +69,14 @@ var stringsOperatorScenarios = []expressionScenario{
"D0, P[], ()::a: xyzzy\nn: \"14\"\n",
},
},
{
skipDoc: true,
document: `!horse xyzzy-14`,
expression: `capture("(?P<a>[a-z]+)-(?P<n>[0-9]+)")`,
expected: []string{
"D0, P[], ()::a: xyzzy\nn: \"14\"\n",
},
},
{
skipDoc: true,
description: "Capture named groups into a map, with null",
@ -78,6 +102,14 @@ var stringsOperatorScenarios = []expressionScenario{
"D0, P[], (!!seq)::- string: cat\n offset: 0\n length: 3\n captures: []\n- string: cat\n offset: 4\n length: 3\n captures: []\n",
},
},
{
skipDoc: true,
document: `!horse cat cat`,
expression: `[match("cat"; "g")]`,
expected: []string{
"D0, P[], (!!seq)::- string: cat\n offset: 0\n length: 3\n captures: []\n- string: cat\n offset: 4\n length: 3\n captures: []\n",
},
},
{
skipDoc: true,
description: "No match",
@ -107,6 +139,15 @@ var stringsOperatorScenarios = []expressionScenario{
"D0, P[1], (!!bool)::false\n",
},
},
{
skipDoc: true,
document: `[!horse "cat", !cat "dog"]`,
expression: `.[] | test("at")`,
expected: []string{
"D0, P[0], (!!bool)::true\n",
"D0, P[1], (!!bool)::false\n",
},
},
{
skipDoc: true,
document: `["cat*", "cat*", "cat"]`,
@ -135,6 +176,15 @@ var stringsOperatorScenarios = []expressionScenario{
"D0, P[], (doc)::a: cart\nb: heart\n",
},
},
{
description: "Custom types: that are really strings",
subdescription: "When custom tags are encountered, yq will try to decode the underlying type.",
document: "a: !horse cat\nb: !goat heat",
expression: `.[] |= sub("(a)", "${1}r")`,
expected: []string{
"D0, P[], (doc)::a: !horse cart\nb: !goat heart\n",
},
},
{
description: "Split strings",
document: `"cat; meow; 1; ; true"`,
@ -151,6 +201,14 @@ var stringsOperatorScenarios = []expressionScenario{
"D0, P[], (!!seq)::- word\n",
},
},
{
skipDoc: true,
document: `!horse "word"`,
expression: `split("; ")`,
expected: []string{
"D0, P[], (!!seq)::- word\n",
},
},
{
skipDoc: true,
document: `""`,