From daf0bfe1b9bcdfec2d92fe17f9cd64cfa4562a3c Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Thu, 15 Apr 2021 10:09:41 +1000 Subject: [PATCH] Added string substitute command --- .../doc/{Multiply.md => Multiply (Merge).md} | 0 pkg/yqlib/doc/String Operators.md | 34 ++++++++++ .../{Multiply.md => Multiply (Merge).md} | 0 pkg/yqlib/expression_tokeniser.go | 2 + pkg/yqlib/lib.go | 1 + pkg/yqlib/operator_multiply_test.go | 2 +- pkg/yqlib/operator_strings.go | 68 ++++++++++++++++++- pkg/yqlib/operator_strings_test.go | 18 +++++ 8 files changed, 123 insertions(+), 2 deletions(-) rename pkg/yqlib/doc/{Multiply.md => Multiply (Merge).md} (100%) rename pkg/yqlib/doc/headers/{Multiply.md => Multiply (Merge).md} (100%) diff --git a/pkg/yqlib/doc/Multiply.md b/pkg/yqlib/doc/Multiply (Merge).md similarity index 100% rename from pkg/yqlib/doc/Multiply.md rename to pkg/yqlib/doc/Multiply (Merge).md diff --git a/pkg/yqlib/doc/String Operators.md b/pkg/yqlib/doc/String Operators.md index d18e323a..1a551816 100644 --- a/pkg/yqlib/doc/String Operators.md +++ b/pkg/yqlib/doc/String Operators.md @@ -18,6 +18,40 @@ will output cat; meow; 1; ; true ``` +## Substitute / Replace string +This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax) + +Given a sample.yml file of: +```yaml +a: dogs are great +``` +then +```bash +yq eval '.a |= sub("dogs", "cats")' sample.yml +``` +will output +```yaml +a: cats are great +``` + +## Substitute / Replace string with regex +This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax) + +Given a sample.yml file of: +```yaml +a: cat +b: heat +``` +then +```bash +yq eval '.[] |= sub("([a])", "${1}r")' sample.yml +``` +will output +```yaml +a: cart +b: heart +``` + ## Split strings Given a sample.yml file of: ```yaml diff --git a/pkg/yqlib/doc/headers/Multiply.md b/pkg/yqlib/doc/headers/Multiply (Merge).md similarity index 100% rename from pkg/yqlib/doc/headers/Multiply.md rename to pkg/yqlib/doc/headers/Multiply (Merge).md diff --git a/pkg/yqlib/expression_tokeniser.go b/pkg/yqlib/expression_tokeniser.go index 1b2d8afd..1ab33611 100644 --- a/pkg/yqlib/expression_tokeniser.go +++ b/pkg/yqlib/expression_tokeniser.go @@ -264,6 +264,8 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`splitDoc`), opToken(splitDocumentOpType)) lexer.Add([]byte(`join`), opToken(joinStringOpType)) + lexer.Add([]byte(`sub`), opToken(subStringOpType)) + lexer.Add([]byte(`split`), opToken(splitStringOpType)) lexer.Add([]byte(`keys`), opToken(keysOpType)) diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 0d05f0a9..c668cbeb 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -75,6 +75,7 @@ var getPathOpType = &operationType{Type: "GET_PATH", NumArgs: 0, Precedence: 50, var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: explodeOperator} var sortKeysOpType = &operationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: sortKeysOperator} var joinStringOpType = &operationType{Type: "JOIN", NumArgs: 1, Precedence: 50, Handler: joinStringOperator} +var subStringOpType = &operationType{Type: "SUBSTR", NumArgs: 1, Precedence: 50, Handler: substituteStringOperator} var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 50, Handler: splitStringOperator} var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 50, Handler: keysOperator} diff --git a/pkg/yqlib/operator_multiply_test.go b/pkg/yqlib/operator_multiply_test.go index 52e14a85..6c2017a7 100644 --- a/pkg/yqlib/operator_multiply_test.go +++ b/pkg/yqlib/operator_multiply_test.go @@ -283,5 +283,5 @@ func TestMultiplyOperatorScenarios(t *testing.T) { for _, tt := range multiplyOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "Multiply", multiplyOperatorScenarios) + documentScenarios(t, "Multiply (Merge)", multiplyOperatorScenarios) } diff --git a/pkg/yqlib/operator_strings.go b/pkg/yqlib/operator_strings.go index 8a0c5365..5b2f5f0d 100644 --- a/pkg/yqlib/operator_strings.go +++ b/pkg/yqlib/operator_strings.go @@ -3,11 +3,77 @@ package yqlib import ( "container/list" "fmt" + "regexp" "strings" "gopkg.in/yaml.v3" ) +func getSubstituteParameters(d *dataTreeNavigator, block *ExpressionNode, context Context) (string, string, error) { + regEx := "" + replacementText := "" + + regExNodes, err := d.GetMatchingNodes(context, block.Lhs) + if err != nil { + return "", "", err + } + if regExNodes.MatchingNodes.Front() != nil { + regEx = regExNodes.MatchingNodes.Front().Value.(*CandidateNode).Node.Value + } + + log.Debug("regEx %v", regEx) + + replacementNodes, err := d.GetMatchingNodes(context, block.Rhs) + if err != nil { + return "", "", err + } + if replacementNodes.MatchingNodes.Front() != nil { + replacementText = replacementNodes.MatchingNodes.Front().Value.(*CandidateNode).Node.Value + } + + return regEx, replacementText, nil +} + +func substitute(original string, regex *regexp.Regexp, replacement string) *yaml.Node { + replacedString := regex.ReplaceAllString(original, replacement) + return &yaml.Node{Kind: yaml.ScalarNode, Value: replacedString, Tag: "!!str"} +} + +func substituteStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { + //rhs block operator + //lhs of block = regex + //rhs of block = replacement expression + block := expressionNode.Rhs + + regExStr, replacementText, err := getSubstituteParameters(d, block, context) + + if err != nil { + return Context{}, err + } + + regEx, err := regexp.Compile(regExStr) + if err != nil { + return Context{}, err + } + + var results = list.New() + + for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + node := unwrapDoc(candidate.Node) + if node.Tag != "!!str" { + return Context{}, fmt.Errorf("cannot sustitute with %v, can only substitute strings", node.Tag) + } + + targetNode := substitute(node.Value, regEx, replacementText) + result := candidate.CreateChild(nil, targetNode) + results.PushBack(result) + } + + return context.ChildContext(results), nil + +} + func joinStringOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("-- joinStringOperator") joinStr := "" @@ -26,7 +92,7 @@ func joinStringOperator(d *dataTreeNavigator, context Context, expressionNode *E candidate := el.Value.(*CandidateNode) node := unwrapDoc(candidate.Node) if node.Kind != yaml.SequenceNode { - return Context{}, fmt.Errorf("Cannot join with %v, can only join arrays of scalars", node.Tag) + return Context{}, fmt.Errorf("cannot join with %v, can only join arrays of scalars", node.Tag) } targetNode := join(node.Content, joinStr) result := candidate.CreateChild(nil, targetNode) diff --git a/pkg/yqlib/operator_strings_test.go b/pkg/yqlib/operator_strings_test.go index 53208c1c..dd7d9358 100644 --- a/pkg/yqlib/operator_strings_test.go +++ b/pkg/yqlib/operator_strings_test.go @@ -13,6 +13,24 @@ var stringsOperatorScenarios = []expressionScenario{ "D0, P[], (!!str)::cat; meow; 1; ; true\n", }, }, + { + description: "Substitute / Replace string", + subdescription: "This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)", + document: `a: dogs are great`, + expression: `.a |= sub("dogs", "cats")`, + expected: []string{ + "D0, P[], (doc)::a: cats are great\n", + }, + }, + { + description: "Substitute / Replace string with regex", + subdescription: "This uses golang regex, described [here](https://github.com/google/re2/wiki/Syntax)", + document: "a: cat\nb: heat", + expression: `.[] |= sub("([a])", "${1}r")`, + expected: []string{ + "D0, P[], (doc)::a: cart\nb: heart\n", + }, + }, { description: "Split strings", document: `"cat; meow; 1; ; true"`,