From 535799462f14cdb1c25e09f57381f86d8b95db66 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Tue, 1 Feb 2022 14:47:51 +1100 Subject: [PATCH] Added eval operator --- cmd/root.go | 2 + cmd/utils.go | 2 +- pkg/yqlib/all_at_once_evaluator.go | 5 +- pkg/yqlib/all_at_once_evaluator_test.go | 1 + .../doc/operators/env-variable-operators.md | 19 ++++++++ pkg/yqlib/doc/operators/eval.md | 48 +++++++++++++++++++ pkg/yqlib/doc/operators/headers/eval.md | 7 +++ pkg/yqlib/expression_parser.go | 4 +- pkg/yqlib/expression_parser_test.go | 33 ++++++++----- pkg/yqlib/expression_tokeniser.go | 2 + pkg/yqlib/json_test.go | 4 +- pkg/yqlib/lib.go | 9 ++++ pkg/yqlib/operator_env_test.go | 10 ++++ pkg/yqlib/operator_eval.go | 42 ++++++++++++++++ pkg/yqlib/operator_eval_test.go | 34 +++++++++++++ pkg/yqlib/operators_test.go | 6 +-- pkg/yqlib/printer_test.go | 2 +- pkg/yqlib/stream_evaluator.go | 7 ++- pkg/yqlib/xml_test.go | 2 +- 19 files changed, 209 insertions(+), 30 deletions(-) create mode 100644 pkg/yqlib/doc/operators/eval.md create mode 100644 pkg/yqlib/doc/operators/headers/eval.md create mode 100644 pkg/yqlib/operator_eval.go create mode 100644 pkg/yqlib/operator_eval_test.go diff --git a/cmd/root.go b/cmd/root.go index 36754ed2..9e901efa 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -34,6 +34,7 @@ yq -i '.stuff = "foo"' myfile.yml # update myfile.yml inplace }, PersistentPreRun: func(cmd *cobra.Command, args []string) { cmd.SetOut(cmd.OutOrStdout()) + var format = logging.MustStringFormatter( `%{color}%{time:15:04:05} %{shortfunc} [%{level:.4s}]%{color:reset} %{message}`, ) @@ -47,6 +48,7 @@ yq -i '.stuff = "foo"' myfile.yml # update myfile.yml inplace } logging.SetBackend(backend) + yqlib.InitExpressionParser() yqlib.XmlPreferences.AttributePrefix = xmlAttributePrefix yqlib.XmlPreferences.ContentName = xmlContentName }, diff --git a/cmd/utils.go b/cmd/utils.go index 7a757ac9..48d1aec9 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -63,7 +63,7 @@ func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yq if splitFileExp != "" { colorsEnabled = forceColor - splitExp, err := yqlib.NewExpressionParser().ParseExpression(splitFileExp) + splitExp, err := yqlib.ExpressionParser.ParseExpression(splitFileExp) if err != nil { return nil, fmt.Errorf("bad split document expression: %w", err) } diff --git a/pkg/yqlib/all_at_once_evaluator.go b/pkg/yqlib/all_at_once_evaluator.go index 371bc48f..69c8c47c 100644 --- a/pkg/yqlib/all_at_once_evaluator.go +++ b/pkg/yqlib/all_at_once_evaluator.go @@ -19,11 +19,10 @@ type Evaluator interface { type allAtOnceEvaluator struct { treeNavigator DataTreeNavigator - treeCreator ExpressionParser } func NewAllAtOnceEvaluator() Evaluator { - return &allAtOnceEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: NewExpressionParser()} + return &allAtOnceEvaluator{treeNavigator: NewDataTreeNavigator()} } func (e *allAtOnceEvaluator) EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error) { @@ -35,7 +34,7 @@ func (e *allAtOnceEvaluator) EvaluateNodes(expression string, nodes ...*yaml.Nod } func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCandidates *list.List) (*list.List, error) { - node, err := e.treeCreator.ParseExpression(expression) + node, err := ExpressionParser.ParseExpression(expression) if err != nil { return nil, err } diff --git a/pkg/yqlib/all_at_once_evaluator_test.go b/pkg/yqlib/all_at_once_evaluator_test.go index 2e9abc43..546e3d84 100644 --- a/pkg/yqlib/all_at_once_evaluator_test.go +++ b/pkg/yqlib/all_at_once_evaluator_test.go @@ -31,6 +31,7 @@ var evaluateNodesScenario = []expressionScenario{ } func TestAllAtOnceEvaluateNodes(t *testing.T) { + InitExpressionParser() var evaluator = NewAllAtOnceEvaluator() for _, tt := range evaluateNodesScenario { node := test.ParseData(tt.document) diff --git a/pkg/yqlib/doc/operators/env-variable-operators.md b/pkg/yqlib/doc/operators/env-variable-operators.md index 34cc5ef3..ea99429a 100644 --- a/pkg/yqlib/doc/operators/env-variable-operators.md +++ b/pkg/yqlib/doc/operators/env-variable-operators.md @@ -77,6 +77,25 @@ will output a: "12" ``` +## Dynamically evaluate a path from an environment variable +The env variable can be any valid yq expression. + +Given a sample.yml file of: +```yaml +a: + b: + - name: dog + - name: cat +``` +then +```bash +myenv=".a.b[0].name" yq 'eval(strenv(myenv))' sample.yml +``` +will output +```yaml +dog +``` + ## Dynamic key lookup with environment variable Given a sample.yml file of: ```yaml diff --git a/pkg/yqlib/doc/operators/eval.md b/pkg/yqlib/doc/operators/eval.md new file mode 100644 index 00000000..6601419d --- /dev/null +++ b/pkg/yqlib/doc/operators/eval.md @@ -0,0 +1,48 @@ +# Eval + +Use `eval` to dynamically process an expression - for instance from an environment variable. + +`eval` takes a single argument, and evaluates that as a `yq` expression. Any valid expression can be used, beit a path `.a.b.c | select(. == "cat")`, or an update `.a.b.c = "gogo"`. + +Tip: This can be useful way parameterise complex scripts. + +## Dynamically evaluate a path +Given a sample.yml file of: +```yaml +pathExp: .a.b[] | select(.name == "cat") +a: + b: + - name: dog + - name: cat +``` +then +```bash +yq 'eval(.pathExp)' sample.yml +``` +will output +```yaml +name: cat +``` + +## Dynamically update a path from an environment variable +The env variable can be any valid yq expression. + +Given a sample.yml file of: +```yaml +a: + b: + - name: dog + - name: cat +``` +then +```bash +myenv=".a.b[0].name" yq 'eval(strenv(myenv)) = "cow"' sample.yml +``` +will output +```yaml +a: + b: + - name: cow + - name: cat +``` + diff --git a/pkg/yqlib/doc/operators/headers/eval.md b/pkg/yqlib/doc/operators/headers/eval.md new file mode 100644 index 00000000..45c1a221 --- /dev/null +++ b/pkg/yqlib/doc/operators/headers/eval.md @@ -0,0 +1,7 @@ +# Eval + +Use `eval` to dynamically process an expression - for instance from an environment variable. + +`eval` takes a single argument, and evaluates that as a `yq` expression. Any valid expression can be used, beit a path `.a.b.c | select(. == "cat")`, or an update `.a.b.c = "gogo"`. + +Tip: This can be useful way parameterise complex scripts. diff --git a/pkg/yqlib/expression_parser.go b/pkg/yqlib/expression_parser.go index ae8da663..3d578fac 100644 --- a/pkg/yqlib/expression_parser.go +++ b/pkg/yqlib/expression_parser.go @@ -11,7 +11,7 @@ type ExpressionNode struct { Rhs *ExpressionNode } -type ExpressionParser interface { +type ExpressionParserInterface interface { ParseExpression(expression string) (*ExpressionNode, error) } @@ -20,7 +20,7 @@ type expressionParserImpl struct { pathPostFixer expressionPostFixer } -func NewExpressionParser() ExpressionParser { +func newExpressionParser() ExpressionParserInterface { return &expressionParserImpl{newExpressionTokeniser(), newExpressionPostFixer()} } diff --git a/pkg/yqlib/expression_parser_test.go b/pkg/yqlib/expression_parser_test.go index f1e05e17..d31792eb 100644 --- a/pkg/yqlib/expression_parser_test.go +++ b/pkg/yqlib/expression_parser_test.go @@ -6,66 +6,73 @@ import ( "github.com/mikefarah/yq/v4/test" ) +func getExpressionParser() ExpressionParserInterface { + if ExpressionParser == nil { + ExpressionParser = newExpressionParser() + } + return ExpressionParser +} + func TestParserNoMatchingCloseCollect(t *testing.T) { - _, err := NewExpressionParser().ParseExpression("[1,2") + _, err := getExpressionParser().ParseExpression("[1,2") test.AssertResultComplex(t, "Bad expression, could not find matching `]`", err.Error()) } func TestParserNoMatchingCloseObjectInCollect(t *testing.T) { - _, err := NewExpressionParser().ParseExpression(`[{"b": "c"]`) + _, err := getExpressionParser().ParseExpression(`[{"b": "c"]`) test.AssertResultComplex(t, "Bad expression, could not find matching `}`", err.Error()) } func TestParserNoMatchingCloseInCollect(t *testing.T) { - _, err := NewExpressionParser().ParseExpression(`[(.a]`) + _, err := getExpressionParser().ParseExpression(`[(.a]`) test.AssertResultComplex(t, "Bad expression, could not find matching `)`", err.Error()) } func TestParserNoMatchingCloseCollectObject(t *testing.T) { - _, err := NewExpressionParser().ParseExpression(`{"a": "b"`) + _, err := getExpressionParser().ParseExpression(`{"a": "b"`) test.AssertResultComplex(t, "Bad expression, could not find matching `}`", err.Error()) } func TestParserNoMatchingCloseCollectInCollectObject(t *testing.T) { - _, err := NewExpressionParser().ParseExpression(`{"b": [1}`) + _, err := getExpressionParser().ParseExpression(`{"b": [1}`) test.AssertResultComplex(t, "Bad expression, could not find matching `]`", err.Error()) } func TestParserNoMatchingCloseBracketInCollectObject(t *testing.T) { - _, err := NewExpressionParser().ParseExpression(`{"b": (1}`) + _, err := getExpressionParser().ParseExpression(`{"b": (1}`) test.AssertResultComplex(t, "Bad expression, could not find matching `)`", err.Error()) } func TestParserNoArgsForTwoArgOp(t *testing.T) { - _, err := NewExpressionParser().ParseExpression("=") + _, err := getExpressionParser().ParseExpression("=") test.AssertResultComplex(t, "'=' expects 2 args but there is 0", err.Error()) } func TestParserOneLhsArgsForTwoArgOp(t *testing.T) { - _, err := NewExpressionParser().ParseExpression(".a =") + _, err := getExpressionParser().ParseExpression(".a =") test.AssertResultComplex(t, "'=' expects 2 args but there is 1", err.Error()) } func TestParserOneRhsArgsForTwoArgOp(t *testing.T) { - _, err := NewExpressionParser().ParseExpression("= .a") + _, err := getExpressionParser().ParseExpression("= .a") test.AssertResultComplex(t, "'=' expects 2 args but there is 1", err.Error()) } func TestParserTwoArgsForTwoArgOp(t *testing.T) { - _, err := NewExpressionParser().ParseExpression(".a = .b") + _, err := getExpressionParser().ParseExpression(".a = .b") test.AssertResultComplex(t, nil, err) } func TestParserNoArgsForOneArgOp(t *testing.T) { - _, err := NewExpressionParser().ParseExpression("explode") + _, err := getExpressionParser().ParseExpression("explode") test.AssertResultComplex(t, "'explode' expects 1 arg but received none", err.Error()) } func TestParserOneArgForOneArgOp(t *testing.T) { - _, err := NewExpressionParser().ParseExpression("explode(.)") + _, err := getExpressionParser().ParseExpression("explode(.)") test.AssertResultComplex(t, nil, err) } func TestParserExtraArgs(t *testing.T) { - _, err := NewExpressionParser().ParseExpression("sortKeys(.) explode(.)") + _, err := getExpressionParser().ParseExpression("sortKeys(.) explode(.)") test.AssertResultComplex(t, "Bad expression, please check expression syntax", err.Error()) } diff --git a/pkg/yqlib/expression_tokeniser.go b/pkg/yqlib/expression_tokeniser.go index 24c7a4ca..2662a57e 100644 --- a/pkg/yqlib/expression_tokeniser.go +++ b/pkg/yqlib/expression_tokeniser.go @@ -316,6 +316,8 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`:\s*`), opToken(createMapOpType)) lexer.Add([]byte(`length`), opToken(lengthOpType)) + lexer.Add([]byte(`eval`), opToken(evalOpType)) + lexer.Add([]byte(`map`), opToken(mapOpType)) lexer.Add([]byte(`map_values`), opToken(mapValuesOpType)) diff --git a/pkg/yqlib/json_test.go b/pkg/yqlib/json_test.go index 1272d096..addbcdac 100644 --- a/pkg/yqlib/json_test.go +++ b/pkg/yqlib/json_test.go @@ -79,7 +79,7 @@ func decodeJson(t *testing.T, jsonString string) *CandidateNode { return nil } - exp, err := NewExpressionParser().ParseExpression(PrettyPrintExp) + exp, err := getExpressionParser().ParseExpression(PrettyPrintExp) if err != nil { t.Error(err) @@ -124,7 +124,7 @@ func processJsonScenario(s formatScenario) string { expression = "." } - exp, err := NewExpressionParser().ParseExpression(expression) + exp, err := getExpressionParser().ParseExpression(expression) if err != nil { panic(err) diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index d5fa63e1..2725880e 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -13,6 +13,14 @@ import ( yaml "gopkg.in/yaml.v3" ) +var ExpressionParser ExpressionParserInterface + +func InitExpressionParser() { + if ExpressionParser == nil { + ExpressionParser = newExpressionParser() + } +} + type xmlPreferences struct { AttributePrefix string ContentName string @@ -74,6 +82,7 @@ var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence: var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator} var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50, Handler: collectOperator} var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler: mapOperator} +var evalOpType = &operationType{Type: "EVAL", NumArgs: 1, Precedence: 50, Handler: evalOperator} var mapValuesOpType = &operationType{Type: "MAP_VALUES", NumArgs: 1, Precedence: 50, Handler: mapValuesOperator} var encodeOpType = &operationType{Type: "ENCODE", NumArgs: 0, Precedence: 50, Handler: encodeOperator} var decodeOpType = &operationType{Type: "DECODE", NumArgs: 0, Precedence: 50, Handler: decodeOperator} diff --git a/pkg/yqlib/operator_env_test.go b/pkg/yqlib/operator_env_test.go index c8d6327f..a762e58a 100644 --- a/pkg/yqlib/operator_env_test.go +++ b/pkg/yqlib/operator_env_test.go @@ -53,6 +53,16 @@ var envOperatorScenarios = []expressionScenario{ "D0, P[], ()::a: \"12\"\n", }, }, + { + description: "Dynamically evaluate a path from an environment variable", + subdescription: "The env variable can be any valid yq expression.", + document: `{a: {b: [{name: dog}, {name: cat}]}}`, + environmentVariable: ".a.b[0].name", + expression: `eval(strenv(myenv))`, + expected: []string{ + "D0, P[a b 0 name], (!!str)::dog\n", + }, + }, { description: "Dynamic key lookup with environment variable", environmentVariable: "cat", diff --git a/pkg/yqlib/operator_eval.go b/pkg/yqlib/operator_eval.go new file mode 100644 index 00000000..27aa77a8 --- /dev/null +++ b/pkg/yqlib/operator_eval.go @@ -0,0 +1,42 @@ +package yqlib + +import ( + "container/list" +) + +func evalOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { + log.Debugf("Eval") + pathExpStrResults, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs) + if err != nil { + return Context{}, err + } + + expressions := make([]*ExpressionNode, pathExpStrResults.MatchingNodes.Len()) + expIndex := 0 + //parse every expression + for pathExpStrEntry := pathExpStrResults.MatchingNodes.Front(); pathExpStrEntry != nil; pathExpStrEntry = pathExpStrEntry.Next() { + expressionStrCandidate := pathExpStrEntry.Value.(*CandidateNode) + + expressions[expIndex], err = ExpressionParser.ParseExpression(expressionStrCandidate.Node.Value) + if err != nil { + return Context{}, err + } + + expIndex++ + } + + results := list.New() + + for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() { + for expIndex = 0; expIndex < len(expressions); expIndex++ { + result, err := d.GetMatchingNodes(context, expressions[expIndex]) + if err != nil { + return Context{}, err + } + results.PushBackList(result.MatchingNodes) + } + } + + return context.ChildContext(results), nil + +} diff --git a/pkg/yqlib/operator_eval_test.go b/pkg/yqlib/operator_eval_test.go new file mode 100644 index 00000000..052f1178 --- /dev/null +++ b/pkg/yqlib/operator_eval_test.go @@ -0,0 +1,34 @@ +package yqlib + +import ( + "testing" +) + +var evalOperatorScenarios = []expressionScenario{ + + { + description: "Dynamically evaluate a path", + document: `{pathExp: '.a.b[] | select(.name == "cat")', a: {b: [{name: dog}, {name: cat}]}}`, + expression: `eval(.pathExp)`, + expected: []string{ + "D0, P[a b 1], (!!map)::{name: cat}\n", + }, + }, + { + description: "Dynamically update a path from an environment variable", + subdescription: "The env variable can be any valid yq expression.", + document: `{a: {b: [{name: dog}, {name: cat}]}}`, + environmentVariable: ".a.b[0].name", + expression: `eval(strenv(myenv)) = "cow"`, + expected: []string{ + "D0, P[], (doc)::{a: {b: [{name: cow}, {name: cat}]}}\n", + }, + }, +} + +func TestEvalOperatorsScenarios(t *testing.T) { + for _, tt := range evalOperatorScenarios { + testScenario(t, &tt) + } + documentOperatorScenarios(t, "eval", evalOperatorScenarios) +} diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index f3382a26..e34d5c15 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -53,7 +53,7 @@ func readDocumentWithLeadingContent(content string, fakefilename string, fakeFil func testScenario(t *testing.T, s *expressionScenario) { var err error - node, err := NewExpressionParser().ParseExpression(s.expression) + node, err := getExpressionParser().ParseExpression(s.expression) if err != nil { t.Error(fmt.Errorf("Error parsing expression %v of %v: %w", s.expression, s.description, err)) return @@ -157,7 +157,7 @@ func formatYaml(yaml string, filename string) string { var output bytes.Buffer printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true) - node, err := NewExpressionParser().ParseExpression(".. style= \"\"") + node, err := getExpressionParser().ParseExpression(".. style= \"\"") if err != nil { panic(err) } @@ -280,7 +280,7 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt var err error printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true) - node, err := NewExpressionParser().ParseExpression(s.expression) + node, err := getExpressionParser().ParseExpression(s.expression) if err != nil { t.Error(fmt.Errorf("Error parsing expression %v of %v: %w", s.expression, s.description, err)) return diff --git a/pkg/yqlib/printer_test.go b/pkg/yqlib/printer_test.go index d23f9b96..81f2ac79 100644 --- a/pkg/yqlib/printer_test.go +++ b/pkg/yqlib/printer_test.go @@ -289,7 +289,7 @@ func TestPrinterScalarWithLeadingCont(t *testing.T) { var writer = bufio.NewWriter(&output) printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true) - node, err := NewExpressionParser().ParseExpression(".a") + node, err := getExpressionParser().ParseExpression(".a") if err != nil { panic(err) } diff --git a/pkg/yqlib/stream_evaluator.go b/pkg/yqlib/stream_evaluator.go index 96826f30..4f4d8f7d 100644 --- a/pkg/yqlib/stream_evaluator.go +++ b/pkg/yqlib/stream_evaluator.go @@ -21,16 +21,15 @@ type StreamEvaluator interface { type streamEvaluator struct { treeNavigator DataTreeNavigator - treeCreator ExpressionParser fileIndex int } func NewStreamEvaluator() StreamEvaluator { - return &streamEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: NewExpressionParser()} + return &streamEvaluator{treeNavigator: NewDataTreeNavigator()} } func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadingContent string) error { - node, err := s.treeCreator.ParseExpression(expression) + node, err := ExpressionParser.ParseExpression(expression) if err != nil { return err } @@ -53,7 +52,7 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadin func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error { var totalProcessDocs uint - node, err := s.treeCreator.ParseExpression(expression) + node, err := ExpressionParser.ParseExpression(expression) if err != nil { return err } diff --git a/pkg/yqlib/xml_test.go b/pkg/yqlib/xml_test.go index 38d9bcf0..01a51f15 100644 --- a/pkg/yqlib/xml_test.go +++ b/pkg/yqlib/xml_test.go @@ -27,7 +27,7 @@ func decodeXml(t *testing.T, s formatScenario) *CandidateNode { expression = "." } - exp, err := NewExpressionParser().ParseExpression(expression) + exp, err := getExpressionParser().ParseExpression(expression) if err != nil { t.Error(err)