Added eval operator

cmd/root.go

@@ -34,6 +34,7 @@ yq -i '.stuff = "foo"' myfile.yml # update myfile.yml inplace
 		},
 		PersistentPreRun: func(cmd *cobra.Command, args []string) {
 			cmd.SetOut(cmd.OutOrStdout())
+
 			var format = logging.MustStringFormatter(
 				`%{color}%{time:15:04:05} %{shortfunc} [%{level:.4s}]%{color:reset} %{message}`,
 			)
@@ -47,6 +48,7 @@ yq -i '.stuff = "foo"' myfile.yml # update myfile.yml inplace
 			}
 
 			logging.SetBackend(backend)
+			yqlib.InitExpressionParser()
 			yqlib.XmlPreferences.AttributePrefix = xmlAttributePrefix
 			yqlib.XmlPreferences.ContentName = xmlContentName
 		},

cmd/utils.go

@@ -63,7 +63,7 @@ func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yq
 
 	if splitFileExp != "" {
 		colorsEnabled = forceColor
-		splitExp, err := yqlib.NewExpressionParser().ParseExpression(splitFileExp)
+		splitExp, err := yqlib.ExpressionParser.ParseExpression(splitFileExp)
 		if err != nil {
 			return nil, fmt.Errorf("bad split document expression: %w", err)
 		}

pkg/yqlib/all_at_once_evaluator.go

@@ -19,11 +19,10 @@ type Evaluator interface {
 
 type allAtOnceEvaluator struct {
 	treeNavigator DataTreeNavigator
-	treeCreator   ExpressionParser
 }
 
 func NewAllAtOnceEvaluator() Evaluator {
-	return &allAtOnceEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: NewExpressionParser()}
+	return &allAtOnceEvaluator{treeNavigator: NewDataTreeNavigator()}
 }
 
 func (e *allAtOnceEvaluator) EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error) {
@@ -35,7 +34,7 @@ func (e *allAtOnceEvaluator) EvaluateNodes(expression string, nodes ...*yaml.Nod
 }
 
 func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCandidates *list.List) (*list.List, error) {
-	node, err := e.treeCreator.ParseExpression(expression)
+	node, err := ExpressionParser.ParseExpression(expression)
 	if err != nil {
 		return nil, err
 	}

pkg/yqlib/all_at_once_evaluator_test.go

@@ -31,6 +31,7 @@ var evaluateNodesScenario = []expressionScenario{
 }
 
 func TestAllAtOnceEvaluateNodes(t *testing.T) {
+	InitExpressionParser()
 	var evaluator = NewAllAtOnceEvaluator()
 	for _, tt := range evaluateNodesScenario {
 		node := test.ParseData(tt.document)

pkg/yqlib/doc/operators/env-variable-operators.md

@@ -77,6 +77,25 @@ will output
 a: "12"
 ```
 
+## Dynamically evaluate a path from an environment variable
+The env variable can be any valid yq expression.
+
+Given a sample.yml file of:
+```yaml
+a:
+  b:
+    - name: dog
+    - name: cat
+```
+then
+```bash
+myenv=".a.b[0].name" yq 'eval(strenv(myenv))' sample.yml
+```
+will output
+```yaml
+dog
+```
+
 ## Dynamic key lookup with environment variable
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/operators/eval.md

@@ -0,0 +1,48 @@
+# Eval
+
+Use `eval` to dynamically process an expression - for instance from an environment variable.
+
+`eval` takes a single argument, and evaluates that as a `yq` expression. Any valid expression can be used, beit a path `.a.b.c | select(. == "cat")`, or an update `.a.b.c = "gogo"`.
+
+Tip: This can be useful way parameterise complex scripts.
+
+## Dynamically evaluate a path
+Given a sample.yml file of:
+```yaml
+pathExp: .a.b[] | select(.name == "cat")
+a:
+  b:
+    - name: dog
+    - name: cat
+```
+then
+```bash
+yq 'eval(.pathExp)' sample.yml
+```
+will output
+```yaml
+name: cat
+```
+
+## Dynamically update a path from an environment variable
+The env variable can be any valid yq expression.
+
+Given a sample.yml file of:
+```yaml
+a:
+  b:
+    - name: dog
+    - name: cat
+```
+then
+```bash
+myenv=".a.b[0].name" yq 'eval(strenv(myenv)) = "cow"' sample.yml
+```
+will output
+```yaml
+a:
+  b:
+    - name: cow
+    - name: cat
+```
+

pkg/yqlib/doc/operators/headers/eval.md

@@ -0,0 +1,7 @@
+# Eval
+
+Use `eval` to dynamically process an expression - for instance from an environment variable.
+
+`eval` takes a single argument, and evaluates that as a `yq` expression. Any valid expression can be used, beit a path `.a.b.c | select(. == "cat")`, or an update `.a.b.c = "gogo"`.
+
+Tip: This can be useful way parameterise complex scripts.

pkg/yqlib/expression_parser.go

@@ -11,7 +11,7 @@ type ExpressionNode struct {
 	Rhs       *ExpressionNode
 }
 
-type ExpressionParser interface {
+type ExpressionParserInterface interface {
 	ParseExpression(expression string) (*ExpressionNode, error)
 }
 
@@ -20,7 +20,7 @@ type expressionParserImpl struct {
 	pathPostFixer expressionPostFixer
 }
 
-func NewExpressionParser() ExpressionParser {
+func newExpressionParser() ExpressionParserInterface {
 	return &expressionParserImpl{newExpressionTokeniser(), newExpressionPostFixer()}
 }
 

pkg/yqlib/expression_parser_test.go

@@ -6,66 +6,73 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
+func getExpressionParser() ExpressionParserInterface {
+	if ExpressionParser == nil {
+		ExpressionParser = newExpressionParser()
+	}
+	return ExpressionParser
+}
+
 func TestParserNoMatchingCloseCollect(t *testing.T) {
-	_, err := NewExpressionParser().ParseExpression("[1,2")
+	_, err := getExpressionParser().ParseExpression("[1,2")
 	test.AssertResultComplex(t, "Bad expression, could not find matching `]`", err.Error())
 }
 func TestParserNoMatchingCloseObjectInCollect(t *testing.T) {
-	_, err := NewExpressionParser().ParseExpression(`[{"b": "c"]`)
+	_, err := getExpressionParser().ParseExpression(`[{"b": "c"]`)
 	test.AssertResultComplex(t, "Bad expression, could not find matching `}`", err.Error())
 }
 
 func TestParserNoMatchingCloseInCollect(t *testing.T) {
-	_, err := NewExpressionParser().ParseExpression(`[(.a]`)
+	_, err := getExpressionParser().ParseExpression(`[(.a]`)
 	test.AssertResultComplex(t, "Bad expression, could not find matching `)`", err.Error())
 }
 
 func TestParserNoMatchingCloseCollectObject(t *testing.T) {
-	_, err := NewExpressionParser().ParseExpression(`{"a": "b"`)
+	_, err := getExpressionParser().ParseExpression(`{"a": "b"`)
 	test.AssertResultComplex(t, "Bad expression, could not find matching `}`", err.Error())
 }
 
 func TestParserNoMatchingCloseCollectInCollectObject(t *testing.T) {
-	_, err := NewExpressionParser().ParseExpression(`{"b": [1}`)
+	_, err := getExpressionParser().ParseExpression(`{"b": [1}`)
 	test.AssertResultComplex(t, "Bad expression, could not find matching `]`", err.Error())
 }
 
 func TestParserNoMatchingCloseBracketInCollectObject(t *testing.T) {
-	_, err := NewExpressionParser().ParseExpression(`{"b": (1}`)
+	_, err := getExpressionParser().ParseExpression(`{"b": (1}`)
 	test.AssertResultComplex(t, "Bad expression, could not find matching `)`", err.Error())
 }
 
 func TestParserNoArgsForTwoArgOp(t *testing.T) {
-	_, err := NewExpressionParser().ParseExpression("=")
+	_, err := getExpressionParser().ParseExpression("=")
 	test.AssertResultComplex(t, "'=' expects 2 args but there is 0", err.Error())
 }
 
 func TestParserOneLhsArgsForTwoArgOp(t *testing.T) {
-	_, err := NewExpressionParser().ParseExpression(".a =")
+	_, err := getExpressionParser().ParseExpression(".a =")
 	test.AssertResultComplex(t, "'=' expects 2 args but there is 1", err.Error())
 }
 
 func TestParserOneRhsArgsForTwoArgOp(t *testing.T) {
-	_, err := NewExpressionParser().ParseExpression("= .a")
+	_, err := getExpressionParser().ParseExpression("= .a")
 	test.AssertResultComplex(t, "'=' expects 2 args but there is 1", err.Error())
 }
 
 func TestParserTwoArgsForTwoArgOp(t *testing.T) {
-	_, err := NewExpressionParser().ParseExpression(".a = .b")
+	_, err := getExpressionParser().ParseExpression(".a = .b")
 	test.AssertResultComplex(t, nil, err)
 }
 
 func TestParserNoArgsForOneArgOp(t *testing.T) {
-	_, err := NewExpressionParser().ParseExpression("explode")
+	_, err := getExpressionParser().ParseExpression("explode")
 	test.AssertResultComplex(t, "'explode' expects 1 arg but received none", err.Error())
 }
 
 func TestParserOneArgForOneArgOp(t *testing.T) {
-	_, err := NewExpressionParser().ParseExpression("explode(.)")
+	_, err := getExpressionParser().ParseExpression("explode(.)")
 	test.AssertResultComplex(t, nil, err)
 }
 
 func TestParserExtraArgs(t *testing.T) {
-	_, err := NewExpressionParser().ParseExpression("sortKeys(.) explode(.)")
+	_, err := getExpressionParser().ParseExpression("sortKeys(.) explode(.)")
 	test.AssertResultComplex(t, "Bad expression, please check expression syntax", err.Error())
 }

pkg/yqlib/expression_tokeniser.go

@@ -316,6 +316,8 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`:\s*`), opToken(createMapOpType))
 	lexer.Add([]byte(`length`), opToken(lengthOpType))
 
+	lexer.Add([]byte(`eval`), opToken(evalOpType))
+
 	lexer.Add([]byte(`map`), opToken(mapOpType))
 	lexer.Add([]byte(`map_values`), opToken(mapValuesOpType))
 

pkg/yqlib/json_test.go

@@ -79,7 +79,7 @@ func decodeJson(t *testing.T, jsonString string) *CandidateNode {
 		return nil
 	}
 
-	exp, err := NewExpressionParser().ParseExpression(PrettyPrintExp)
+	exp, err := getExpressionParser().ParseExpression(PrettyPrintExp)
 
 	if err != nil {
 		t.Error(err)
@@ -124,7 +124,7 @@ func processJsonScenario(s formatScenario) string {
 		expression = "."
 	}
 
-	exp, err := NewExpressionParser().ParseExpression(expression)
+	exp, err := getExpressionParser().ParseExpression(expression)
 
 	if err != nil {
 		panic(err)

pkg/yqlib/lib.go

@@ -13,6 +13,14 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
+var ExpressionParser ExpressionParserInterface
+
+func InitExpressionParser() {
+	if ExpressionParser == nil {
+		ExpressionParser = newExpressionParser()
+	}
+}
+
 type xmlPreferences struct {
 	AttributePrefix string
 	ContentName     string
@@ -74,6 +82,7 @@ var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence:
 var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator}
 var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50, Handler: collectOperator}
 var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler: mapOperator}
+var evalOpType = &operationType{Type: "EVAL", NumArgs: 1, Precedence: 50, Handler: evalOperator}
 var mapValuesOpType = &operationType{Type: "MAP_VALUES", NumArgs: 1, Precedence: 50, Handler: mapValuesOperator}
 var encodeOpType = &operationType{Type: "ENCODE", NumArgs: 0, Precedence: 50, Handler: encodeOperator}
 var decodeOpType = &operationType{Type: "DECODE", NumArgs: 0, Precedence: 50, Handler: decodeOperator}

pkg/yqlib/operator_env_test.go

@@ -53,6 +53,16 @@ var envOperatorScenarios = []expressionScenario{
 			"D0, P[], ()::a: \"12\"\n",
 		},
 	},
+	{
+		description:         "Dynamically evaluate a path from an environment variable",
+		subdescription:      "The env variable can be any valid yq expression.",
+		document:            `{a: {b: [{name: dog}, {name: cat}]}}`,
+		environmentVariable: ".a.b[0].name",
+		expression:          `eval(strenv(myenv))`,
+		expected: []string{
+			"D0, P[a b 0 name], (!!str)::dog\n",
+		},
+	},
 	{
 		description:         "Dynamic key lookup with environment variable",
 		environmentVariable: "cat",

pkg/yqlib/operator_eval.go

@@ -0,0 +1,42 @@
+package yqlib
+
+import (
+	"container/list"
+)
+
+func evalOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	log.Debugf("Eval")
+	pathExpStrResults, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
+	if err != nil {
+		return Context{}, err
+	}
+
+	expressions := make([]*ExpressionNode, pathExpStrResults.MatchingNodes.Len())
+	expIndex := 0
+	//parse every expression
+	for pathExpStrEntry := pathExpStrResults.MatchingNodes.Front(); pathExpStrEntry != nil; pathExpStrEntry = pathExpStrEntry.Next() {
+		expressionStrCandidate := pathExpStrEntry.Value.(*CandidateNode)
+
+		expressions[expIndex], err = ExpressionParser.ParseExpression(expressionStrCandidate.Node.Value)
+		if err != nil {
+			return Context{}, err
+		}
+
+		expIndex++
+	}
+
+	results := list.New()
+
+	for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {
+		for expIndex = 0; expIndex < len(expressions); expIndex++ {
+			result, err := d.GetMatchingNodes(context, expressions[expIndex])
+			if err != nil {
+				return Context{}, err
+			}
+			results.PushBackList(result.MatchingNodes)
+		}
+	}
+
+	return context.ChildContext(results), nil
+
+}

pkg/yqlib/operator_eval_test.go

@@ -0,0 +1,34 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var evalOperatorScenarios = []expressionScenario{
+
+	{
+		description: "Dynamically evaluate a path",
+		document:    `{pathExp: '.a.b[] | select(.name == "cat")', a: {b: [{name: dog}, {name: cat}]}}`,
+		expression:  `eval(.pathExp)`,
+		expected: []string{
+			"D0, P[a b 1], (!!map)::{name: cat}\n",
+		},
+	},
+	{
+		description:         "Dynamically update a path from an environment variable",
+		subdescription:      "The env variable can be any valid yq expression.",
+		document:            `{a: {b: [{name: dog}, {name: cat}]}}`,
+		environmentVariable: ".a.b[0].name",
+		expression:          `eval(strenv(myenv)) = "cow"`,
+		expected: []string{
+			"D0, P[], (doc)::{a: {b: [{name: cow}, {name: cat}]}}\n",
+		},
+	},
+}
+
+func TestEvalOperatorsScenarios(t *testing.T) {
+	for _, tt := range evalOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentOperatorScenarios(t, "eval", evalOperatorScenarios)
+}

pkg/yqlib/operators_test.go

@@ -53,7 +53,7 @@ func readDocumentWithLeadingContent(content string, fakefilename string, fakeFil
 
 func testScenario(t *testing.T, s *expressionScenario) {
 	var err error
-	node, err := NewExpressionParser().ParseExpression(s.expression)
+	node, err := getExpressionParser().ParseExpression(s.expression)
 	if err != nil {
 		t.Error(fmt.Errorf("Error parsing expression %v of %v: %w", s.expression, s.description, err))
 		return
@@ -157,7 +157,7 @@ func formatYaml(yaml string, filename string) string {
 	var output bytes.Buffer
 	printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
 
-	node, err := NewExpressionParser().ParseExpression(".. style= \"\"")
+	node, err := getExpressionParser().ParseExpression(".. style= \"\"")
 	if err != nil {
 		panic(err)
 	}
@@ -280,7 +280,7 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt
 	var err error
 	printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
 
-	node, err := NewExpressionParser().ParseExpression(s.expression)
+	node, err := getExpressionParser().ParseExpression(s.expression)
 	if err != nil {
 		t.Error(fmt.Errorf("Error parsing expression %v of %v: %w", s.expression, s.description, err))
 		return

pkg/yqlib/printer_test.go

@@ -289,7 +289,7 @@ func TestPrinterScalarWithLeadingCont(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
 
-	node, err := NewExpressionParser().ParseExpression(".a")
+	node, err := getExpressionParser().ParseExpression(".a")
 	if err != nil {
 		panic(err)
 	}

pkg/yqlib/stream_evaluator.go

@@ -21,16 +21,15 @@ type StreamEvaluator interface {
 
 type streamEvaluator struct {
 	treeNavigator DataTreeNavigator
-	treeCreator   ExpressionParser
 	fileIndex     int
 }
 
 func NewStreamEvaluator() StreamEvaluator {
-	return &streamEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: NewExpressionParser()}
+	return &streamEvaluator{treeNavigator: NewDataTreeNavigator()}
 }
 
 func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadingContent string) error {
-	node, err := s.treeCreator.ParseExpression(expression)
+	node, err := ExpressionParser.ParseExpression(expression)
 	if err != nil {
 		return err
 	}
@@ -53,7 +52,7 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadin
 
 func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error {
 	var totalProcessDocs uint
-	node, err := s.treeCreator.ParseExpression(expression)
+	node, err := ExpressionParser.ParseExpression(expression)
 	if err != nil {
 		return err
 	}

pkg/yqlib/xml_test.go

@@ -27,7 +27,7 @@ func decodeXml(t *testing.T, s formatScenario) *CandidateNode {
 		expression = "."
 	}
 
-	exp, err := NewExpressionParser().ParseExpression(expression)
+	exp, err := getExpressionParser().ParseExpression(expression)
 
 	if err != nil {
 		t.Error(err)