Added eval operator

This commit is contained in:
Mike Farah 2022-02-01 14:47:51 +11:00
parent 77204551ab
commit 535799462f
19 changed files with 209 additions and 30 deletions

View File

@ -34,6 +34,7 @@ yq -i '.stuff = "foo"' myfile.yml # update myfile.yml inplace
},
PersistentPreRun: func(cmd *cobra.Command, args []string) {
cmd.SetOut(cmd.OutOrStdout())
var format = logging.MustStringFormatter(
`%{color}%{time:15:04:05} %{shortfunc} [%{level:.4s}]%{color:reset} %{message}`,
)
@ -47,6 +48,7 @@ yq -i '.stuff = "foo"' myfile.yml # update myfile.yml inplace
}
logging.SetBackend(backend)
yqlib.InitExpressionParser()
yqlib.XmlPreferences.AttributePrefix = xmlAttributePrefix
yqlib.XmlPreferences.ContentName = xmlContentName
},

View File

@ -63,7 +63,7 @@ func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yq
if splitFileExp != "" {
colorsEnabled = forceColor
splitExp, err := yqlib.NewExpressionParser().ParseExpression(splitFileExp)
splitExp, err := yqlib.ExpressionParser.ParseExpression(splitFileExp)
if err != nil {
return nil, fmt.Errorf("bad split document expression: %w", err)
}

View File

@ -19,11 +19,10 @@ type Evaluator interface {
type allAtOnceEvaluator struct {
treeNavigator DataTreeNavigator
treeCreator ExpressionParser
}
func NewAllAtOnceEvaluator() Evaluator {
return &allAtOnceEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: NewExpressionParser()}
return &allAtOnceEvaluator{treeNavigator: NewDataTreeNavigator()}
}
func (e *allAtOnceEvaluator) EvaluateNodes(expression string, nodes ...*yaml.Node) (*list.List, error) {
@ -35,7 +34,7 @@ func (e *allAtOnceEvaluator) EvaluateNodes(expression string, nodes ...*yaml.Nod
}
func (e *allAtOnceEvaluator) EvaluateCandidateNodes(expression string, inputCandidates *list.List) (*list.List, error) {
node, err := e.treeCreator.ParseExpression(expression)
node, err := ExpressionParser.ParseExpression(expression)
if err != nil {
return nil, err
}

View File

@ -31,6 +31,7 @@ var evaluateNodesScenario = []expressionScenario{
}
func TestAllAtOnceEvaluateNodes(t *testing.T) {
InitExpressionParser()
var evaluator = NewAllAtOnceEvaluator()
for _, tt := range evaluateNodesScenario {
node := test.ParseData(tt.document)

View File

@ -77,6 +77,25 @@ will output
a: "12"
```
## Dynamically evaluate a path from an environment variable
The env variable can be any valid yq expression.
Given a sample.yml file of:
```yaml
a:
b:
- name: dog
- name: cat
```
then
```bash
myenv=".a.b[0].name" yq 'eval(strenv(myenv))' sample.yml
```
will output
```yaml
dog
```
## Dynamic key lookup with environment variable
Given a sample.yml file of:
```yaml

View File

@ -0,0 +1,48 @@
# Eval
Use `eval` to dynamically process an expression - for instance from an environment variable.
`eval` takes a single argument, and evaluates that as a `yq` expression. Any valid expression can be used, beit a path `.a.b.c | select(. == "cat")`, or an update `.a.b.c = "gogo"`.
Tip: This can be useful way parameterise complex scripts.
## Dynamically evaluate a path
Given a sample.yml file of:
```yaml
pathExp: .a.b[] | select(.name == "cat")
a:
b:
- name: dog
- name: cat
```
then
```bash
yq 'eval(.pathExp)' sample.yml
```
will output
```yaml
name: cat
```
## Dynamically update a path from an environment variable
The env variable can be any valid yq expression.
Given a sample.yml file of:
```yaml
a:
b:
- name: dog
- name: cat
```
then
```bash
myenv=".a.b[0].name" yq 'eval(strenv(myenv)) = "cow"' sample.yml
```
will output
```yaml
a:
b:
- name: cow
- name: cat
```

View File

@ -0,0 +1,7 @@
# Eval
Use `eval` to dynamically process an expression - for instance from an environment variable.
`eval` takes a single argument, and evaluates that as a `yq` expression. Any valid expression can be used, beit a path `.a.b.c | select(. == "cat")`, or an update `.a.b.c = "gogo"`.
Tip: This can be useful way parameterise complex scripts.

View File

@ -11,7 +11,7 @@ type ExpressionNode struct {
Rhs *ExpressionNode
}
type ExpressionParser interface {
type ExpressionParserInterface interface {
ParseExpression(expression string) (*ExpressionNode, error)
}
@ -20,7 +20,7 @@ type expressionParserImpl struct {
pathPostFixer expressionPostFixer
}
func NewExpressionParser() ExpressionParser {
func newExpressionParser() ExpressionParserInterface {
return &expressionParserImpl{newExpressionTokeniser(), newExpressionPostFixer()}
}

View File

@ -6,66 +6,73 @@ import (
"github.com/mikefarah/yq/v4/test"
)
func getExpressionParser() ExpressionParserInterface {
if ExpressionParser == nil {
ExpressionParser = newExpressionParser()
}
return ExpressionParser
}
func TestParserNoMatchingCloseCollect(t *testing.T) {
_, err := NewExpressionParser().ParseExpression("[1,2")
_, err := getExpressionParser().ParseExpression("[1,2")
test.AssertResultComplex(t, "Bad expression, could not find matching `]`", err.Error())
}
func TestParserNoMatchingCloseObjectInCollect(t *testing.T) {
_, err := NewExpressionParser().ParseExpression(`[{"b": "c"]`)
_, err := getExpressionParser().ParseExpression(`[{"b": "c"]`)
test.AssertResultComplex(t, "Bad expression, could not find matching `}`", err.Error())
}
func TestParserNoMatchingCloseInCollect(t *testing.T) {
_, err := NewExpressionParser().ParseExpression(`[(.a]`)
_, err := getExpressionParser().ParseExpression(`[(.a]`)
test.AssertResultComplex(t, "Bad expression, could not find matching `)`", err.Error())
}
func TestParserNoMatchingCloseCollectObject(t *testing.T) {
_, err := NewExpressionParser().ParseExpression(`{"a": "b"`)
_, err := getExpressionParser().ParseExpression(`{"a": "b"`)
test.AssertResultComplex(t, "Bad expression, could not find matching `}`", err.Error())
}
func TestParserNoMatchingCloseCollectInCollectObject(t *testing.T) {
_, err := NewExpressionParser().ParseExpression(`{"b": [1}`)
_, err := getExpressionParser().ParseExpression(`{"b": [1}`)
test.AssertResultComplex(t, "Bad expression, could not find matching `]`", err.Error())
}
func TestParserNoMatchingCloseBracketInCollectObject(t *testing.T) {
_, err := NewExpressionParser().ParseExpression(`{"b": (1}`)
_, err := getExpressionParser().ParseExpression(`{"b": (1}`)
test.AssertResultComplex(t, "Bad expression, could not find matching `)`", err.Error())
}
func TestParserNoArgsForTwoArgOp(t *testing.T) {
_, err := NewExpressionParser().ParseExpression("=")
_, err := getExpressionParser().ParseExpression("=")
test.AssertResultComplex(t, "'=' expects 2 args but there is 0", err.Error())
}
func TestParserOneLhsArgsForTwoArgOp(t *testing.T) {
_, err := NewExpressionParser().ParseExpression(".a =")
_, err := getExpressionParser().ParseExpression(".a =")
test.AssertResultComplex(t, "'=' expects 2 args but there is 1", err.Error())
}
func TestParserOneRhsArgsForTwoArgOp(t *testing.T) {
_, err := NewExpressionParser().ParseExpression("= .a")
_, err := getExpressionParser().ParseExpression("= .a")
test.AssertResultComplex(t, "'=' expects 2 args but there is 1", err.Error())
}
func TestParserTwoArgsForTwoArgOp(t *testing.T) {
_, err := NewExpressionParser().ParseExpression(".a = .b")
_, err := getExpressionParser().ParseExpression(".a = .b")
test.AssertResultComplex(t, nil, err)
}
func TestParserNoArgsForOneArgOp(t *testing.T) {
_, err := NewExpressionParser().ParseExpression("explode")
_, err := getExpressionParser().ParseExpression("explode")
test.AssertResultComplex(t, "'explode' expects 1 arg but received none", err.Error())
}
func TestParserOneArgForOneArgOp(t *testing.T) {
_, err := NewExpressionParser().ParseExpression("explode(.)")
_, err := getExpressionParser().ParseExpression("explode(.)")
test.AssertResultComplex(t, nil, err)
}
func TestParserExtraArgs(t *testing.T) {
_, err := NewExpressionParser().ParseExpression("sortKeys(.) explode(.)")
_, err := getExpressionParser().ParseExpression("sortKeys(.) explode(.)")
test.AssertResultComplex(t, "Bad expression, please check expression syntax", err.Error())
}

View File

@ -316,6 +316,8 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`:\s*`), opToken(createMapOpType))
lexer.Add([]byte(`length`), opToken(lengthOpType))
lexer.Add([]byte(`eval`), opToken(evalOpType))
lexer.Add([]byte(`map`), opToken(mapOpType))
lexer.Add([]byte(`map_values`), opToken(mapValuesOpType))

View File

@ -79,7 +79,7 @@ func decodeJson(t *testing.T, jsonString string) *CandidateNode {
return nil
}
exp, err := NewExpressionParser().ParseExpression(PrettyPrintExp)
exp, err := getExpressionParser().ParseExpression(PrettyPrintExp)
if err != nil {
t.Error(err)
@ -124,7 +124,7 @@ func processJsonScenario(s formatScenario) string {
expression = "."
}
exp, err := NewExpressionParser().ParseExpression(expression)
exp, err := getExpressionParser().ParseExpression(expression)
if err != nil {
panic(err)

View File

@ -13,6 +13,14 @@ import (
yaml "gopkg.in/yaml.v3"
)
var ExpressionParser ExpressionParserInterface
func InitExpressionParser() {
if ExpressionParser == nil {
ExpressionParser = newExpressionParser()
}
}
type xmlPreferences struct {
AttributePrefix string
ContentName string
@ -74,6 +82,7 @@ var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence:
var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator}
var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50, Handler: collectOperator}
var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler: mapOperator}
var evalOpType = &operationType{Type: "EVAL", NumArgs: 1, Precedence: 50, Handler: evalOperator}
var mapValuesOpType = &operationType{Type: "MAP_VALUES", NumArgs: 1, Precedence: 50, Handler: mapValuesOperator}
var encodeOpType = &operationType{Type: "ENCODE", NumArgs: 0, Precedence: 50, Handler: encodeOperator}
var decodeOpType = &operationType{Type: "DECODE", NumArgs: 0, Precedence: 50, Handler: decodeOperator}

View File

@ -53,6 +53,16 @@ var envOperatorScenarios = []expressionScenario{
"D0, P[], ()::a: \"12\"\n",
},
},
{
description: "Dynamically evaluate a path from an environment variable",
subdescription: "The env variable can be any valid yq expression.",
document: `{a: {b: [{name: dog}, {name: cat}]}}`,
environmentVariable: ".a.b[0].name",
expression: `eval(strenv(myenv))`,
expected: []string{
"D0, P[a b 0 name], (!!str)::dog\n",
},
},
{
description: "Dynamic key lookup with environment variable",
environmentVariable: "cat",

View File

@ -0,0 +1,42 @@
package yqlib
import (
"container/list"
)
func evalOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debugf("Eval")
pathExpStrResults, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
if err != nil {
return Context{}, err
}
expressions := make([]*ExpressionNode, pathExpStrResults.MatchingNodes.Len())
expIndex := 0
//parse every expression
for pathExpStrEntry := pathExpStrResults.MatchingNodes.Front(); pathExpStrEntry != nil; pathExpStrEntry = pathExpStrEntry.Next() {
expressionStrCandidate := pathExpStrEntry.Value.(*CandidateNode)
expressions[expIndex], err = ExpressionParser.ParseExpression(expressionStrCandidate.Node.Value)
if err != nil {
return Context{}, err
}
expIndex++
}
results := list.New()
for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {
for expIndex = 0; expIndex < len(expressions); expIndex++ {
result, err := d.GetMatchingNodes(context, expressions[expIndex])
if err != nil {
return Context{}, err
}
results.PushBackList(result.MatchingNodes)
}
}
return context.ChildContext(results), nil
}

View File

@ -0,0 +1,34 @@
package yqlib
import (
"testing"
)
var evalOperatorScenarios = []expressionScenario{
{
description: "Dynamically evaluate a path",
document: `{pathExp: '.a.b[] | select(.name == "cat")', a: {b: [{name: dog}, {name: cat}]}}`,
expression: `eval(.pathExp)`,
expected: []string{
"D0, P[a b 1], (!!map)::{name: cat}\n",
},
},
{
description: "Dynamically update a path from an environment variable",
subdescription: "The env variable can be any valid yq expression.",
document: `{a: {b: [{name: dog}, {name: cat}]}}`,
environmentVariable: ".a.b[0].name",
expression: `eval(strenv(myenv)) = "cow"`,
expected: []string{
"D0, P[], (doc)::{a: {b: [{name: cow}, {name: cat}]}}\n",
},
},
}
func TestEvalOperatorsScenarios(t *testing.T) {
for _, tt := range evalOperatorScenarios {
testScenario(t, &tt)
}
documentOperatorScenarios(t, "eval", evalOperatorScenarios)
}

View File

@ -53,7 +53,7 @@ func readDocumentWithLeadingContent(content string, fakefilename string, fakeFil
func testScenario(t *testing.T, s *expressionScenario) {
var err error
node, err := NewExpressionParser().ParseExpression(s.expression)
node, err := getExpressionParser().ParseExpression(s.expression)
if err != nil {
t.Error(fmt.Errorf("Error parsing expression %v of %v: %w", s.expression, s.description, err))
return
@ -157,7 +157,7 @@ func formatYaml(yaml string, filename string) string {
var output bytes.Buffer
printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
node, err := NewExpressionParser().ParseExpression(".. style= \"\"")
node, err := getExpressionParser().ParseExpression(".. style= \"\"")
if err != nil {
panic(err)
}
@ -280,7 +280,7 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt
var err error
printer := NewSimpleYamlPrinter(bufio.NewWriter(&output), YamlOutputFormat, true, false, 2, true)
node, err := NewExpressionParser().ParseExpression(s.expression)
node, err := getExpressionParser().ParseExpression(s.expression)
if err != nil {
t.Error(fmt.Errorf("Error parsing expression %v of %v: %w", s.expression, s.description, err))
return

View File

@ -289,7 +289,7 @@ func TestPrinterScalarWithLeadingCont(t *testing.T) {
var writer = bufio.NewWriter(&output)
printer := NewSimpleYamlPrinter(writer, YamlOutputFormat, true, false, 2, true)
node, err := NewExpressionParser().ParseExpression(".a")
node, err := getExpressionParser().ParseExpression(".a")
if err != nil {
panic(err)
}

View File

@ -21,16 +21,15 @@ type StreamEvaluator interface {
type streamEvaluator struct {
treeNavigator DataTreeNavigator
treeCreator ExpressionParser
fileIndex int
}
func NewStreamEvaluator() StreamEvaluator {
return &streamEvaluator{treeNavigator: NewDataTreeNavigator(), treeCreator: NewExpressionParser()}
return &streamEvaluator{treeNavigator: NewDataTreeNavigator()}
}
func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadingContent string) error {
node, err := s.treeCreator.ParseExpression(expression)
node, err := ExpressionParser.ParseExpression(expression)
if err != nil {
return err
}
@ -53,7 +52,7 @@ func (s *streamEvaluator) EvaluateNew(expression string, printer Printer, leadin
func (s *streamEvaluator) EvaluateFiles(expression string, filenames []string, printer Printer, leadingContentPreProcessing bool, decoder Decoder) error {
var totalProcessDocs uint
node, err := s.treeCreator.ParseExpression(expression)
node, err := ExpressionParser.ParseExpression(expression)
if err != nil {
return err
}

View File

@ -27,7 +27,7 @@ func decodeXml(t *testing.T, s formatScenario) *CandidateNode {
expression = "."
}
exp, err := NewExpressionParser().ParseExpression(expression)
exp, err := getExpressionParser().ParseExpression(expression)
if err != nil {
t.Error(err)