From 7a184bef78bc5fb259a0c65b0adab88f41adf412 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sat, 9 Jan 2021 12:06:19 +1100 Subject: [PATCH] Env Ops! --- pkg/yqlib/doc/Env Variable Operators.md | 63 +++++++++++++++++-- pkg/yqlib/doc/Traverse.md | 18 ++++++ .../doc/headers/Env Variable Operators.md | 0 pkg/yqlib/operator_env.go | 36 ++++++++--- pkg/yqlib/operator_env_test.go | 49 +++++++++++++-- pkg/yqlib/operator_traverse_path_test.go | 9 +++ pkg/yqlib/operators_test.go | 12 ++-- pkg/yqlib/path_tokeniser.go | 12 ++-- 8 files changed, 173 insertions(+), 26 deletions(-) create mode 100644 pkg/yqlib/doc/headers/Env Variable Operators.md diff --git a/pkg/yqlib/doc/Env Variable Operators.md b/pkg/yqlib/doc/Env Variable Operators.md index 6f69515a..c166f3ef 100644 --- a/pkg/yqlib/doc/Env Variable Operators.md +++ b/pkg/yqlib/doc/Env Variable Operators.md @@ -1,21 +1,76 @@ +## Read string environment variable +Running +```bash +myenv="cat meow" yq eval --null-input '.a = env(myenv)' +``` +will output +```yaml +a: cat meow +``` + +## Read boolean environment variable +Running +```bash +myenv="true" yq eval --null-input '.a = env(myenv)' +``` +will output +```yaml +a: true +``` + +## Read numeric environment variable +Running +```bash +myenv="12" yq eval --null-input '.a = env(myenv)' +``` +will output +```yaml +a: 12 +``` + +## Read yaml environment variable +Running +```bash +myenv="{b: fish}" yq eval --null-input '.a = env(myenv)' +``` +will output +```yaml +a: {b: fish} +``` + ## Read boolean environment variable as a string Running ```bash -myenv="true" yq eval --null-input 'strenv(myenv)' +myenv="true" yq eval --null-input '.a = strenv(myenv)' ``` will output ```yaml -12 +a: "true" ``` ## Read numeric environment variable as a string Running ```bash -myenv="12" yq eval --null-input 'strenv(myenv)' +myenv="12" yq eval --null-input '.a = strenv(myenv)' ``` will output ```yaml -12 +a: "12" +``` + +## Dynamic key lookup with environment variable +Given a sample.yml file of: +```yaml +cat: meow +dog: woof +``` +then +```bash +myenv="cat" yq eval '.[env(myenv)]' sample.yml +``` +will output +```yaml +meow ``` diff --git a/pkg/yqlib/doc/Traverse.md b/pkg/yqlib/doc/Traverse.md index b08ae705..4442dc1e 100644 --- a/pkg/yqlib/doc/Traverse.md +++ b/pkg/yqlib/doc/Traverse.md @@ -48,6 +48,24 @@ will output frog ``` +## Dynamic keys +Expressions within [] can be used to dynamically lookup / calculate keys + +Given a sample.yml file of: +```yaml +b: apple +apple: crispy yum +banana: soft yum +``` +then +```bash +yq eval '.[.b]' sample.yml +``` +will output +```yaml +crispy yum +``` + ## Children don't exist Nodes are added dynamically while traversing diff --git a/pkg/yqlib/doc/headers/Env Variable Operators.md b/pkg/yqlib/doc/headers/Env Variable Operators.md new file mode 100644 index 00000000..e69de29b diff --git a/pkg/yqlib/operator_env.go b/pkg/yqlib/operator_env.go index 597a62be..4cfeeb3c 100644 --- a/pkg/yqlib/operator_env.go +++ b/pkg/yqlib/operator_env.go @@ -3,13 +3,14 @@ package yqlib import ( "container/list" "os" + "strings" yaml "gopkg.in/yaml.v3" ) -// type EnvOpPreferences struct { -// StringValue bool -// } +type EnvOpPreferences struct { + StringValue bool +} func EnvOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) { envName := pathNode.Operation.CandidateNode.Node.Value @@ -17,15 +18,34 @@ func EnvOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNo rawValue := os.Getenv(envName) + preferences := pathNode.Operation.Preferences.(*EnvOpPreferences) + + var node *yaml.Node + if preferences.StringValue { + node = &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: rawValue, + } + } else { + var dataBucket yaml.Node + decoder := yaml.NewDecoder(strings.NewReader(rawValue)) + errorReading := decoder.Decode(&dataBucket) + if errorReading != nil { + return nil, errorReading + } + //first node is a doc + node = UnwrapDoc(&dataBucket) + } + log.Debug("ENV tag", node.Tag) + log.Debug("ENV value", node.Value) + log.Debug("ENV Kind", node.Kind) + target := &CandidateNode{ Path: make([]interface{}, 0), Document: 0, Filename: "", - Node: &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: "!!str", - Value: rawValue, - }, + Node: node, } return nodeToMap(target), nil diff --git a/pkg/yqlib/operator_env_test.go b/pkg/yqlib/operator_env_test.go index 0569e38e..8066e356 100644 --- a/pkg/yqlib/operator_env_test.go +++ b/pkg/yqlib/operator_env_test.go @@ -5,20 +5,61 @@ import ( ) var envOperatorScenarios = []expressionScenario{ + { + description: "Read string environment variable", + environmentVariable: "cat meow", + expression: `.a = env(myenv)`, + expected: []string{ + "D0, P[], ()::a: cat meow\n", + }, + }, + { + description: "Read boolean environment variable", + environmentVariable: "true", + expression: `.a = env(myenv)`, + expected: []string{ + "D0, P[], ()::a: true\n", + }, + }, + { + description: "Read numeric environment variable", + environmentVariable: "12", + expression: `.a = env(myenv)`, + expected: []string{ + "D0, P[], ()::a: 12\n", + }, + }, + { + description: "Read yaml environment variable", + environmentVariable: "{b: fish}", + expression: `.a = env(myenv)`, + expected: []string{ + "D0, P[], ()::a: {b: fish}\n", + }, + }, { description: "Read boolean environment variable as a string", environmentVariable: "true", - expression: `strenv(myenv)`, + expression: `.a = strenv(myenv)`, expected: []string{ - "D0, P[], (!!str)::\"true\"\n", + "D0, P[], ()::a: \"true\"\n", }, }, { description: "Read numeric environment variable as a string", environmentVariable: "12", - expression: `strenv(myenv)`, + expression: `.a = strenv(myenv)`, expected: []string{ - "D0, P[], (!!str)::\"12\"\n", + "D0, P[], ()::a: \"12\"\n", + }, + }, + { + description: "Dynamic key lookup with environment variable", + environmentVariable: "cat", + document: `{cat: meow, dog: woof}`, + expression: `.[env(myenv)]`, + expected: []string{ + "D0, P[cat], (!!str)::meow\n", }, }, } diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go index 9d4a44a3..ffd2327d 100644 --- a/pkg/yqlib/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -54,6 +54,15 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[{}], (!!str)::frog\n", }, }, + { + description: "Dynamic keys", + subdescription: `Expressions within [] can be used to dynamically lookup / calculate keys`, + document: `{b: apple, apple: crispy yum, banana: soft yum}`, + expression: `.[.b]`, + expected: []string{ + "D0, P[apple], (!!str)::crispy yum\n", + }, + }, { description: "Children don't exist", subdescription: "Nodes are added dynamically while traversing", diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index 5064410f..5d6dc1ea 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -17,7 +17,7 @@ import ( type expressionScenario struct { description string subdescription string - environmentVariable string + environmentVariable string document string document2 string expression string @@ -170,10 +170,11 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) { envCommand := "" - if(s.environmentVariable != "") { - envCommand = fmt.Sprintf("myenv=\"%v\" ", s.environmentVariable) - } - + if s.environmentVariable != "" { + envCommand = fmt.Sprintf("myenv=\"%v\" ", s.environmentVariable) + os.Setenv("myenv", s.environmentVariable) + } + if s.document != "" { if s.dontFormatInputForDoc { formattedDoc = s.document + "\n" @@ -200,7 +201,6 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) { } writeOrPanic(w, "then\n") - if s.expression != "" { writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v '%v' %v\n```\n", envCommand, command, s.expression, files)) diff --git a/pkg/yqlib/path_tokeniser.go b/pkg/yqlib/path_tokeniser.go index 20f77fdd..a25d78c7 100644 --- a/pkg/yqlib/path_tokeniser.go +++ b/pkg/yqlib/path_tokeniser.go @@ -181,17 +181,20 @@ func stringValue(wrapped bool) lex.Action { func envOp(strenv bool) lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { value := string(m.Bytes) + preferences := &EnvOpPreferences{} if strenv { // strenv( ) - value = value[7:len(value)-1] - } else { + value = value[7 : len(value)-1] + preferences.StringValue = true + } else { //env( ) - value = value[4:len(value)-1] - } + value = value[4 : len(value)-1] + } envOperation := CreateValueOperation(value, value) envOperation.OperationType = EnvOp + envOperation.Preferences = preferences return &Token{TokenType: OperationToken, Operation: envOperation}, nil } @@ -286,6 +289,7 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`"[^"]*"`), stringValue(true)) lexer.Add([]byte(`strenv\([^\)]+\)`), envOp(true)) + lexer.Add([]byte(`env\([^\)]+\)`), envOp(false)) lexer.Add([]byte(`\[`), literalToken(OpenCollect, false)) lexer.Add([]byte(`\]`), literalToken(CloseCollect, true))