diff --git a/pkg/yqlib/doc/Env Variable Operators.md b/pkg/yqlib/doc/Env Variable Operators.md new file mode 100644 index 00000000..c166f3ef --- /dev/null +++ b/pkg/yqlib/doc/Env Variable Operators.md @@ -0,0 +1,76 @@ + +## Read string environment variable +Running +```bash +myenv="cat meow" yq eval --null-input '.a = env(myenv)' +``` +will output +```yaml +a: cat meow +``` + +## Read boolean environment variable +Running +```bash +myenv="true" yq eval --null-input '.a = env(myenv)' +``` +will output +```yaml +a: true +``` + +## Read numeric environment variable +Running +```bash +myenv="12" yq eval --null-input '.a = env(myenv)' +``` +will output +```yaml +a: 12 +``` + +## Read yaml environment variable +Running +```bash +myenv="{b: fish}" yq eval --null-input '.a = env(myenv)' +``` +will output +```yaml +a: {b: fish} +``` + +## Read boolean environment variable as a string +Running +```bash +myenv="true" yq eval --null-input '.a = strenv(myenv)' +``` +will output +```yaml +a: "true" +``` + +## Read numeric environment variable as a string +Running +```bash +myenv="12" yq eval --null-input '.a = strenv(myenv)' +``` +will output +```yaml +a: "12" +``` + +## Dynamic key lookup with environment variable +Given a sample.yml file of: +```yaml +cat: meow +dog: woof +``` +then +```bash +myenv="cat" yq eval '.[env(myenv)]' sample.yml +``` +will output +```yaml +meow +``` + diff --git a/pkg/yqlib/doc/Traverse.md b/pkg/yqlib/doc/Traverse.md index b08ae705..4442dc1e 100644 --- a/pkg/yqlib/doc/Traverse.md +++ b/pkg/yqlib/doc/Traverse.md @@ -48,6 +48,24 @@ will output frog ``` +## Dynamic keys +Expressions within [] can be used to dynamically lookup / calculate keys + +Given a sample.yml file of: +```yaml +b: apple +apple: crispy yum +banana: soft yum +``` +then +```bash +yq eval '.[.b]' sample.yml +``` +will output +```yaml +crispy yum +``` + ## Children don't exist Nodes are added dynamically while traversing diff --git a/pkg/yqlib/doc/headers/Env Variable Operators.md b/pkg/yqlib/doc/headers/Env Variable Operators.md new file mode 100644 index 00000000..e69de29b diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index fbe69dd0..db8402f3 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -70,6 +70,7 @@ var TraverseArray = &OperationType{Type: "TRAVERSE_ARRAY", NumArgs: 1, Precedenc var DocumentFilter = &OperationType{Type: "DOCUMENT_FILTER", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator} var SelfReference = &OperationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: SelfOperator} var ValueOp = &OperationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: ValueOperator} +var EnvOp = &OperationType{Type: "ENV", NumArgs: 0, Precedence: 50, Handler: EnvOperator} var Not = &OperationType{Type: "NOT", NumArgs: 0, Precedence: 50, Handler: NotOperator} var Empty = &OperationType{Type: "EMPTY", NumArgs: 50, Handler: EmptyOperator} diff --git a/pkg/yqlib/operator_env.go b/pkg/yqlib/operator_env.go new file mode 100644 index 00000000..4cfeeb3c --- /dev/null +++ b/pkg/yqlib/operator_env.go @@ -0,0 +1,52 @@ +package yqlib + +import ( + "container/list" + "os" + "strings" + + yaml "gopkg.in/yaml.v3" +) + +type EnvOpPreferences struct { + StringValue bool +} + +func EnvOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) { + envName := pathNode.Operation.CandidateNode.Node.Value + log.Debug("EnvOperator, env name:", envName) + + rawValue := os.Getenv(envName) + + preferences := pathNode.Operation.Preferences.(*EnvOpPreferences) + + var node *yaml.Node + if preferences.StringValue { + node = &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: rawValue, + } + } else { + var dataBucket yaml.Node + decoder := yaml.NewDecoder(strings.NewReader(rawValue)) + errorReading := decoder.Decode(&dataBucket) + if errorReading != nil { + return nil, errorReading + } + //first node is a doc + node = UnwrapDoc(&dataBucket) + } + log.Debug("ENV tag", node.Tag) + log.Debug("ENV value", node.Value) + log.Debug("ENV Kind", node.Kind) + + target := &CandidateNode{ + Path: make([]interface{}, 0), + Document: 0, + Filename: "", + Node: node, + } + + return nodeToMap(target), nil +} diff --git a/pkg/yqlib/operator_env_test.go b/pkg/yqlib/operator_env_test.go new file mode 100644 index 00000000..8066e356 --- /dev/null +++ b/pkg/yqlib/operator_env_test.go @@ -0,0 +1,72 @@ +package yqlib + +import ( + "testing" +) + +var envOperatorScenarios = []expressionScenario{ + { + description: "Read string environment variable", + environmentVariable: "cat meow", + expression: `.a = env(myenv)`, + expected: []string{ + "D0, P[], ()::a: cat meow\n", + }, + }, + { + description: "Read boolean environment variable", + environmentVariable: "true", + expression: `.a = env(myenv)`, + expected: []string{ + "D0, P[], ()::a: true\n", + }, + }, + { + description: "Read numeric environment variable", + environmentVariable: "12", + expression: `.a = env(myenv)`, + expected: []string{ + "D0, P[], ()::a: 12\n", + }, + }, + { + description: "Read yaml environment variable", + environmentVariable: "{b: fish}", + expression: `.a = env(myenv)`, + expected: []string{ + "D0, P[], ()::a: {b: fish}\n", + }, + }, + { + description: "Read boolean environment variable as a string", + environmentVariable: "true", + expression: `.a = strenv(myenv)`, + expected: []string{ + "D0, P[], ()::a: \"true\"\n", + }, + }, + { + description: "Read numeric environment variable as a string", + environmentVariable: "12", + expression: `.a = strenv(myenv)`, + expected: []string{ + "D0, P[], ()::a: \"12\"\n", + }, + }, + { + description: "Dynamic key lookup with environment variable", + environmentVariable: "cat", + document: `{cat: meow, dog: woof}`, + expression: `.[env(myenv)]`, + expected: []string{ + "D0, P[cat], (!!str)::meow\n", + }, + }, +} + +func TestEnvOperatorScenarios(t *testing.T) { + for _, tt := range envOperatorScenarios { + testScenario(t, &tt) + } + documentScenarios(t, "Env Variable Operators", envOperatorScenarios) +} diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go index 9d4a44a3..ffd2327d 100644 --- a/pkg/yqlib/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -54,6 +54,15 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[{}], (!!str)::frog\n", }, }, + { + description: "Dynamic keys", + subdescription: `Expressions within [] can be used to dynamically lookup / calculate keys`, + document: `{b: apple, apple: crispy yum, banana: soft yum}`, + expression: `.[.b]`, + expected: []string{ + "D0, P[apple], (!!str)::crispy yum\n", + }, + }, { description: "Children don't exist", subdescription: "Nodes are added dynamically while traversing", diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index c8881bb5..5d6dc1ea 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -17,6 +17,7 @@ import ( type expressionScenario struct { description string subdescription string + environmentVariable string document string document2 string expression string @@ -61,6 +62,10 @@ func testScenario(t *testing.T, s *expressionScenario) { } + if s.environmentVariable != "" { + os.Setenv("myenv", s.environmentVariable) + } + results, err = treeNavigator.GetMatchingNodes(inputs, node) if err != nil { @@ -162,6 +167,14 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) { formattedDoc := "" formattedDoc2 := "" command := "eval" + + envCommand := "" + + if s.environmentVariable != "" { + envCommand = fmt.Sprintf("myenv=\"%v\" ", s.environmentVariable) + os.Setenv("myenv", s.environmentVariable) + } + if s.document != "" { if s.dontFormatInputForDoc { formattedDoc = s.document + "\n" @@ -188,14 +201,15 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) { } writeOrPanic(w, "then\n") + if s.expression != "" { - writeOrPanic(w, fmt.Sprintf("```bash\nyq %v '%v' %v\n```\n", command, s.expression, files)) + writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v '%v' %v\n```\n", envCommand, command, s.expression, files)) } else { - writeOrPanic(w, fmt.Sprintf("```bash\nyq %v %v\n```\n", command, files)) + writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v %v\n```\n", envCommand, command, files)) } } else { writeOrPanic(w, "Running\n") - writeOrPanic(w, fmt.Sprintf("```bash\nyq %v --null-input '%v'\n```\n", command, s.expression)) + writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v --null-input '%v'\n```\n", envCommand, command, s.expression)) } return formattedDoc, formattedDoc2 } diff --git a/pkg/yqlib/path_tokeniser.go b/pkg/yqlib/path_tokeniser.go index 6afac412..a25d78c7 100644 --- a/pkg/yqlib/path_tokeniser.go +++ b/pkg/yqlib/path_tokeniser.go @@ -178,6 +178,28 @@ func stringValue(wrapped bool) lex.Action { } } +func envOp(strenv bool) lex.Action { + return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { + value := string(m.Bytes) + preferences := &EnvOpPreferences{} + + if strenv { + // strenv( ) + value = value[7 : len(value)-1] + preferences.StringValue = true + } else { + //env( ) + value = value[4 : len(value)-1] + } + + envOperation := CreateValueOperation(value, value) + envOperation.OperationType = EnvOp + envOperation.Preferences = preferences + + return &Token{TokenType: OperationToken, Operation: envOperation}, nil + } +} + func nullValue() lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { return &Token{TokenType: OperationToken, Operation: CreateValueOperation(nil, string(m.Bytes))}, nil @@ -266,6 +288,8 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`~`), nullValue()) lexer.Add([]byte(`"[^"]*"`), stringValue(true)) + lexer.Add([]byte(`strenv\([^\)]+\)`), envOp(true)) + lexer.Add([]byte(`env\([^\)]+\)`), envOp(false)) lexer.Add([]byte(`\[`), literalToken(OpenCollect, false)) lexer.Add([]byte(`\]`), literalToken(CloseCollect, true))