Merge branch 'env_var'

pkg/yqlib/doc/Env Variable Operators.md

@@ -0,0 +1,76 @@
+
+## Read string environment variable
+Running
+```bash
+myenv="cat meow" yq eval --null-input '.a = env(myenv)'
+```
+will output
+```yaml
+a: cat meow
+```
+
+## Read boolean environment variable
+Running
+```bash
+myenv="true" yq eval --null-input '.a = env(myenv)'
+```
+will output
+```yaml
+a: true
+```
+
+## Read numeric environment variable
+Running
+```bash
+myenv="12" yq eval --null-input '.a = env(myenv)'
+```
+will output
+```yaml
+a: 12
+```
+
+## Read yaml environment variable
+Running
+```bash
+myenv="{b: fish}" yq eval --null-input '.a = env(myenv)'
+```
+will output
+```yaml
+a: {b: fish}
+```
+
+## Read boolean environment variable as a string
+Running
+```bash
+myenv="true" yq eval --null-input '.a = strenv(myenv)'
+```
+will output
+```yaml
+a: "true"
+```
+
+## Read numeric environment variable as a string
+Running
+```bash
+myenv="12" yq eval --null-input '.a = strenv(myenv)'
+```
+will output
+```yaml
+a: "12"
+```
+
+## Dynamic key lookup with environment variable
+Given a sample.yml file of:
+```yaml
+cat: meow
+dog: woof
+```
+then
+```bash
+myenv="cat" yq eval '.[env(myenv)]' sample.yml
+```
+will output
+```yaml
+meow
+```
+

pkg/yqlib/doc/Traverse.md

@@ -48,6 +48,24 @@ will output
 frog
 ```
 
+## Dynamic keys
+Expressions within [] can be used to dynamically lookup / calculate keys
+
+Given a sample.yml file of:
+```yaml
+b: apple
+apple: crispy yum
+banana: soft yum
+```
+then
+```bash
+yq eval '.[.b]' sample.yml
+```
+will output
+```yaml
+crispy yum
+```
+
 ## Children don't exist
 Nodes are added dynamically while traversing
 

pkg/yqlib/lib.go

@@ -70,6 +70,7 @@ var TraverseArray = &OperationType{Type: "TRAVERSE_ARRAY", NumArgs: 1, Precedenc
 var DocumentFilter = &OperationType{Type: "DOCUMENT_FILTER", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
 var SelfReference = &OperationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: SelfOperator}
 var ValueOp = &OperationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: ValueOperator}
+var EnvOp = &OperationType{Type: "ENV", NumArgs: 0, Precedence: 50, Handler: EnvOperator}
 var Not = &OperationType{Type: "NOT", NumArgs: 0, Precedence: 50, Handler: NotOperator}
 var Empty = &OperationType{Type: "EMPTY", NumArgs: 50, Handler: EmptyOperator}
 

pkg/yqlib/operator_env.go

@@ -0,0 +1,52 @@
+package yqlib
+
+import (
+	"container/list"
+	"os"
+	"strings"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+type EnvOpPreferences struct {
+	StringValue bool
+}
+
+func EnvOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) {
+	envName := pathNode.Operation.CandidateNode.Node.Value
+	log.Debug("EnvOperator, env name:", envName)
+
+	rawValue := os.Getenv(envName)
+
+	preferences := pathNode.Operation.Preferences.(*EnvOpPreferences)
+
+	var node *yaml.Node
+	if preferences.StringValue {
+		node = &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!str",
+			Value: rawValue,
+		}
+	} else {
+		var dataBucket yaml.Node
+		decoder := yaml.NewDecoder(strings.NewReader(rawValue))
+		errorReading := decoder.Decode(&dataBucket)
+		if errorReading != nil {
+			return nil, errorReading
+		}
+		//first node is a doc
+		node = UnwrapDoc(&dataBucket)
+	}
+	log.Debug("ENV tag", node.Tag)
+	log.Debug("ENV value", node.Value)
+	log.Debug("ENV Kind", node.Kind)
+
+	target := &CandidateNode{
+		Path:     make([]interface{}, 0),
+		Document: 0,
+		Filename: "",
+		Node:     node,
+	}
+
+	return nodeToMap(target), nil
+}

pkg/yqlib/operator_env_test.go

@@ -0,0 +1,72 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var envOperatorScenarios = []expressionScenario{
+	{
+		description:         "Read string environment variable",
+		environmentVariable: "cat meow",
+		expression:          `.a = env(myenv)`,
+		expected: []string{
+			"D0, P[], ()::a: cat meow\n",
+		},
+	},
+	{
+		description:         "Read boolean environment variable",
+		environmentVariable: "true",
+		expression:          `.a = env(myenv)`,
+		expected: []string{
+			"D0, P[], ()::a: true\n",
+		},
+	},
+	{
+		description:         "Read numeric environment variable",
+		environmentVariable: "12",
+		expression:          `.a = env(myenv)`,
+		expected: []string{
+			"D0, P[], ()::a: 12\n",
+		},
+	},
+	{
+		description:         "Read yaml environment variable",
+		environmentVariable: "{b: fish}",
+		expression:          `.a = env(myenv)`,
+		expected: []string{
+			"D0, P[], ()::a: {b: fish}\n",
+		},
+	},
+	{
+		description:         "Read boolean environment variable as a string",
+		environmentVariable: "true",
+		expression:          `.a = strenv(myenv)`,
+		expected: []string{
+			"D0, P[], ()::a: \"true\"\n",
+		},
+	},
+	{
+		description:         "Read numeric environment variable as a string",
+		environmentVariable: "12",
+		expression:          `.a = strenv(myenv)`,
+		expected: []string{
+			"D0, P[], ()::a: \"12\"\n",
+		},
+	},
+	{
+		description:         "Dynamic key lookup with environment variable",
+		environmentVariable: "cat",
+		document:            `{cat: meow, dog: woof}`,
+		expression:          `.[env(myenv)]`,
+		expected: []string{
+			"D0, P[cat], (!!str)::meow\n",
+		},
+	},
+}
+
+func TestEnvOperatorScenarios(t *testing.T) {
+	for _, tt := range envOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "Env Variable Operators", envOperatorScenarios)
+}

pkg/yqlib/operator_traverse_path_test.go

@@ -54,6 +54,15 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[{}], (!!str)::frog\n",
 		},
 	},
+	{
+		description:    "Dynamic keys",
+		subdescription: `Expressions within [] can be used to dynamically lookup / calculate keys`,
+		document:       `{b: apple, apple: crispy yum, banana: soft yum}`,
+		expression:     `.[.b]`,
+		expected: []string{
+			"D0, P[apple], (!!str)::crispy yum\n",
+		},
+	},
 	{
 		description:    "Children don't exist",
 		subdescription: "Nodes are added dynamically while traversing",

pkg/yqlib/operators_test.go

@@ -17,6 +17,7 @@ import (
 type expressionScenario struct {
 	description           string
 	subdescription        string
+	environmentVariable   string
 	document              string
 	document2             string
 	expression            string
@@ -61,6 +62,10 @@ func testScenario(t *testing.T, s *expressionScenario) {
 
 	}
 
+	if s.environmentVariable != "" {
+		os.Setenv("myenv", s.environmentVariable)
+	}
+
 	results, err = treeNavigator.GetMatchingNodes(inputs, node)
 
 	if err != nil {
@@ -162,6 +167,14 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
 	formattedDoc := ""
 	formattedDoc2 := ""
 	command := "eval"
+
+	envCommand := ""
+
+	if s.environmentVariable != "" {
+		envCommand = fmt.Sprintf("myenv=\"%v\" ", s.environmentVariable)
+		os.Setenv("myenv", s.environmentVariable)
+	}
+
 	if s.document != "" {
 		if s.dontFormatInputForDoc {
 			formattedDoc = s.document + "\n"
@@ -188,14 +201,15 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) {
 		}
 
 		writeOrPanic(w, "then\n")
+
 		if s.expression != "" {
-			writeOrPanic(w, fmt.Sprintf("```bash\nyq %v '%v' %v\n```\n", command, s.expression, files))
+			writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v '%v' %v\n```\n", envCommand, command, s.expression, files))
 		} else {
-			writeOrPanic(w, fmt.Sprintf("```bash\nyq %v %v\n```\n", command, files))
+			writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v %v\n```\n", envCommand, command, files))
 		}
 	} else {
 		writeOrPanic(w, "Running\n")
-		writeOrPanic(w, fmt.Sprintf("```bash\nyq %v --null-input '%v'\n```\n", command, s.expression))
+		writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v --null-input '%v'\n```\n", envCommand, command, s.expression))
 	}
 	return formattedDoc, formattedDoc2
 }

pkg/yqlib/path_tokeniser.go

@@ -178,6 +178,28 @@ func stringValue(wrapped bool) lex.Action {
 	}
 }
 
+func envOp(strenv bool) lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		value := string(m.Bytes)
+		preferences := &EnvOpPreferences{}
+
+		if strenv {
+			// strenv( )
+			value = value[7 : len(value)-1]
+			preferences.StringValue = true
+		} else {
+			//env( )
+			value = value[4 : len(value)-1]
+		}
+
+		envOperation := CreateValueOperation(value, value)
+		envOperation.OperationType = EnvOp
+		envOperation.Preferences = preferences
+
+		return &Token{TokenType: OperationToken, Operation: envOperation}, nil
+	}
+}
+
 func nullValue() lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		return &Token{TokenType: OperationToken, Operation: CreateValueOperation(nil, string(m.Bytes))}, nil
@@ -266,6 +288,8 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`~`), nullValue())
 
 	lexer.Add([]byte(`"[^"]*"`), stringValue(true))
+	lexer.Add([]byte(`strenv\([^\)]+\)`), envOp(true))
+	lexer.Add([]byte(`env\([^\)]+\)`), envOp(false))
 
 	lexer.Add([]byte(`\[`), literalToken(OpenCollect, false))
 	lexer.Add([]byte(`\]`), literalToken(CloseCollect, true))