Add a new `envsubst` operator to replace environment variables in strings (#1082)

go.mod

@@ -1,6 +1,7 @@
 module github.com/mikefarah/yq/v4
 
 require (
+	github.com/a8m/envsubst v1.2.0
 	github.com/elliotchance/orderedmap v1.4.0
 	github.com/fatih/color v1.13.0
 	github.com/goccy/go-yaml v1.9.5

go.sum

@@ -50,6 +50,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/a8m/envsubst v1.2.0 h1:yvzAhJD2QKdo35Ut03wIfXQmg+ta3wC/1bskfZynz+Q=
+github.com/a8m/envsubst v1.2.0/go.mod h1:PpvLvNWa+Rvu/10qXmFbFiGICIU5hZvFJNPCCkUaObg=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=

pkg/yqlib/doc/operators/envsubst.md

@@ -0,0 +1,55 @@
+# Envsubst
+
+This operator is used to replace environment variables in strings using [envsubst](https://github.com/a8m/envsubst).
+
+To replace environment variables across all values in a document, this can be used with the recursive descent operator
+as follows:
+
+```bash
+yq eval '(.. | select(tag == "!!str)) |= envsubst' file.yaml
+```
+
+## Replace strings with envsubst
+Running
+```bash
+myenv="cat" yq eval --null-input '"the ${myenv} meows" | envsubst'
+```
+will output
+```yaml
+the cat meows
+```
+
+## Replace strings with envsubst, missing variables
+Running
+```bash
+myenv="cat" yq eval --null-input '"the ${myenvnonexisting} meows" | envsubst'
+```
+will output
+```yaml
+the  meows
+```
+
+## Replace strings with envsubst, missing variables with defaults
+Running
+```bash
+myenv="cat" yq eval --null-input '"the ${myenvnonexisting-dog} meows" | envsubst'
+```
+will output
+```yaml
+the dog meows
+```
+
+## Replace string environment variable in document
+Given a sample.yml file of:
+```yaml
+v: ${myenv}
+```
+then
+```bash
+myenv="cat meow" yq eval '.v |= envsubst' sample.yml
+```
+will output
+```yaml
+v: cat meow
+```
+

pkg/yqlib/doc/operators/headers/envsubst.md

@@ -0,0 +1,10 @@
+# Envsubst
+
+This operator is used to replace environment variables in strings using [envsubst](https://github.com/a8m/envsubst).
+
+To replace environment variables across all values in a document, this can be used with the recursive descent operator
+as follows:
+
+```bash
+yq eval '(.. | select(tag == "!!str")) |= envsubst' file.yaml
+```

pkg/yqlib/expression_tokeniser.go

@@ -472,6 +472,8 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`strenv\([^\)]+\)`), envOp(true))
 	lexer.Add([]byte(`env\([^\)]+\)`), envOp(false))
 
+	lexer.Add([]byte(`envsubst`), opToken(envsubstOpType))
+
 	lexer.Add([]byte(`\[`), literalToken(openCollect, false))
 	lexer.Add([]byte(`\]\??`), literalToken(closeCollect, true))
 	lexer.Add([]byte(`\{`), literalToken(openCollectObject, false))

pkg/yqlib/lib.go

@@ -131,6 +131,8 @@ var envOpType = &operationType{Type: "ENV", NumArgs: 0, Precedence: 50, Handler:
 var notOpType = &operationType{Type: "NOT", NumArgs: 0, Precedence: 50, Handler: notOperator}
 var emptyOpType = &operationType{Type: "EMPTY", Precedence: 50, Handler: emptyOperator}
 
+var envsubstOpType = &operationType{Type: "ENVSUBST", NumArgs: 0, Precedence: 50, Handler: envsubstOperator}
+
 var recursiveDescentOpType = &operationType{Type: "RECURSIVE_DESCENT", NumArgs: 0, Precedence: 50, Handler: recursiveDescentOperator}
 
 var selectOpType = &operationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Handler: selectOperator}

pkg/yqlib/operator_envsubst.go

@@ -0,0 +1,32 @@
+package yqlib
+
+import (
+	"container/list"
+	"fmt"
+
+	envsubst "github.com/a8m/envsubst"
+	yaml "gopkg.in/yaml.v3"
+)
+
+func envsubstOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		node := unwrapDoc(candidate.Node)
+		if node.Tag != "!!str" {
+			log.Warning("EnvSubstOperator, env name:", node.Tag, node.Value)
+			return Context{}, fmt.Errorf("cannot substitute with %v, can only substitute strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
+		}
+
+		value, err := envsubst.String(node.Value)
+		if err != nil {
+			return Context{}, err
+		}
+		targetNode := &yaml.Node{Kind: yaml.ScalarNode, Value: value, Tag: "!!str"}
+		result := candidate.CreateReplacement(targetNode)
+		results.PushBack(result)
+	}
+
+	return context.ChildContext(results), nil
+}

pkg/yqlib/operator_envsubst_test.go

@@ -0,0 +1,48 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var envsubstOperatorScenarios = []expressionScenario{
+	{
+		description:         "Replace strings with envsubst",
+		environmentVariable: "cat",
+		expression:          `"the ${myenv} meows" | envsubst`,
+		expected: []string{
+			"D0, P[], (!!str)::the cat meows\n",
+		},
+	},
+	{
+		description:         "Replace strings with envsubst, missing variables",
+		environmentVariable: "cat",
+		expression:          `"the ${myenvnonexisting} meows" | envsubst`,
+		expected: []string{
+			"D0, P[], (!!str)::the  meows\n",
+		},
+	},
+	{
+		description:         "Replace strings with envsubst, missing variables with defaults",
+		environmentVariable: "cat",
+		expression:          `"the ${myenvnonexisting-dog} meows" | envsubst`,
+		expected: []string{
+			"D0, P[], (!!str)::the dog meows\n",
+		},
+	},
+	{
+		description:         "Replace string environment variable in document",
+		environmentVariable: "cat meow",
+		document:            "{v: \"${myenv}\"}",
+		expression:          `.v |= envsubst`,
+		expected: []string{
+			"D0, P[], (doc)::{v: \"cat meow\"}\n",
+		},
+	},
+}
+
+func TestEnvSubstOperatorScenarios(t *testing.T) {
+	for _, tt := range envsubstOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentOperatorScenarios(t, "envsubst", envsubstOperatorScenarios)
+}