Envsubst params (#1147)

* Can give envsubst optional arguments

pkg/yqlib/doc/operators/env-variable-operators.md

@@ -8,6 +8,17 @@ There are three operators:
 - `envsubst` which you pipe strings into and it interpolates environment variables in strings using [envsubst](https://github.com/a8m/envsubst). 
 
 
+## EnvSubst Options
+You can optionally pass envsubst any of the following options:
+  - nu: NoUnset, this will fail if there are any referenced variables that are not set
+  - ne: NoEmpty, this will fail if there are any referenced variables that are empty
+  - ff: FailFast, this will abort on the first failure (rather than collect all the errors)
+
+E.g:
+`envsubst(ne, ff)` will fail on the first empty variable.
+
+See [Imposing Restrictions](https://github.com/a8m/envsubst#imposing-restrictions) in the `envsubst` documentation for more information, and below for examples.
+
 ## Tip
 To replace environment variables across all values in a document, `envsubst` can be used with the recursive descent operator
 as follows:
@@ -133,23 +144,83 @@ the cat meows
 ## Replace strings with envsubst, missing variables
 Running
 ```bash
-myenv="cat" yq --null-input '"the ${myenvnonexisting} meows" | envsubst'
+yq --null-input '"the ${myenvnonexisting} meows" | envsubst'
 ```
 will output
 ```yaml
 the  meows
 ```
 
+## Replace strings with envsubst(nu), missing variables
+(nu) not unset, will fail if there are unset (missing) variables
+
+Running
+```bash
+yq --null-input '"the ${myenvnonexisting} meows" | envsubst(nu)'
+```
+will output
+```bash
+Error: variable ${myenvnonexisting} not set
+```
+
+## Replace strings with envsubst(ne), missing variables
+(ne) not empty, only validates set variables
+
+Running
+```bash
+yq --null-input '"the ${myenvnonexisting} meows" | envsubst(ne)'
+```
+will output
+```yaml
+the  meows
+```
+
+## Replace strings with envsubst(ne), empty variable
+(ne) not empty, will fail if a references variable is empty
+
+Running
+```bash
+myenv="" yq --null-input '"the ${myenv} meows" | envsubst(ne)'
+```
+will output
+```bash
+Error: variable ${myenv} set but empty
+```
+
 ## Replace strings with envsubst, missing variables with defaults
 Running
 ```bash
-myenv="cat" yq --null-input '"the ${myenvnonexisting-dog} meows" | envsubst'
+yq --null-input '"the ${myenvnonexisting-dog} meows" | envsubst'
 ```
 will output
 ```yaml
 the dog meows
 ```
 
+## Replace strings with envsubst(nu), missing variables with defaults
+Having a default specified skips over the missing variable.
+
+Running
+```bash
+yq --null-input '"the ${myenvnonexisting-dog} meows" | envsubst(nu)'
+```
+will output
+```yaml
+the dog meows
+```
+
+## Replace strings with envsubst(ne), missing variables with defaults
+Fails, because the variable is explicitly set to blank.
+
+Running
+```bash
+myEmptyEnv="" yq --null-input '"the ${myEmptyEnv-dog} meows" | envsubst(ne)'
+```
+will output
+```bash
+Error: variable ${myEmptyEnv} set but empty
+```
+
 ## Replace string environment variable in document
 Given a sample.yml file of:
 ```yaml
@@ -164,3 +235,26 @@ will output
 v: cat meow
 ```
 
+## (Default) Return all envsubst errors
+By default, all errors are returned at once.
+
+Running
+```bash
+yq --null-input '"the ${notThere} ${alsoNotThere}" | envsubst(nu)'
+```
+will output
+```bash
+Error: variable ${notThere} not set
+variable ${alsoNotThere} not set
+```
+
+## Fail fast, return the first envsubst error (and abort)
+Running
+```bash
+yq --null-input '"the ${notThere} ${alsoNotThere}" | envsubst(nu,ff)'
+```
+will output
+```bash
+Error: variable ${notThere} not set
+```
+

pkg/yqlib/doc/operators/headers/env-variable-operators.md

@@ -8,6 +8,17 @@ There are three operators:
 - `envsubst` which you pipe strings into and it interpolates environment variables in strings using [envsubst](https://github.com/a8m/envsubst). 
 
 
+## EnvSubst Options
+You can optionally pass envsubst any of the following options:
+  - nu: NoUnset, this will fail if there are any referenced variables that are not set
+  - ne: NoEmpty, this will fail if there are any referenced variables that are empty
+  - ff: FailFast, this will abort on the first failure (rather than collect all the errors)
+
+E.g:
+`envsubst(ne, ff)` will fail on the first empty variable.
+
+See [Imposing Restrictions](https://github.com/a8m/envsubst#imposing-restrictions) in the `envsubst` documentation for more information, and below for examples.
+
 ## Tip
 To replace environment variables across all values in a document, `envsubst` can be used with the recursive descent operator
 as follows:

pkg/yqlib/expression_processing_test.go

@@ -15,6 +15,26 @@ var pathTests = []struct {
 	expectedTokens  []interface{}
 	expectedPostFix []interface{}
 }{
+	{
+		`envsubst(ne)`,
+		append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY"),
+		append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY"),
+	},
+	{
+		`envsubst(nu)`,
+		append(make([]interface{}, 0), "ENVSUBST_NO_UNSET"),
+		append(make([]interface{}, 0), "ENVSUBST_NO_UNSET"),
+	},
+	{
+		`envsubst(nu, ne)`,
+		append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY_NO_UNSET"),
+		append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY_NO_UNSET"),
+	},
+	{
+		`envsubst(ne, nu)`,
+		append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY_NO_UNSET"),
+		append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY_NO_UNSET"),
+	},
 	{
 		`[.a, .b]`,
 		append(make([]interface{}, 0), "[", "a", "UNION", "b", "]"),

pkg/yqlib/expression_tokeniser.go

@@ -135,7 +135,18 @@ func opTokenWithPrefs(op *operationType, assignOpType *operationType, preference
 	}
 }
 
-func extractNumberParamter(value string) (int, error) {
+func hasOptionParameter(value string, option string) bool {
+	parameterParser := regexp.MustCompile(`.*\([^\)]*\)`)
+	matches := parameterParser.FindStringSubmatch(value)
+	if len(matches) == 0 {
+		return false
+	}
+	parameterString := matches[0]
+	optionParser := regexp.MustCompile(fmt.Sprintf("\\b%v\\b", option))
+	return len(optionParser.FindStringSubmatch(parameterString)) > 0
+}
+
+func extractNumberParameter(value string) (int, error) {
 	parameterParser := regexp.MustCompile(`.*\(([0-9]+)\)`)
 	matches := parameterParser.FindStringSubmatch(value)
 	var indent, errParsingInt = strconv.ParseInt(matches[1], 10, 32)
@@ -145,10 +156,30 @@ func extractNumberParamter(value string) (int, error) {
 	return int(indent), nil
 }
 
+func envSubstWithOptions() lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		value := string(m.Bytes)
+		noEmpty := hasOptionParameter(value, "ne")
+		noUnset := hasOptionParameter(value, "nu")
+		failFast := hasOptionParameter(value, "ff")
+		envsubstOpType.Type = "ENVSUBST"
+		prefs := envOpPreferences{NoUnset: noUnset, NoEmpty: noEmpty, FailFast: failFast}
+		if noEmpty {
+			envsubstOpType.Type = envsubstOpType.Type + "_NO_EMPTY"
+		}
+		if noUnset {
+			envsubstOpType.Type = envsubstOpType.Type + "_NO_UNSET"
+		}
+
+		op := &Operation{OperationType: envsubstOpType, Value: envsubstOpType.Type, StringValue: value, Preferences: prefs}
+		return &token{TokenType: operationToken, Operation: op}, nil
+	}
+}
+
 func flattenWithDepth() lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		value := string(m.Bytes)
-		var depth, errParsingInt = extractNumberParamter(value)
+		var depth, errParsingInt = extractNumberParameter(value)
 		if errParsingInt != nil {
 			return nil, errParsingInt
 		}
@@ -162,7 +193,7 @@ func flattenWithDepth() lex.Action {
 func encodeWithIndent(outputFormat PrinterOutputFormat) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		value := string(m.Bytes)
-		var indent, errParsingInt = extractNumberParamter(value)
+		var indent, errParsingInt = extractNumberParameter(value)
 		if errParsingInt != nil {
 			return nil, errParsingInt
 		}
@@ -507,6 +538,7 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`strenv\([^\)]+\)`), envOp(true))
 	lexer.Add([]byte(`env\([^\)]+\)`), envOp(false))
 
+	lexer.Add([]byte(`envsubst\((ne|nu|ff| |,)+\)`), envSubstWithOptions())
 	lexer.Add([]byte(`envsubst`), opToken(envsubstOpType))
 
 	lexer.Add([]byte(`\[`), literalToken(openCollect, false))

pkg/yqlib/operator_env.go

@@ -6,12 +6,15 @@ import (
 	"os"
 	"strings"
 
-	envsubst "github.com/a8m/envsubst"
+	parse "github.com/a8m/envsubst/parse"
 	yaml "gopkg.in/yaml.v3"
 )
 
 type envOpPreferences struct {
 	StringValue bool
+	NoUnset     bool
+	NoEmpty     bool
+	FailFast    bool
 }
 
 func envOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
@@ -52,6 +55,19 @@ func envOperator(d *dataTreeNavigator, context Context, expressionNode *Expressi
 
 func envsubstOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	var results = list.New()
+	preferences := envOpPreferences{}
+	if expressionNode.Operation.Preferences != nil {
+		preferences = expressionNode.Operation.Preferences.(envOpPreferences)
+	}
+
+	parser := parse.New("string", os.Environ(),
+		&parse.Restrictions{NoUnset: preferences.NoUnset, NoEmpty: preferences.NoEmpty})
+
+	if preferences.FailFast {
+		parser.Mode = parse.Quick
+	} else {
+		parser.Mode = parse.AllErrors
+	}
 
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
@@ -61,7 +77,7 @@ func envsubstOperator(d *dataTreeNavigator, context Context, expressionNode *Exp
 			return Context{}, fmt.Errorf("cannot substitute with %v, can only substitute strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag)
 		}
 
-		value, err := envsubst.String(node.Value)
+		value, err := parser.Parse(node.Value)
 		if err != nil {
 			return Context{}, err
 		}

pkg/yqlib/operator_env_test.go

@@ -81,21 +81,55 @@ var envOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description:          "Replace strings with envsubst, missing variables",
+		description: "Replace strings with envsubst, missing variables",
-		environmentVariables: map[string]string{"myenv": "cat"},
+		expression:  `"the ${myenvnonexisting} meows" | envsubst`,
-		expression:           `"the ${myenvnonexisting} meows" | envsubst`,
 		expected: []string{
 			"D0, P[], (!!str)::the  meows\n",
 		},
 	},
 	{
-		description:          "Replace strings with envsubst, missing variables with defaults",
+		description:    "Replace strings with envsubst(nu), missing variables",
-		environmentVariables: map[string]string{"myenv": "cat"},
+		subdescription: "(nu) not unset, will fail if there are unset (missing) variables",
-		expression:           `"the ${myenvnonexisting-dog} meows" | envsubst`,
+		expression:     `"the ${myenvnonexisting} meows" | envsubst(nu)`,
+		expectedError:  "variable ${myenvnonexisting} not set",
+	},
+	{
+		description:    "Replace strings with envsubst(ne), missing variables",
+		subdescription: "(ne) not empty, only validates set variables",
+		expression:     `"the ${myenvnonexisting} meows" | envsubst(ne)`,
+		expected: []string{
+			"D0, P[], (!!str)::the  meows\n",
+		},
+	},
+	{
+		description:          "Replace strings with envsubst(ne), empty variable",
+		subdescription:       "(ne) not empty, will fail if a references variable is empty",
+		environmentVariables: map[string]string{"myenv": ""},
+		expression:           `"the ${myenv} meows" | envsubst(ne)`,
+		expectedError:        "variable ${myenv} set but empty",
+	},
+	{
+		description: "Replace strings with envsubst, missing variables with defaults",
+		expression:  `"the ${myenvnonexisting-dog} meows" | envsubst`,
 		expected: []string{
 			"D0, P[], (!!str)::the dog meows\n",
 		},
 	},
+	{
+		description:    "Replace strings with envsubst(nu), missing variables with defaults",
+		subdescription: "Having a default specified skips over the missing variable.",
+		expression:     `"the ${myenvnonexisting-dog} meows" | envsubst(nu)`,
+		expected: []string{
+			"D0, P[], (!!str)::the dog meows\n",
+		},
+	},
+	{
+		description:          "Replace strings with envsubst(ne), missing variables with defaults",
+		subdescription:       "Fails, because the variable is explicitly set to blank.",
+		environmentVariables: map[string]string{"myEmptyEnv": ""},
+		expression:           `"the ${myEmptyEnv-dog} meows" | envsubst(ne)`,
+		expectedError:        "variable ${myEmptyEnv} set but empty",
+	},
 	{
 		description:          "Replace string environment variable in document",
 		environmentVariables: map[string]string{"myenv": "cat meow"},
@@ -105,6 +139,17 @@ var envOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{v: \"cat meow\"}\n",
 		},
 	},
+	{
+		description:    "(Default) Return all envsubst errors",
+		subdescription: "By default, all errors are returned at once.",
+		expression:     `"the ${notThere} ${alsoNotThere}" | envsubst(nu)`,
+		expectedError:  "variable ${notThere} not set\nvariable ${alsoNotThere} not set",
+	},
+	{
+		description:   "Fail fast, return the first envsubst error (and abort)",
+		expression:    `"the ${notThere} ${alsoNotThere}" | envsubst(nu,ff)`,
+		expectedError: "variable ${notThere} not set",
+	},
 }
 
 func TestEnvOperatorScenarios(t *testing.T) {

pkg/yqlib/operators_test.go

@@ -105,7 +105,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
 	}
 
 	if err != nil {
-		t.Error(fmt.Errorf("%w: %v", err, s.expression))
+		t.Error(fmt.Errorf("%w: %v: %v", err, s.description, s.expression))
 		return
 	}
 	test.AssertResultComplexWithContext(t, s.expected, resultsToString(t, context.MatchingNodes), fmt.Sprintf("desc: %v\nexp: %v\ndoc: %v", s.description, s.expression, s.document))
@@ -343,8 +343,13 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt
 	}
 
 	context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: inputs}, node)
-	if err != nil {
+
+	if s.expectedError != "" && err != nil {
+		writeOrPanic(w, fmt.Sprintf("```bash\nError: %v\n```\n\n", err.Error()))
+		return
+	} else if err != nil {
 		t.Error(err, s.expression)
+		return
 	}
 
 	err = printer.PrintResults(context.MatchingNodes)