diff --git a/pkg/yqlib/doc/operators/env-variable-operators.md b/pkg/yqlib/doc/operators/env-variable-operators.md index b8adf077..f837a2a7 100644 --- a/pkg/yqlib/doc/operators/env-variable-operators.md +++ b/pkg/yqlib/doc/operators/env-variable-operators.md @@ -8,6 +8,17 @@ There are three operators: - `envsubst` which you pipe strings into and it interpolates environment variables in strings using [envsubst](https://github.com/a8m/envsubst). +## EnvSubst Options +You can optionally pass envsubst any of the following options: + - nu: NoUnset, this will fail if there are any referenced variables that are not set + - ne: NoEmpty, this will fail if there are any referenced variables that are empty + - ff: FailFast, this will abort on the first failure (rather than collect all the errors) + +E.g: +`envsubst(ne, ff)` will fail on the first empty variable. + +See [Imposing Restrictions](https://github.com/a8m/envsubst#imposing-restrictions) in the `envsubst` documentation for more information, and below for examples. + ## Tip To replace environment variables across all values in a document, `envsubst` can be used with the recursive descent operator as follows: @@ -133,23 +144,83 @@ the cat meows ## Replace strings with envsubst, missing variables Running ```bash -myenv="cat" yq --null-input '"the ${myenvnonexisting} meows" | envsubst' +yq --null-input '"the ${myenvnonexisting} meows" | envsubst' ``` will output ```yaml the meows ``` +## Replace strings with envsubst(nu), missing variables +(nu) not unset, will fail if there are unset (missing) variables + +Running +```bash +yq --null-input '"the ${myenvnonexisting} meows" | envsubst(nu)' +``` +will output +```bash +Error: variable ${myenvnonexisting} not set +``` + +## Replace strings with envsubst(ne), missing variables +(ne) not empty, only validates set variables + +Running +```bash +yq --null-input '"the ${myenvnonexisting} meows" | envsubst(ne)' +``` +will output +```yaml +the meows +``` + +## Replace strings with envsubst(ne), empty variable +(ne) not empty, will fail if a references variable is empty + +Running +```bash +myenv="" yq --null-input '"the ${myenv} meows" | envsubst(ne)' +``` +will output +```bash +Error: variable ${myenv} set but empty +``` + ## Replace strings with envsubst, missing variables with defaults Running ```bash -myenv="cat" yq --null-input '"the ${myenvnonexisting-dog} meows" | envsubst' +yq --null-input '"the ${myenvnonexisting-dog} meows" | envsubst' ``` will output ```yaml the dog meows ``` +## Replace strings with envsubst(nu), missing variables with defaults +Having a default specified skips over the missing variable. + +Running +```bash +yq --null-input '"the ${myenvnonexisting-dog} meows" | envsubst(nu)' +``` +will output +```yaml +the dog meows +``` + +## Replace strings with envsubst(ne), missing variables with defaults +Fails, because the variable is explicitly set to blank. + +Running +```bash +myEmptyEnv="" yq --null-input '"the ${myEmptyEnv-dog} meows" | envsubst(ne)' +``` +will output +```bash +Error: variable ${myEmptyEnv} set but empty +``` + ## Replace string environment variable in document Given a sample.yml file of: ```yaml @@ -164,3 +235,26 @@ will output v: cat meow ``` +## (Default) Return all envsubst errors +By default, all errors are returned at once. + +Running +```bash +yq --null-input '"the ${notThere} ${alsoNotThere}" | envsubst(nu)' +``` +will output +```bash +Error: variable ${notThere} not set +variable ${alsoNotThere} not set +``` + +## Fail fast, return the first envsubst error (and abort) +Running +```bash +yq --null-input '"the ${notThere} ${alsoNotThere}" | envsubst(nu,ff)' +``` +will output +```bash +Error: variable ${notThere} not set +``` + diff --git a/pkg/yqlib/doc/operators/headers/env-variable-operators.md b/pkg/yqlib/doc/operators/headers/env-variable-operators.md index fe8c2bfa..b4765c2a 100644 --- a/pkg/yqlib/doc/operators/headers/env-variable-operators.md +++ b/pkg/yqlib/doc/operators/headers/env-variable-operators.md @@ -8,6 +8,17 @@ There are three operators: - `envsubst` which you pipe strings into and it interpolates environment variables in strings using [envsubst](https://github.com/a8m/envsubst). +## EnvSubst Options +You can optionally pass envsubst any of the following options: + - nu: NoUnset, this will fail if there are any referenced variables that are not set + - ne: NoEmpty, this will fail if there are any referenced variables that are empty + - ff: FailFast, this will abort on the first failure (rather than collect all the errors) + +E.g: +`envsubst(ne, ff)` will fail on the first empty variable. + +See [Imposing Restrictions](https://github.com/a8m/envsubst#imposing-restrictions) in the `envsubst` documentation for more information, and below for examples. + ## Tip To replace environment variables across all values in a document, `envsubst` can be used with the recursive descent operator as follows: diff --git a/pkg/yqlib/expression_processing_test.go b/pkg/yqlib/expression_processing_test.go index 3e4b38bf..18b13a9b 100644 --- a/pkg/yqlib/expression_processing_test.go +++ b/pkg/yqlib/expression_processing_test.go @@ -15,6 +15,26 @@ var pathTests = []struct { expectedTokens []interface{} expectedPostFix []interface{} }{ + { + `envsubst(ne)`, + append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY"), + append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY"), + }, + { + `envsubst(nu)`, + append(make([]interface{}, 0), "ENVSUBST_NO_UNSET"), + append(make([]interface{}, 0), "ENVSUBST_NO_UNSET"), + }, + { + `envsubst(nu, ne)`, + append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY_NO_UNSET"), + append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY_NO_UNSET"), + }, + { + `envsubst(ne, nu)`, + append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY_NO_UNSET"), + append(make([]interface{}, 0), "ENVSUBST_NO_EMPTY_NO_UNSET"), + }, { `[.a, .b]`, append(make([]interface{}, 0), "[", "a", "UNION", "b", "]"), diff --git a/pkg/yqlib/expression_tokeniser.go b/pkg/yqlib/expression_tokeniser.go index 57c0bde9..ded0bd7e 100644 --- a/pkg/yqlib/expression_tokeniser.go +++ b/pkg/yqlib/expression_tokeniser.go @@ -135,7 +135,18 @@ func opTokenWithPrefs(op *operationType, assignOpType *operationType, preference } } -func extractNumberParamter(value string) (int, error) { +func hasOptionParameter(value string, option string) bool { + parameterParser := regexp.MustCompile(`.*\([^\)]*\)`) + matches := parameterParser.FindStringSubmatch(value) + if len(matches) == 0 { + return false + } + parameterString := matches[0] + optionParser := regexp.MustCompile(fmt.Sprintf("\\b%v\\b", option)) + return len(optionParser.FindStringSubmatch(parameterString)) > 0 +} + +func extractNumberParameter(value string) (int, error) { parameterParser := regexp.MustCompile(`.*\(([0-9]+)\)`) matches := parameterParser.FindStringSubmatch(value) var indent, errParsingInt = strconv.ParseInt(matches[1], 10, 32) @@ -145,10 +156,30 @@ func extractNumberParamter(value string) (int, error) { return int(indent), nil } +func envSubstWithOptions() lex.Action { + return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { + value := string(m.Bytes) + noEmpty := hasOptionParameter(value, "ne") + noUnset := hasOptionParameter(value, "nu") + failFast := hasOptionParameter(value, "ff") + envsubstOpType.Type = "ENVSUBST" + prefs := envOpPreferences{NoUnset: noUnset, NoEmpty: noEmpty, FailFast: failFast} + if noEmpty { + envsubstOpType.Type = envsubstOpType.Type + "_NO_EMPTY" + } + if noUnset { + envsubstOpType.Type = envsubstOpType.Type + "_NO_UNSET" + } + + op := &Operation{OperationType: envsubstOpType, Value: envsubstOpType.Type, StringValue: value, Preferences: prefs} + return &token{TokenType: operationToken, Operation: op}, nil + } +} + func flattenWithDepth() lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { value := string(m.Bytes) - var depth, errParsingInt = extractNumberParamter(value) + var depth, errParsingInt = extractNumberParameter(value) if errParsingInt != nil { return nil, errParsingInt } @@ -162,7 +193,7 @@ func flattenWithDepth() lex.Action { func encodeWithIndent(outputFormat PrinterOutputFormat) lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { value := string(m.Bytes) - var indent, errParsingInt = extractNumberParamter(value) + var indent, errParsingInt = extractNumberParameter(value) if errParsingInt != nil { return nil, errParsingInt } @@ -507,6 +538,7 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`strenv\([^\)]+\)`), envOp(true)) lexer.Add([]byte(`env\([^\)]+\)`), envOp(false)) + lexer.Add([]byte(`envsubst\((ne|nu|ff| |,)+\)`), envSubstWithOptions()) lexer.Add([]byte(`envsubst`), opToken(envsubstOpType)) lexer.Add([]byte(`\[`), literalToken(openCollect, false)) diff --git a/pkg/yqlib/operator_env.go b/pkg/yqlib/operator_env.go index d4990c34..ed3f51e2 100644 --- a/pkg/yqlib/operator_env.go +++ b/pkg/yqlib/operator_env.go @@ -6,12 +6,15 @@ import ( "os" "strings" - envsubst "github.com/a8m/envsubst" + parse "github.com/a8m/envsubst/parse" yaml "gopkg.in/yaml.v3" ) type envOpPreferences struct { StringValue bool + NoUnset bool + NoEmpty bool + FailFast bool } func envOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { @@ -52,6 +55,19 @@ func envOperator(d *dataTreeNavigator, context Context, expressionNode *Expressi func envsubstOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { var results = list.New() + preferences := envOpPreferences{} + if expressionNode.Operation.Preferences != nil { + preferences = expressionNode.Operation.Preferences.(envOpPreferences) + } + + parser := parse.New("string", os.Environ(), + &parse.Restrictions{NoUnset: preferences.NoUnset, NoEmpty: preferences.NoEmpty}) + + if preferences.FailFast { + parser.Mode = parse.Quick + } else { + parser.Mode = parse.AllErrors + } for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) @@ -61,7 +77,7 @@ func envsubstOperator(d *dataTreeNavigator, context Context, expressionNode *Exp return Context{}, fmt.Errorf("cannot substitute with %v, can only substitute strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag) } - value, err := envsubst.String(node.Value) + value, err := parser.Parse(node.Value) if err != nil { return Context{}, err } diff --git a/pkg/yqlib/operator_env_test.go b/pkg/yqlib/operator_env_test.go index 000b6faf..dd6fb9dc 100644 --- a/pkg/yqlib/operator_env_test.go +++ b/pkg/yqlib/operator_env_test.go @@ -81,21 +81,55 @@ var envOperatorScenarios = []expressionScenario{ }, }, { - description: "Replace strings with envsubst, missing variables", - environmentVariables: map[string]string{"myenv": "cat"}, - expression: `"the ${myenvnonexisting} meows" | envsubst`, + description: "Replace strings with envsubst, missing variables", + expression: `"the ${myenvnonexisting} meows" | envsubst`, expected: []string{ "D0, P[], (!!str)::the meows\n", }, }, { - description: "Replace strings with envsubst, missing variables with defaults", - environmentVariables: map[string]string{"myenv": "cat"}, - expression: `"the ${myenvnonexisting-dog} meows" | envsubst`, + description: "Replace strings with envsubst(nu), missing variables", + subdescription: "(nu) not unset, will fail if there are unset (missing) variables", + expression: `"the ${myenvnonexisting} meows" | envsubst(nu)`, + expectedError: "variable ${myenvnonexisting} not set", + }, + { + description: "Replace strings with envsubst(ne), missing variables", + subdescription: "(ne) not empty, only validates set variables", + expression: `"the ${myenvnonexisting} meows" | envsubst(ne)`, + expected: []string{ + "D0, P[], (!!str)::the meows\n", + }, + }, + { + description: "Replace strings with envsubst(ne), empty variable", + subdescription: "(ne) not empty, will fail if a references variable is empty", + environmentVariables: map[string]string{"myenv": ""}, + expression: `"the ${myenv} meows" | envsubst(ne)`, + expectedError: "variable ${myenv} set but empty", + }, + { + description: "Replace strings with envsubst, missing variables with defaults", + expression: `"the ${myenvnonexisting-dog} meows" | envsubst`, expected: []string{ "D0, P[], (!!str)::the dog meows\n", }, }, + { + description: "Replace strings with envsubst(nu), missing variables with defaults", + subdescription: "Having a default specified skips over the missing variable.", + expression: `"the ${myenvnonexisting-dog} meows" | envsubst(nu)`, + expected: []string{ + "D0, P[], (!!str)::the dog meows\n", + }, + }, + { + description: "Replace strings with envsubst(ne), missing variables with defaults", + subdescription: "Fails, because the variable is explicitly set to blank.", + environmentVariables: map[string]string{"myEmptyEnv": ""}, + expression: `"the ${myEmptyEnv-dog} meows" | envsubst(ne)`, + expectedError: "variable ${myEmptyEnv} set but empty", + }, { description: "Replace string environment variable in document", environmentVariables: map[string]string{"myenv": "cat meow"}, @@ -105,6 +139,17 @@ var envOperatorScenarios = []expressionScenario{ "D0, P[], (doc)::{v: \"cat meow\"}\n", }, }, + { + description: "(Default) Return all envsubst errors", + subdescription: "By default, all errors are returned at once.", + expression: `"the ${notThere} ${alsoNotThere}" | envsubst(nu)`, + expectedError: "variable ${notThere} not set\nvariable ${alsoNotThere} not set", + }, + { + description: "Fail fast, return the first envsubst error (and abort)", + expression: `"the ${notThere} ${alsoNotThere}" | envsubst(nu,ff)`, + expectedError: "variable ${notThere} not set", + }, } func TestEnvOperatorScenarios(t *testing.T) { diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index 37e45c2f..92b2caeb 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -105,7 +105,7 @@ func testScenario(t *testing.T, s *expressionScenario) { } if err != nil { - t.Error(fmt.Errorf("%w: %v", err, s.expression)) + t.Error(fmt.Errorf("%w: %v: %v", err, s.description, s.expression)) return } test.AssertResultComplexWithContext(t, s.expected, resultsToString(t, context.MatchingNodes), fmt.Sprintf("desc: %v\nexp: %v\ndoc: %v", s.description, s.expression, s.document)) @@ -343,8 +343,13 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt } context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: inputs}, node) - if err != nil { + + if s.expectedError != "" && err != nil { + writeOrPanic(w, fmt.Sprintf("```bash\nError: %v\n```\n\n", err.Error())) + return + } else if err != nil { t.Error(err, s.expression) + return } err = printer.PrintResults(context.MatchingNodes)