From 4e4c77a399037d265c66948d25a7ae7edce93551 Mon Sep 17 00:00:00 2001 From: Samuel Cormier-Iijima Date: Mon, 24 Jan 2022 20:51:38 -0500 Subject: [PATCH] Add a new `envsubst` operator to replace environment variables in strings --- go.mod | 1 + go.sum | 2 + pkg/yqlib/doc/operators/envsubst.md | 55 +++++++++++++++++++++ pkg/yqlib/doc/operators/headers/envsubst.md | 10 ++++ pkg/yqlib/expression_tokeniser.go | 2 + pkg/yqlib/lib.go | 2 + pkg/yqlib/operator_envsubst.go | 32 ++++++++++++ pkg/yqlib/operator_envsubst_test.go | 48 ++++++++++++++++++ 8 files changed, 152 insertions(+) create mode 100644 pkg/yqlib/doc/operators/envsubst.md create mode 100644 pkg/yqlib/doc/operators/headers/envsubst.md create mode 100644 pkg/yqlib/operator_envsubst.go create mode 100644 pkg/yqlib/operator_envsubst_test.go diff --git a/go.mod b/go.mod index 1334bfc3..eed65ad3 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,7 @@ module github.com/mikefarah/yq/v4 require ( + github.com/a8m/envsubst v1.2.0 github.com/elliotchance/orderedmap v1.4.0 github.com/fatih/color v1.13.0 github.com/goccy/go-yaml v1.9.5 diff --git a/go.sum b/go.sum index fb1d358a..8b2b9750 100644 --- a/go.sum +++ b/go.sum @@ -50,6 +50,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/a8m/envsubst v1.2.0 h1:yvzAhJD2QKdo35Ut03wIfXQmg+ta3wC/1bskfZynz+Q= +github.com/a8m/envsubst v1.2.0/go.mod h1:PpvLvNWa+Rvu/10qXmFbFiGICIU5hZvFJNPCCkUaObg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= diff --git a/pkg/yqlib/doc/operators/envsubst.md b/pkg/yqlib/doc/operators/envsubst.md new file mode 100644 index 00000000..73810fc8 --- /dev/null +++ b/pkg/yqlib/doc/operators/envsubst.md @@ -0,0 +1,55 @@ +# Envsubst + +This operator is used to replace environment variables in strings using [envsubst](https://github.com/a8m/envsubst). + +To replace environment variables across all values in a document, this can be used with the recursive descent operator +as follows: + +```bash +yq eval '(.. | select(tag == "!!str)) |= envsubst' file.yaml +``` + +## Replace strings with envsubst +Running +```bash +myenv="cat" yq eval --null-input '"the ${myenv} meows" | envsubst' +``` +will output +```yaml +the cat meows +``` + +## Replace strings with envsubst, missing variables +Running +```bash +myenv="cat" yq eval --null-input '"the ${myenvnonexisting} meows" | envsubst' +``` +will output +```yaml +the meows +``` + +## Replace strings with envsubst, missing variables with defaults +Running +```bash +myenv="cat" yq eval --null-input '"the ${myenvnonexisting-dog} meows" | envsubst' +``` +will output +```yaml +the dog meows +``` + +## Replace string environment variable in document +Given a sample.yml file of: +```yaml +v: ${myenv} +``` +then +```bash +myenv="cat meow" yq eval '.v |= envsubst' sample.yml +``` +will output +```yaml +v: cat meow +``` + diff --git a/pkg/yqlib/doc/operators/headers/envsubst.md b/pkg/yqlib/doc/operators/headers/envsubst.md new file mode 100644 index 00000000..ce03894c --- /dev/null +++ b/pkg/yqlib/doc/operators/headers/envsubst.md @@ -0,0 +1,10 @@ +# Envsubst + +This operator is used to replace environment variables in strings using [envsubst](https://github.com/a8m/envsubst). + +To replace environment variables across all values in a document, this can be used with the recursive descent operator +as follows: + +```bash +yq eval '(.. | select(tag == "!!str")) |= envsubst' file.yaml +``` diff --git a/pkg/yqlib/expression_tokeniser.go b/pkg/yqlib/expression_tokeniser.go index 4be52af1..24c7a4ca 100644 --- a/pkg/yqlib/expression_tokeniser.go +++ b/pkg/yqlib/expression_tokeniser.go @@ -472,6 +472,8 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`strenv\([^\)]+\)`), envOp(true)) lexer.Add([]byte(`env\([^\)]+\)`), envOp(false)) + lexer.Add([]byte(`envsubst`), opToken(envsubstOpType)) + lexer.Add([]byte(`\[`), literalToken(openCollect, false)) lexer.Add([]byte(`\]\??`), literalToken(closeCollect, true)) lexer.Add([]byte(`\{`), literalToken(openCollectObject, false)) diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 76c73e3a..d5fa63e1 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -131,6 +131,8 @@ var envOpType = &operationType{Type: "ENV", NumArgs: 0, Precedence: 50, Handler: var notOpType = &operationType{Type: "NOT", NumArgs: 0, Precedence: 50, Handler: notOperator} var emptyOpType = &operationType{Type: "EMPTY", Precedence: 50, Handler: emptyOperator} +var envsubstOpType = &operationType{Type: "ENVSUBST", NumArgs: 0, Precedence: 50, Handler: envsubstOperator} + var recursiveDescentOpType = &operationType{Type: "RECURSIVE_DESCENT", NumArgs: 0, Precedence: 50, Handler: recursiveDescentOperator} var selectOpType = &operationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Handler: selectOperator} diff --git a/pkg/yqlib/operator_envsubst.go b/pkg/yqlib/operator_envsubst.go new file mode 100644 index 00000000..83181648 --- /dev/null +++ b/pkg/yqlib/operator_envsubst.go @@ -0,0 +1,32 @@ +package yqlib + +import ( + "container/list" + "fmt" + + envsubst "github.com/a8m/envsubst" + yaml "gopkg.in/yaml.v3" +) + +func envsubstOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { + var results = list.New() + + for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + node := unwrapDoc(candidate.Node) + if node.Tag != "!!str" { + log.Warning("EnvSubstOperator, env name:", node.Tag, node.Value) + return Context{}, fmt.Errorf("cannot substitute with %v, can only substitute strings. Hint: Most often you'll want to use '|=' over '=' for this operation", node.Tag) + } + + value, err := envsubst.String(node.Value) + if err != nil { + return Context{}, err + } + targetNode := &yaml.Node{Kind: yaml.ScalarNode, Value: value, Tag: "!!str"} + result := candidate.CreateReplacement(targetNode) + results.PushBack(result) + } + + return context.ChildContext(results), nil +} diff --git a/pkg/yqlib/operator_envsubst_test.go b/pkg/yqlib/operator_envsubst_test.go new file mode 100644 index 00000000..1371ec03 --- /dev/null +++ b/pkg/yqlib/operator_envsubst_test.go @@ -0,0 +1,48 @@ +package yqlib + +import ( + "testing" +) + +var envsubstOperatorScenarios = []expressionScenario{ + { + description: "Replace strings with envsubst", + environmentVariable: "cat", + expression: `"the ${myenv} meows" | envsubst`, + expected: []string{ + "D0, P[], (!!str)::the cat meows\n", + }, + }, + { + description: "Replace strings with envsubst, missing variables", + environmentVariable: "cat", + expression: `"the ${myenvnonexisting} meows" | envsubst`, + expected: []string{ + "D0, P[], (!!str)::the meows\n", + }, + }, + { + description: "Replace strings with envsubst, missing variables with defaults", + environmentVariable: "cat", + expression: `"the ${myenvnonexisting-dog} meows" | envsubst`, + expected: []string{ + "D0, P[], (!!str)::the dog meows\n", + }, + }, + { + description: "Replace string environment variable in document", + environmentVariable: "cat meow", + document: "{v: \"${myenv}\"}", + expression: `.v |= envsubst`, + expected: []string{ + "D0, P[], (doc)::{v: \"cat meow\"}\n", + }, + }, +} + +func TestEnvSubstOperatorScenarios(t *testing.T) { + for _, tt := range envsubstOperatorScenarios { + testScenario(t, &tt) + } + documentOperatorScenarios(t, "envsubst", envsubstOperatorScenarios) +}