From 1d5ecb244d6da0ad9aae02fab88542ed660f4662 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 8 Jan 2021 21:07:46 +1100 Subject: [PATCH 1/4] wip --- pkg/yqlib/operator_env.go | 32 +++++++++++ pkg/yqlib/operator_env_test.go | 100 +++++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+) create mode 100644 pkg/yqlib/operator_env.go create mode 100644 pkg/yqlib/operator_env_test.go diff --git a/pkg/yqlib/operator_env.go b/pkg/yqlib/operator_env.go new file mode 100644 index 00000000..597a62be --- /dev/null +++ b/pkg/yqlib/operator_env.go @@ -0,0 +1,32 @@ +package yqlib + +import ( + "container/list" + "os" + + yaml "gopkg.in/yaml.v3" +) + +// type EnvOpPreferences struct { +// StringValue bool +// } + +func EnvOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) { + envName := pathNode.Operation.CandidateNode.Node.Value + log.Debug("EnvOperator, env name:", envName) + + rawValue := os.Getenv(envName) + + target := &CandidateNode{ + Path: make([]interface{}, 0), + Document: 0, + Filename: "", + Node: &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: rawValue, + }, + } + + return nodeToMap(target), nil +} diff --git a/pkg/yqlib/operator_env_test.go b/pkg/yqlib/operator_env_test.go new file mode 100644 index 00000000..33c28127 --- /dev/null +++ b/pkg/yqlib/operator_env_test.go @@ -0,0 +1,100 @@ +package yqlib + +import ( + "testing" +) + +var envOperatorScenarios = []expressionScenario{ + { + document: ``, + expression: `1`, + expected: []string{ + "D0, P[], (!!int)::1\n", + }, + }, + { + document: ``, + expression: `-1`, + expected: []string{ + "D0, P[], (!!int)::-1\n", + }, + }, { + document: ``, + expression: `1.2`, + expected: []string{ + "D0, P[], (!!float)::1.2\n", + }, + }, { + document: ``, + expression: `-5.2e11`, + expected: []string{ + "D0, P[], (!!float)::-5.2e11\n", + }, + }, { + document: ``, + expression: `5e-10`, + expected: []string{ + "D0, P[], (!!float)::5e-10\n", + }, + }, + { + document: ``, + expression: `"cat"`, + expected: []string{ + "D0, P[], (!!str)::cat\n", + }, + }, + { + document: ``, + expression: `"frog jumps"`, + expected: []string{ + "D0, P[], (!!str)::frog jumps\n", + }, + }, + { + document: ``, + expression: `"1.3"`, + expected: []string{ + "D0, P[], (!!str)::\"1.3\"\n", + }, + }, { + document: ``, + expression: `"true"`, + expected: []string{ + "D0, P[], (!!str)::\"true\"\n", + }, + }, { + document: ``, + expression: `true`, + expected: []string{ + "D0, P[], (!!bool)::true\n", + }, + }, { + document: ``, + expression: `false`, + expected: []string{ + "D0, P[], (!!bool)::false\n", + }, + }, + { + document: ``, + expression: `Null`, + expected: []string{ + "D0, P[], (!!null)::Null\n", + }, + }, + { + document: ``, + expression: `~`, + expected: []string{ + "D0, P[], (!!null)::~\n", + }, + }, +} + +func TestEnvOperatorScenarios(t *testing.T) { + for _, tt := range envOperatorScenarios { + testScenario(t, &tt) + } + documentScenarios(t, "Env Variable Operators", addOperatorScenarios) +} From c12764dba8fa46f7847262aa9ed0e4184511868c Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 8 Jan 2021 21:09:43 +1100 Subject: [PATCH 2/4] wip --- pkg/yqlib/operator_env_test.go | 83 +++------------------------------- 1 file changed, 7 insertions(+), 76 deletions(-) diff --git a/pkg/yqlib/operator_env_test.go b/pkg/yqlib/operator_env_test.go index 33c28127..d70cef32 100644 --- a/pkg/yqlib/operator_env_test.go +++ b/pkg/yqlib/operator_env_test.go @@ -6,88 +6,19 @@ import ( var envOperatorScenarios = []expressionScenario{ { - document: ``, - expression: `1`, - expected: []string{ - "D0, P[], (!!int)::1\n", - }, - }, - { - document: ``, - expression: `-1`, - expected: []string{ - "D0, P[], (!!int)::-1\n", - }, - }, { - document: ``, - expression: `1.2`, - expected: []string{ - "D0, P[], (!!float)::1.2\n", - }, - }, { - document: ``, - expression: `-5.2e11`, - expected: []string{ - "D0, P[], (!!float)::-5.2e11\n", - }, - }, { - document: ``, - expression: `5e-10`, - expected: []string{ - "D0, P[], (!!float)::5e-10\n", - }, - }, - { - document: ``, - expression: `"cat"`, - expected: []string{ - "D0, P[], (!!str)::cat\n", - }, - }, - { - document: ``, - expression: `"frog jumps"`, - expected: []string{ - "D0, P[], (!!str)::frog jumps\n", - }, - }, - { - document: ``, - expression: `"1.3"`, - expected: []string{ - "D0, P[], (!!str)::\"1.3\"\n", - }, - }, { - document: ``, - expression: `"true"`, + description: "Read boolean environment variable as a string", + environmentVariable: "true", + expression: `strenv(myenv)`, expected: []string{ "D0, P[], (!!str)::\"true\"\n", }, - }, { - document: ``, - expression: `true`, - expected: []string{ - "D0, P[], (!!bool)::true\n", - }, - }, { - document: ``, - expression: `false`, - expected: []string{ - "D0, P[], (!!bool)::false\n", - }, }, { - document: ``, - expression: `Null`, + description: "Read numeric environment variable as a string", + environmentVariable: "12", + expression: `strenv(myenv)`, expected: []string{ - "D0, P[], (!!null)::Null\n", - }, - }, - { - document: ``, - expression: `~`, - expected: []string{ - "D0, P[], (!!null)::~\n", + "D0, P[], (!!str)::\"12\"\n", }, }, } From aabed1a237309968d996bfb80901218194e466cd Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sat, 9 Jan 2021 11:33:39 +1100 Subject: [PATCH 3/4] strenv --- pkg/yqlib/doc/Env Variable Operators.md | 21 +++++++++++++++++++++ pkg/yqlib/lib.go | 1 + pkg/yqlib/operator_env_test.go | 2 +- pkg/yqlib/operators_test.go | 20 +++++++++++++++++--- pkg/yqlib/path_tokeniser.go | 20 ++++++++++++++++++++ 5 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 pkg/yqlib/doc/Env Variable Operators.md diff --git a/pkg/yqlib/doc/Env Variable Operators.md b/pkg/yqlib/doc/Env Variable Operators.md new file mode 100644 index 00000000..6f69515a --- /dev/null +++ b/pkg/yqlib/doc/Env Variable Operators.md @@ -0,0 +1,21 @@ + +## Read boolean environment variable as a string +Running +```bash +myenv="true" yq eval --null-input 'strenv(myenv)' +``` +will output +```yaml +12 +``` + +## Read numeric environment variable as a string +Running +```bash +myenv="12" yq eval --null-input 'strenv(myenv)' +``` +will output +```yaml +12 +``` + diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index fbe69dd0..db8402f3 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -70,6 +70,7 @@ var TraverseArray = &OperationType{Type: "TRAVERSE_ARRAY", NumArgs: 1, Precedenc var DocumentFilter = &OperationType{Type: "DOCUMENT_FILTER", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator} var SelfReference = &OperationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: SelfOperator} var ValueOp = &OperationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: ValueOperator} +var EnvOp = &OperationType{Type: "ENV", NumArgs: 0, Precedence: 50, Handler: EnvOperator} var Not = &OperationType{Type: "NOT", NumArgs: 0, Precedence: 50, Handler: NotOperator} var Empty = &OperationType{Type: "EMPTY", NumArgs: 50, Handler: EmptyOperator} diff --git a/pkg/yqlib/operator_env_test.go b/pkg/yqlib/operator_env_test.go index d70cef32..0569e38e 100644 --- a/pkg/yqlib/operator_env_test.go +++ b/pkg/yqlib/operator_env_test.go @@ -27,5 +27,5 @@ func TestEnvOperatorScenarios(t *testing.T) { for _, tt := range envOperatorScenarios { testScenario(t, &tt) } - documentScenarios(t, "Env Variable Operators", addOperatorScenarios) + documentScenarios(t, "Env Variable Operators", envOperatorScenarios) } diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index c8881bb5..5064410f 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -17,6 +17,7 @@ import ( type expressionScenario struct { description string subdescription string + environmentVariable string document string document2 string expression string @@ -61,6 +62,10 @@ func testScenario(t *testing.T, s *expressionScenario) { } + if s.environmentVariable != "" { + os.Setenv("myenv", s.environmentVariable) + } + results, err = treeNavigator.GetMatchingNodes(inputs, node) if err != nil { @@ -162,6 +167,13 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) { formattedDoc := "" formattedDoc2 := "" command := "eval" + + envCommand := "" + + if(s.environmentVariable != "") { + envCommand = fmt.Sprintf("myenv=\"%v\" ", s.environmentVariable) + } + if s.document != "" { if s.dontFormatInputForDoc { formattedDoc = s.document + "\n" @@ -188,14 +200,16 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) { } writeOrPanic(w, "then\n") + + if s.expression != "" { - writeOrPanic(w, fmt.Sprintf("```bash\nyq %v '%v' %v\n```\n", command, s.expression, files)) + writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v '%v' %v\n```\n", envCommand, command, s.expression, files)) } else { - writeOrPanic(w, fmt.Sprintf("```bash\nyq %v %v\n```\n", command, files)) + writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v %v\n```\n", envCommand, command, files)) } } else { writeOrPanic(w, "Running\n") - writeOrPanic(w, fmt.Sprintf("```bash\nyq %v --null-input '%v'\n```\n", command, s.expression)) + writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v --null-input '%v'\n```\n", envCommand, command, s.expression)) } return formattedDoc, formattedDoc2 } diff --git a/pkg/yqlib/path_tokeniser.go b/pkg/yqlib/path_tokeniser.go index 6afac412..20f77fdd 100644 --- a/pkg/yqlib/path_tokeniser.go +++ b/pkg/yqlib/path_tokeniser.go @@ -178,6 +178,25 @@ func stringValue(wrapped bool) lex.Action { } } +func envOp(strenv bool) lex.Action { + return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { + value := string(m.Bytes) + + if strenv { + // strenv( ) + value = value[7:len(value)-1] + } else { + //env( ) + value = value[4:len(value)-1] + } + + envOperation := CreateValueOperation(value, value) + envOperation.OperationType = EnvOp + + return &Token{TokenType: OperationToken, Operation: envOperation}, nil + } +} + func nullValue() lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { return &Token{TokenType: OperationToken, Operation: CreateValueOperation(nil, string(m.Bytes))}, nil @@ -266,6 +285,7 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`~`), nullValue()) lexer.Add([]byte(`"[^"]*"`), stringValue(true)) + lexer.Add([]byte(`strenv\([^\)]+\)`), envOp(true)) lexer.Add([]byte(`\[`), literalToken(OpenCollect, false)) lexer.Add([]byte(`\]`), literalToken(CloseCollect, true)) From 644063646e2e4384fd830d47e2af30121d37ffbd Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sat, 9 Jan 2021 12:06:19 +1100 Subject: [PATCH 4/4] Env Ops! --- pkg/yqlib/doc/Env Variable Operators.md | 63 +++++++++++++++++-- pkg/yqlib/doc/Traverse.md | 18 ++++++ .../doc/headers/Env Variable Operators.md | 0 pkg/yqlib/operator_env.go | 36 ++++++++--- pkg/yqlib/operator_env_test.go | 49 +++++++++++++-- pkg/yqlib/operator_traverse_path_test.go | 9 +++ pkg/yqlib/operators_test.go | 12 ++-- pkg/yqlib/path_tokeniser.go | 12 ++-- 8 files changed, 173 insertions(+), 26 deletions(-) create mode 100644 pkg/yqlib/doc/headers/Env Variable Operators.md diff --git a/pkg/yqlib/doc/Env Variable Operators.md b/pkg/yqlib/doc/Env Variable Operators.md index 6f69515a..c166f3ef 100644 --- a/pkg/yqlib/doc/Env Variable Operators.md +++ b/pkg/yqlib/doc/Env Variable Operators.md @@ -1,21 +1,76 @@ +## Read string environment variable +Running +```bash +myenv="cat meow" yq eval --null-input '.a = env(myenv)' +``` +will output +```yaml +a: cat meow +``` + +## Read boolean environment variable +Running +```bash +myenv="true" yq eval --null-input '.a = env(myenv)' +``` +will output +```yaml +a: true +``` + +## Read numeric environment variable +Running +```bash +myenv="12" yq eval --null-input '.a = env(myenv)' +``` +will output +```yaml +a: 12 +``` + +## Read yaml environment variable +Running +```bash +myenv="{b: fish}" yq eval --null-input '.a = env(myenv)' +``` +will output +```yaml +a: {b: fish} +``` + ## Read boolean environment variable as a string Running ```bash -myenv="true" yq eval --null-input 'strenv(myenv)' +myenv="true" yq eval --null-input '.a = strenv(myenv)' ``` will output ```yaml -12 +a: "true" ``` ## Read numeric environment variable as a string Running ```bash -myenv="12" yq eval --null-input 'strenv(myenv)' +myenv="12" yq eval --null-input '.a = strenv(myenv)' ``` will output ```yaml -12 +a: "12" +``` + +## Dynamic key lookup with environment variable +Given a sample.yml file of: +```yaml +cat: meow +dog: woof +``` +then +```bash +myenv="cat" yq eval '.[env(myenv)]' sample.yml +``` +will output +```yaml +meow ``` diff --git a/pkg/yqlib/doc/Traverse.md b/pkg/yqlib/doc/Traverse.md index b08ae705..4442dc1e 100644 --- a/pkg/yqlib/doc/Traverse.md +++ b/pkg/yqlib/doc/Traverse.md @@ -48,6 +48,24 @@ will output frog ``` +## Dynamic keys +Expressions within [] can be used to dynamically lookup / calculate keys + +Given a sample.yml file of: +```yaml +b: apple +apple: crispy yum +banana: soft yum +``` +then +```bash +yq eval '.[.b]' sample.yml +``` +will output +```yaml +crispy yum +``` + ## Children don't exist Nodes are added dynamically while traversing diff --git a/pkg/yqlib/doc/headers/Env Variable Operators.md b/pkg/yqlib/doc/headers/Env Variable Operators.md new file mode 100644 index 00000000..e69de29b diff --git a/pkg/yqlib/operator_env.go b/pkg/yqlib/operator_env.go index 597a62be..4cfeeb3c 100644 --- a/pkg/yqlib/operator_env.go +++ b/pkg/yqlib/operator_env.go @@ -3,13 +3,14 @@ package yqlib import ( "container/list" "os" + "strings" yaml "gopkg.in/yaml.v3" ) -// type EnvOpPreferences struct { -// StringValue bool -// } +type EnvOpPreferences struct { + StringValue bool +} func EnvOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) { envName := pathNode.Operation.CandidateNode.Node.Value @@ -17,15 +18,34 @@ func EnvOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNo rawValue := os.Getenv(envName) + preferences := pathNode.Operation.Preferences.(*EnvOpPreferences) + + var node *yaml.Node + if preferences.StringValue { + node = &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: rawValue, + } + } else { + var dataBucket yaml.Node + decoder := yaml.NewDecoder(strings.NewReader(rawValue)) + errorReading := decoder.Decode(&dataBucket) + if errorReading != nil { + return nil, errorReading + } + //first node is a doc + node = UnwrapDoc(&dataBucket) + } + log.Debug("ENV tag", node.Tag) + log.Debug("ENV value", node.Value) + log.Debug("ENV Kind", node.Kind) + target := &CandidateNode{ Path: make([]interface{}, 0), Document: 0, Filename: "", - Node: &yaml.Node{ - Kind: yaml.ScalarNode, - Tag: "!!str", - Value: rawValue, - }, + Node: node, } return nodeToMap(target), nil diff --git a/pkg/yqlib/operator_env_test.go b/pkg/yqlib/operator_env_test.go index 0569e38e..8066e356 100644 --- a/pkg/yqlib/operator_env_test.go +++ b/pkg/yqlib/operator_env_test.go @@ -5,20 +5,61 @@ import ( ) var envOperatorScenarios = []expressionScenario{ + { + description: "Read string environment variable", + environmentVariable: "cat meow", + expression: `.a = env(myenv)`, + expected: []string{ + "D0, P[], ()::a: cat meow\n", + }, + }, + { + description: "Read boolean environment variable", + environmentVariable: "true", + expression: `.a = env(myenv)`, + expected: []string{ + "D0, P[], ()::a: true\n", + }, + }, + { + description: "Read numeric environment variable", + environmentVariable: "12", + expression: `.a = env(myenv)`, + expected: []string{ + "D0, P[], ()::a: 12\n", + }, + }, + { + description: "Read yaml environment variable", + environmentVariable: "{b: fish}", + expression: `.a = env(myenv)`, + expected: []string{ + "D0, P[], ()::a: {b: fish}\n", + }, + }, { description: "Read boolean environment variable as a string", environmentVariable: "true", - expression: `strenv(myenv)`, + expression: `.a = strenv(myenv)`, expected: []string{ - "D0, P[], (!!str)::\"true\"\n", + "D0, P[], ()::a: \"true\"\n", }, }, { description: "Read numeric environment variable as a string", environmentVariable: "12", - expression: `strenv(myenv)`, + expression: `.a = strenv(myenv)`, expected: []string{ - "D0, P[], (!!str)::\"12\"\n", + "D0, P[], ()::a: \"12\"\n", + }, + }, + { + description: "Dynamic key lookup with environment variable", + environmentVariable: "cat", + document: `{cat: meow, dog: woof}`, + expression: `.[env(myenv)]`, + expected: []string{ + "D0, P[cat], (!!str)::meow\n", }, }, } diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go index 9d4a44a3..ffd2327d 100644 --- a/pkg/yqlib/operator_traverse_path_test.go +++ b/pkg/yqlib/operator_traverse_path_test.go @@ -54,6 +54,15 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[{}], (!!str)::frog\n", }, }, + { + description: "Dynamic keys", + subdescription: `Expressions within [] can be used to dynamically lookup / calculate keys`, + document: `{b: apple, apple: crispy yum, banana: soft yum}`, + expression: `.[.b]`, + expected: []string{ + "D0, P[apple], (!!str)::crispy yum\n", + }, + }, { description: "Children don't exist", subdescription: "Nodes are added dynamically while traversing", diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index 5064410f..5d6dc1ea 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -17,7 +17,7 @@ import ( type expressionScenario struct { description string subdescription string - environmentVariable string + environmentVariable string document string document2 string expression string @@ -170,10 +170,11 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) { envCommand := "" - if(s.environmentVariable != "") { - envCommand = fmt.Sprintf("myenv=\"%v\" ", s.environmentVariable) - } - + if s.environmentVariable != "" { + envCommand = fmt.Sprintf("myenv=\"%v\" ", s.environmentVariable) + os.Setenv("myenv", s.environmentVariable) + } + if s.document != "" { if s.dontFormatInputForDoc { formattedDoc = s.document + "\n" @@ -200,7 +201,6 @@ func documentInput(w *bufio.Writer, s expressionScenario) (string, string) { } writeOrPanic(w, "then\n") - if s.expression != "" { writeOrPanic(w, fmt.Sprintf("```bash\n%vyq %v '%v' %v\n```\n", envCommand, command, s.expression, files)) diff --git a/pkg/yqlib/path_tokeniser.go b/pkg/yqlib/path_tokeniser.go index 20f77fdd..a25d78c7 100644 --- a/pkg/yqlib/path_tokeniser.go +++ b/pkg/yqlib/path_tokeniser.go @@ -181,17 +181,20 @@ func stringValue(wrapped bool) lex.Action { func envOp(strenv bool) lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { value := string(m.Bytes) + preferences := &EnvOpPreferences{} if strenv { // strenv( ) - value = value[7:len(value)-1] - } else { + value = value[7 : len(value)-1] + preferences.StringValue = true + } else { //env( ) - value = value[4:len(value)-1] - } + value = value[4 : len(value)-1] + } envOperation := CreateValueOperation(value, value) envOperation.OperationType = EnvOp + envOperation.Preferences = preferences return &Token{TokenType: OperationToken, Operation: envOperation}, nil } @@ -286,6 +289,7 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`"[^"]*"`), stringValue(true)) lexer.Add([]byte(`strenv\([^\)]+\)`), envOp(true)) + lexer.Add([]byte(`env\([^\)]+\)`), envOp(false)) lexer.Add([]byte(`\[`), literalToken(OpenCollect, false)) lexer.Add([]byte(`\]`), literalToken(CloseCollect, true))