From b2ee131a4cc9a6839056d8fe1ef45f172ff74d60 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sun, 12 Sep 2021 16:55:55 +1000 Subject: [PATCH] Updated var to work like jq #934 --- pkg/yqlib/context.go | 22 ++++++++++++ pkg/yqlib/doc/Style.md | 4 +-- pkg/yqlib/doc/Variable Operators.md | 38 ++++++++++++++++++++- pkg/yqlib/doc/headers/Variable Operators.md | 4 ++- pkg/yqlib/expression_tokeniser.go | 3 +- pkg/yqlib/operator_collect_object.go | 3 +- pkg/yqlib/operator_style_test.go | 4 +-- pkg/yqlib/operator_variables.go | 15 +++++++- pkg/yqlib/operator_variables_test.go | 22 ++++++++++-- 9 files changed, 102 insertions(+), 13 deletions(-) diff --git a/pkg/yqlib/context.go b/pkg/yqlib/context.go index 67c9bb6d..fe1f372a 100644 --- a/pkg/yqlib/context.go +++ b/pkg/yqlib/context.go @@ -61,9 +61,31 @@ func (n *Context) ToString() string { return result + NodesToString(n.MatchingNodes) } +func (n *Context) DeepClone() Context { + clone := Context{} + err := copier.Copy(&clone, n) + // copier doesn't do lists properly for some reason + clone.MatchingNodes = list.New() + for el := n.MatchingNodes.Front(); el != nil; el = el.Next() { + clonedNode, err := el.Value.(*CandidateNode).Copy() + if err != nil { + log.Error("Error cloning context :(") + panic(err) + } + clone.MatchingNodes.PushBack(clonedNode) + } + + if err != nil { + log.Error("Error cloning context :(") + panic(err) + } + return clone +} + func (n *Context) Clone() Context { clone := Context{} err := copier.Copy(&clone, n) + if err != nil { log.Error("Error cloning context :(") panic(err) diff --git a/pkg/yqlib/doc/Style.md b/pkg/yqlib/doc/Style.md index a8a220c1..d7c07673 100644 --- a/pkg/yqlib/doc/Style.md +++ b/pkg/yqlib/doc/Style.md @@ -18,7 +18,7 @@ a: ``` ## Update and set style of a particular node using path variables -You can use a variable to re-use a path +You can use a variable reference to re-use a path Given a sample.yml file of: ```yaml @@ -28,7 +28,7 @@ a: ``` then ```bash -yq eval '.a.b as $x | $x = "new" | $x style="double"' sample.yml +yq eval '.a.b ref $x | $x = "new" | $x style="double"' sample.yml ``` will output ```yaml diff --git a/pkg/yqlib/doc/Variable Operators.md b/pkg/yqlib/doc/Variable Operators.md index 25c706a9..c434a0c5 100644 --- a/pkg/yqlib/doc/Variable Operators.md +++ b/pkg/yqlib/doc/Variable Operators.md @@ -1,4 +1,6 @@ -For more complex scenarios, variables can be used to hold values of expression to be used in other expressions. +Like the `jq` equivalents, variables are sometimes required for the more complex expressions (or swapping values between fields). + +Note that there is also an additional `ref` operator that holds a reference (instead of a copy) of the path, allowing you to make multiple changes to the same path. ## Single value variable Given a sample.yml file of: @@ -56,3 +58,37 @@ title: A well-written article author: Person McPherson ``` +## Using variables to swap values +Given a sample.yml file of: +```yaml +a: a_value +b: b_value +``` +then +```bash +yq eval '.a as $x | .b as $y | .b = $x | .a = $y' sample.yml +``` +will output +```yaml +a: b_value +b: a_value +``` + +## Use ref to reference a path repeatedly +Given a sample.yml file of: +```yaml +a: + b: thing + c: something +``` +then +```bash +yq eval '.a.b ref $x | $x = "new" | $x style="double"' sample.yml +``` +will output +```yaml +a: + b: "new" + c: something +``` + diff --git a/pkg/yqlib/doc/headers/Variable Operators.md b/pkg/yqlib/doc/headers/Variable Operators.md index 887b7f7e..a76ccff4 100644 --- a/pkg/yqlib/doc/headers/Variable Operators.md +++ b/pkg/yqlib/doc/headers/Variable Operators.md @@ -1 +1,3 @@ -For more complex scenarios, variables can be used to hold values of expression to be used in other expressions. +Like the `jq` equivalents, variables are sometimes required for the more complex expressions (or swapping values between fields). + +Note that there is also an additional `ref` operator that holds a reference (instead of a copy) of the path, allowing you to make multiple changes to the same path. diff --git a/pkg/yqlib/expression_tokeniser.go b/pkg/yqlib/expression_tokeniser.go index f3e81be7..09929136 100644 --- a/pkg/yqlib/expression_tokeniser.go +++ b/pkg/yqlib/expression_tokeniser.go @@ -367,7 +367,8 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`\-`), opToken(subtractOpType)) lexer.Add([]byte(`\-=`), opToken(subtractAssignOpType)) lexer.Add([]byte(`\$[a-zA-Z_-0-9]+`), getVariableOpToken()) - lexer.Add([]byte(`as`), opToken(assignVariableOpType)) + lexer.Add([]byte(`as`), opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{})) + lexer.Add([]byte(`ref`), opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{IsReference: true})) err := lexer.CompileNFA() if err != nil { diff --git a/pkg/yqlib/operator_collect_object.go b/pkg/yqlib/operator_collect_object.go index 75c47827..446c066c 100644 --- a/pkg/yqlib/operator_collect_object.go +++ b/pkg/yqlib/operator_collect_object.go @@ -20,8 +20,7 @@ import ( func collectObjectOperator(d *dataTreeNavigator, originalContext Context, expressionNode *ExpressionNode) (Context, error) { log.Debugf("-- collectObjectOperation") - context := originalContext.Clone() - context.DontAutoCreate = false + context := originalContext.WritableClone() if context.MatchingNodes.Len() == 0 { node := &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map", Value: "{}"} diff --git a/pkg/yqlib/operator_style_test.go b/pkg/yqlib/operator_style_test.go index 988b8382..4288240f 100644 --- a/pkg/yqlib/operator_style_test.go +++ b/pkg/yqlib/operator_style_test.go @@ -15,9 +15,9 @@ var styleOperatorScenarios = []expressionScenario{ }, { description: "Update and set style of a particular node using path variables", - subdescription: "You can use a variable to re-use a path", + subdescription: "You can use a variable reference to re-use a path", document: `a: {b: thing, c: something}`, - expression: `.a.b as $x | $x = "new" | $x style="double"`, + expression: `.a.b ref $x | $x = "new" | $x style="double"`, expected: []string{ "D0, P[], (doc)::a: {b: \"new\", c: something}\n", }, diff --git a/pkg/yqlib/operator_variables.go b/pkg/yqlib/operator_variables.go index 52081324..eda93a2f 100644 --- a/pkg/yqlib/operator_variables.go +++ b/pkg/yqlib/operator_variables.go @@ -15,6 +15,10 @@ func getVariableOperator(d *dataTreeNavigator, context Context, expressionNode * return context.ChildContext(result), nil } +type assignVarPreferences struct { + IsReference bool +} + func assignVariableOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { lhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Lhs) if err != nil { @@ -24,6 +28,15 @@ func assignVariableOperator(d *dataTreeNavigator, context Context, expressionNod return Context{}, fmt.Errorf("RHS of 'as' operator must be a variable name e.g. $foo") } variableName := expressionNode.Rhs.Operation.StringValue - context.SetVariable(variableName, lhs.MatchingNodes) + + prefs := expressionNode.Operation.Preferences.(assignVarPreferences) + + var variableValue *list.List + if prefs.IsReference { + variableValue = lhs.MatchingNodes + } else { + variableValue = lhs.DeepClone().MatchingNodes + } + context.SetVariable(variableName, variableValue) return context, nil } diff --git a/pkg/yqlib/operator_variables_test.go b/pkg/yqlib/operator_variables_test.go index 68ccac1f..69502347 100644 --- a/pkg/yqlib/operator_variables_test.go +++ b/pkg/yqlib/operator_variables_test.go @@ -34,15 +34,31 @@ var variableOperatorScenarios = []expressionScenario{ description: "Using variables as a lookup", subdescription: "Example taken from [jq](https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|...)", document: `{"posts": [{"title": "Frist psot", "author": "anon"}, - {"title": "A well-written article", "author": "person1"}], -"realnames": {"anon": "Anonymous Coward", - "person1": "Person McPherson"}}`, + {"title": "A well-written article", "author": "person1"}], + "realnames": {"anon": "Anonymous Coward", + "person1": "Person McPherson"}}`, expression: `.realnames as $names | .posts[] | {"title":.title, "author": $names[.author]}`, expected: []string{ "D0, P[], (!!map)::title: \"Frist psot\"\nauthor: \"Anonymous Coward\"\n", "D0, P[], (!!map)::title: \"A well-written article\"\nauthor: \"Person McPherson\"\n", }, }, + { + description: "Using variables to swap values", + document: "a: a_value\nb: b_value", + expression: `.a as $x | .b as $y | .b = $x | .a = $y`, + expected: []string{ + "D0, P[], (doc)::a: b_value\nb: a_value\n", + }, + }, + { + description: "Use ref to reference a path repeatedly", + document: `a: {b: thing, c: something}`, + expression: `.a.b ref $x | $x = "new" | $x style="double"`, + expected: []string{ + "D0, P[], (doc)::a: {b: \"new\", c: something}\n", + }, + }, } func TestVariableOperatorScenarios(t *testing.T) {