From 36084a60a949b9d89284d0d2e8e838213dbd9ba3 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Thu, 19 Nov 2020 16:45:05 +1100 Subject: [PATCH] Added tag operator --- pkg/yqlib/doc/Collect into Object.md | 5 +- pkg/yqlib/doc/Comments Operator.md | 4 +- pkg/yqlib/doc/Delete Operator.md | 8 ++- pkg/yqlib/doc/Document Index Operator.md | 1 - pkg/yqlib/doc/Explode Operator.md | 8 ++- pkg/yqlib/doc/Recursive Descent Operator.md | 16 +----- pkg/yqlib/doc/Style Operator.md | 49 +--------------- pkg/yqlib/doc/Tag Operator.md | 45 +++++++++++++++ pkg/yqlib/doc/Union Operator.md | 18 ------ pkg/yqlib/doc/headers/Tag Operator.md | 1 + pkg/yqlib/lib.go | 5 ++ pkg/yqlib/operator_style_test.go | 30 +++++++--- pkg/yqlib/operator_tag.go | 51 +++++++++++++++++ pkg/yqlib/operator_tag_test.go | 36 ++++++++++++ pkg/yqlib/path_parse_test.go | 21 +++++++ pkg/yqlib/path_tokeniser.go | 63 ++++++++++++++------- 16 files changed, 241 insertions(+), 120 deletions(-) create mode 100644 pkg/yqlib/doc/Tag Operator.md create mode 100644 pkg/yqlib/doc/headers/Tag Operator.md create mode 100644 pkg/yqlib/operator_tag.go create mode 100644 pkg/yqlib/operator_tag_test.go diff --git a/pkg/yqlib/doc/Collect into Object.md b/pkg/yqlib/doc/Collect into Object.md index a54a262d..c46f5d79 100644 --- a/pkg/yqlib/doc/Collect into Object.md +++ b/pkg/yqlib/doc/Collect into Object.md @@ -20,7 +20,8 @@ yq eval '{"wrap": .}' sample.yml ``` will output ```yaml -wrap: {name: Mike} +wrap: + name: Mike ``` ### Using splat to create multiple objects @@ -62,9 +63,7 @@ will output ```yaml Mike: cat Mike: dog ---- Rosey: monkey ---- Rosey: sheep ``` diff --git a/pkg/yqlib/doc/Comments Operator.md b/pkg/yqlib/doc/Comments Operator.md index 78b41b44..ab8b8864 100644 --- a/pkg/yqlib/doc/Comments Operator.md +++ b/pkg/yqlib/doc/Comments Operator.md @@ -101,7 +101,7 @@ yq eval '. | headComment' sample.yml ``` will output ```yaml -welcome! + ``` ### Get foot comment @@ -115,6 +115,6 @@ yq eval '. | footComment' sample.yml ``` will output ```yaml -have a great day + ``` diff --git a/pkg/yqlib/doc/Delete Operator.md b/pkg/yqlib/doc/Delete Operator.md index 6b42520c..dff1282f 100644 --- a/pkg/yqlib/doc/Delete Operator.md +++ b/pkg/yqlib/doc/Delete Operator.md @@ -12,7 +12,7 @@ yq eval 'del(.b)' sample.yml ``` will output ```yaml -{a: cat} +a: cat ``` ### Delete entry in array @@ -28,7 +28,8 @@ yq eval 'del(.[1])' sample.yml ``` will output ```yaml -[1, 3] +- 1 +- 3 ``` ### Delete no matches @@ -43,6 +44,7 @@ yq eval 'del(.c)' sample.yml ``` will output ```yaml -{a: cat, b: dog} +a: cat +b: dog ``` diff --git a/pkg/yqlib/doc/Document Index Operator.md b/pkg/yqlib/doc/Document Index Operator.md index 33126730..5dd62822 100644 --- a/pkg/yqlib/doc/Document Index Operator.md +++ b/pkg/yqlib/doc/Document Index Operator.md @@ -49,7 +49,6 @@ will output ```yaml match: cat doc: 0 ---- match: frog doc: 1 ``` diff --git a/pkg/yqlib/doc/Explode Operator.md b/pkg/yqlib/doc/Explode Operator.md index 80e70890..3ca0e208 100644 --- a/pkg/yqlib/doc/Explode Operator.md +++ b/pkg/yqlib/doc/Explode Operator.md @@ -13,7 +13,9 @@ yq eval 'explode(.f)' sample.yml ``` will output ```yaml -{f: {a: cat, b: cat}} +f: + a: cat + b: cat ``` ### Explode with no aliases or anchors @@ -43,7 +45,9 @@ yq eval 'explode(.f)' sample.yml ``` will output ```yaml -{f: {a: cat, cat: b}} +f: + a: cat + cat: b ``` ### Explode with merge anchors diff --git a/pkg/yqlib/doc/Recursive Descent Operator.md b/pkg/yqlib/doc/Recursive Descent Operator.md index 6d0069b0..9eec8436 100644 --- a/pkg/yqlib/doc/Recursive Descent Operator.md +++ b/pkg/yqlib/doc/Recursive Descent Operator.md @@ -1,23 +1,9 @@ This operator recursively matches all children nodes given of a particular element, including that node itself. This is most often used to apply a filter recursively against all matches, for instance to set the `style` of all nodes in a yaml doc: ```bash -yq eval '.. style = "flow"' file.yaml +yq eval '.. style= "flow"' file.yaml ``` ## Examples -### Matches single scalar value -Given a sample.yml file of: -```yaml -cat -``` -then -```bash -yq eval '..' sample.yml -``` -will output -```yaml -cat -``` - ### Map Given a sample.yml file of: ```yaml diff --git a/pkg/yqlib/doc/Style Operator.md b/pkg/yqlib/doc/Style Operator.md index f0080286..0c78cbe5 100644 --- a/pkg/yqlib/doc/Style Operator.md +++ b/pkg/yqlib/doc/Style Operator.md @@ -146,39 +146,7 @@ c: 3.2 e: true ``` -### Set style using a path -Given a sample.yml file of: -```yaml -a: cat -b: double -``` -then -```bash -yq eval '.a style=.b' sample.yml -``` -will output -```yaml -a: "cat" -b: double -``` - -### Example 8 -Given a sample.yml file of: -```yaml -a: cat -b: dog -``` -then -```bash -yq eval '.. style=""' sample.yml -``` -will output -```yaml -a: cat -b: dog -``` - -### Example 9 +### Read style Given a sample.yml file of: ```yaml a: cat @@ -193,20 +161,5 @@ will output -``` - -### Example 10 -Given a sample.yml file of: -```yaml -a: cat -``` -then -```bash -yq eval '.. | style' sample.yml -``` -will output -```yaml - - ``` diff --git a/pkg/yqlib/doc/Tag Operator.md b/pkg/yqlib/doc/Tag Operator.md new file mode 100644 index 00000000..e9e4de21 --- /dev/null +++ b/pkg/yqlib/doc/Tag Operator.md @@ -0,0 +1,45 @@ +The tag operator can be used to get or set the tag of ndoes (e.g. `!!str`, `!!int`, `!!bool`). +## Examples +### Get tag +Given a sample.yml file of: +```yaml +a: cat +b: 5 +c: 3.2 +e: true +f: [] +``` +then +```bash +yq eval '.. | tag' sample.yml +``` +will output +```yaml +!!map +!!str +!!int +!!float +!!bool +!!seq +``` + +### Convert numbers to strings +Given a sample.yml file of: +```yaml +a: cat +b: 5 +c: 3.2 +e: true +``` +then +```bash +yq eval '(.. | select(tag == "!!int")) tag = "!!str"' sample.yml +``` +will output +```yaml +a: cat +b: "5" +c: 3.2 +e: true +``` + diff --git a/pkg/yqlib/doc/Union Operator.md b/pkg/yqlib/doc/Union Operator.md index e1ce5fb2..e8975de3 100644 --- a/pkg/yqlib/doc/Union Operator.md +++ b/pkg/yqlib/doc/Union Operator.md @@ -29,21 +29,3 @@ fieldA fieldC ``` -### Combine selected paths -Given a sample.yml file of: -```yaml -a: fieldA -b: fieldB -c: fieldC -``` -then -```bash -yq eval '(.a, .c) |= "potatoe"' sample.yml -``` -will output -```yaml -a: potatoe -b: fieldB -c: potatoe -``` - diff --git a/pkg/yqlib/doc/headers/Tag Operator.md b/pkg/yqlib/doc/headers/Tag Operator.md new file mode 100644 index 00000000..1243de10 --- /dev/null +++ b/pkg/yqlib/doc/headers/Tag Operator.md @@ -0,0 +1 @@ +The tag operator can be used to get or set the tag of ndoes (e.g. `!!str`, `!!int`, `!!bool`). \ No newline at end of file diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 10e76917..d690e2ef 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -36,8 +36,12 @@ var And = &OperationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: AndOp var Union = &OperationType{Type: "UNION", NumArgs: 2, Precedence: 10, Handler: UnionOperator} var Assign = &OperationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: AssignUpdateOperator} + +// TODO: implement this +var PlainAssign = &OperationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: AssignUpdateOperator} var AssignAttributes = &OperationType{Type: "ASSIGN_ATTRIBUTES", NumArgs: 2, Precedence: 40, Handler: AssignAttributesOperator} var AssignStyle = &OperationType{Type: "ASSIGN_STYLE", NumArgs: 2, Precedence: 40, Handler: AssignStyleOperator} +var AssignTag = &OperationType{Type: "ASSIGN_TAG", NumArgs: 2, Precedence: 40, Handler: AssignTagOperator} var AssignComment = &OperationType{Type: "ASSIGN_COMMENT", NumArgs: 2, Precedence: 40, Handler: AssignCommentsOperator} var Multiply = &OperationType{Type: "MULTIPLY", NumArgs: 2, Precedence: 40, Handler: MultiplyOperator} @@ -49,6 +53,7 @@ var Pipe = &OperationType{Type: "PIPE", NumArgs: 2, Precedence: 45, Handler: Pip var Length = &OperationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: LengthOperator} var Collect = &OperationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: CollectOperator} var GetStyle = &OperationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: GetStyleOperator} +var GetTag = &OperationType{Type: "GET_TAG", NumArgs: 0, Precedence: 50, Handler: GetTagOperator} var GetComment = &OperationType{Type: "GET_COMMENT", NumArgs: 0, Precedence: 50, Handler: GetCommentsOperator} var GetDocumentIndex = &OperationType{Type: "GET_DOCUMENT_INDEX", NumArgs: 0, Precedence: 50, Handler: GetDocumentIndexOperator} diff --git a/pkg/yqlib/operator_style_test.go b/pkg/yqlib/operator_style_test.go index 1bdaf983..ff84df12 100644 --- a/pkg/yqlib/operator_style_test.go +++ b/pkg/yqlib/operator_style_test.go @@ -10,7 +10,7 @@ var styleOperatorScenarios = []expressionScenario{ document: `{a: cat, b: 5, c: 3.2, e: true}`, expression: `.. style="tagged"`, expected: []string{ - "D0, P[], (doc)::{a: 'cat'}\n", + "D0, P[], (!!map)::!!map\na: !!str cat\nb: !!int 5\nc: !!float 3.2\ne: !!bool true\n", }, }, { @@ -18,7 +18,7 @@ var styleOperatorScenarios = []expressionScenario{ document: `{a: cat, b: 5, c: 3.2, e: true}`, expression: `.. style="double"`, expected: []string{ - "D0, P[], (doc)::{a: 'cat'}\n", + "D0, P[], (!!map)::a: \"cat\"\nb: \"5\"\nc: \"3.2\"\ne: \"true\"\n", }, }, { @@ -26,7 +26,7 @@ var styleOperatorScenarios = []expressionScenario{ document: `{a: cat, b: 5, c: 3.2, e: true}`, expression: `.. style="single"`, expected: []string{ - "D0, P[], (doc)::{a: 'cat'}\n", + "D0, P[], (!!map)::a: 'cat'\nb: '5'\nc: '3.2'\ne: 'true'\n", }, }, { @@ -34,7 +34,15 @@ var styleOperatorScenarios = []expressionScenario{ document: `{a: cat, b: 5, c: 3.2, e: true}`, expression: `.. style="literal"`, expected: []string{ - "D0, P[], (doc)::{a: 'cat'}\n", + `D0, P[], (!!map)::a: |- + cat +b: |- + 5 +c: |- + 3.2 +e: |- + true +`, }, }, { @@ -42,7 +50,15 @@ var styleOperatorScenarios = []expressionScenario{ document: `{a: cat, b: 5, c: 3.2, e: true}`, expression: `.. style="folded"`, expected: []string{ - "D0, P[], (doc)::{a: 'cat'}\n", + `D0, P[], (!!map)::a: >- + cat +b: >- + 5 +c: >- + 3.2 +e: >- + true +`, }, }, { @@ -50,7 +66,7 @@ var styleOperatorScenarios = []expressionScenario{ document: `{a: cat, b: 5, c: 3.2, e: true}`, expression: `.. style="flow"`, expected: []string{ - "D0, P[], (doc)::{a: 'cat'}\n", + "D0, P[], (!!map)::{a: cat, b: 5, c: 3.2, e: true}\n", }, }, { @@ -58,7 +74,7 @@ var styleOperatorScenarios = []expressionScenario{ document: `{a: cat, b: 5, c: 3.2, e: true}`, expression: `.. style=""`, expected: []string{ - "D0, P[], (doc)::{a: 'cat'}\n", + "D0, P[], (!!map)::a: cat\nb: 5\nc: 3.2\ne: true\n", }, }, { diff --git a/pkg/yqlib/operator_tag.go b/pkg/yqlib/operator_tag.go new file mode 100644 index 00000000..8b9c515b --- /dev/null +++ b/pkg/yqlib/operator_tag.go @@ -0,0 +1,51 @@ +package yqlib + +import ( + "container/list" + + "gopkg.in/yaml.v3" +) + +func AssignTagOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) { + + log.Debugf("AssignTagOperator: %v") + + rhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Rhs) + if err != nil { + return nil, err + } + tag := "" + + if rhs.Front() != nil { + tag = rhs.Front().Value.(*CandidateNode).Node.Value + } + + lhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Lhs) + + if err != nil { + return nil, err + } + + for el := lhs.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + log.Debugf("Setting tag of : %v", candidate.GetKey()) + candidate.Node.Tag = tag + } + + return matchingNodes, nil +} + +func GetTagOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) { + log.Debugf("GetTagOperator") + + var results = list.New() + + for el := matchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + node := &yaml.Node{Kind: yaml.ScalarNode, Value: candidate.Node.Tag, Tag: "!!str"} + lengthCand := &CandidateNode{Node: node, Document: candidate.Document, Path: candidate.Path} + results.PushBack(lengthCand) + } + + return results, nil +} diff --git a/pkg/yqlib/operator_tag_test.go b/pkg/yqlib/operator_tag_test.go new file mode 100644 index 00000000..9d4878dc --- /dev/null +++ b/pkg/yqlib/operator_tag_test.go @@ -0,0 +1,36 @@ +package yqlib + +import ( + "testing" +) + +var tagOperatorScenarios = []expressionScenario{ + { + description: "Get tag", + document: `{a: cat, b: 5, c: 3.2, e: true, f: []}`, + expression: `.. | tag`, + expected: []string{ + "D0, P[], (!!str)::'!!map'\n", + "D0, P[a], (!!str)::'!!str'\n", + "D0, P[b], (!!str)::'!!int'\n", + "D0, P[c], (!!str)::'!!float'\n", + "D0, P[e], (!!str)::'!!bool'\n", + "D0, P[f], (!!str)::'!!seq'\n", + }, + }, + { + description: "Convert numbers to strings", + document: `{a: cat, b: 5, c: 3.2, e: true}`, + expression: `(.. | select(tag == "!!int")) tag= "!!str"`, + expected: []string{ + "D0, P[], (!!map)::{a: cat, b: \"5\", c: 3.2, e: true}\n", + }, + }, +} + +func TestTagOperatorScenarios(t *testing.T) { + for _, tt := range tagOperatorScenarios { + testScenario(t, &tt) + } + documentScenarios(t, "Tag Operator", tagOperatorScenarios) +} diff --git a/pkg/yqlib/path_parse_test.go b/pkg/yqlib/path_parse_test.go index 45037f35..b78a4bd7 100644 --- a/pkg/yqlib/path_parse_test.go +++ b/pkg/yqlib/path_parse_test.go @@ -99,6 +99,27 @@ var pathTests = []struct { append(make([]interface{}, 0), "a", "PIPE", "b", "ASSIGN_STYLE", "folded (string)"), append(make([]interface{}, 0), "a", "b", "PIPE", "folded (string)", "ASSIGN_STYLE"), }, + { + `tag == "str"`, + append(make([]interface{}, 0), "GET_TAG", "EQUALS", "str (string)"), + append(make([]interface{}, 0), "GET_TAG", "str (string)", "EQUALS"), + }, + { + `. tag= "str"`, + append(make([]interface{}, 0), "SELF", "ASSIGN_TAG", "str (string)"), + append(make([]interface{}, 0), "SELF", "str (string)", "ASSIGN_TAG"), + }, + { + `lineComment == "str"`, + append(make([]interface{}, 0), "GET_COMMENT", "EQUALS", "str (string)"), + append(make([]interface{}, 0), "GET_COMMENT", "str (string)", "EQUALS"), + }, + { + `. lineComment= "str"`, + append(make([]interface{}, 0), "SELF", "ASSIGN_COMMENT", "str (string)"), + append(make([]interface{}, 0), "SELF", "str (string)", "ASSIGN_COMMENT"), + }, + // { // `.a.b tag="!!str"`, // append(make([]interface{}, 0), "EXPLODE", "(", "a", "PIPE", "b", ")"), diff --git a/pkg/yqlib/path_tokeniser.go b/pkg/yqlib/path_tokeniser.go index aba11192..71614c0e 100644 --- a/pkg/yqlib/path_tokeniser.go +++ b/pkg/yqlib/path_tokeniser.go @@ -24,10 +24,11 @@ const ( ) type Token struct { - TokenType TokenType - Operation *Operation + TokenType TokenType + Operation *Operation + AssignOperation *Operation // e.g. tag (GetTag) op becomes AssignTag if '=' follows it + CheckForPostTraverse bool // e.g. [1]cat should really be [1].cat - CheckForPostTraverse bool // e.g. [1]cat should really be [1].cat } func (t *Token) toString() string { @@ -83,14 +84,22 @@ func documentToken() lex.Action { } func opToken(op *OperationType) lex.Action { - return opTokenWithPrefs(op, nil) + return opTokenWithPrefs(op, nil, nil) } -func opTokenWithPrefs(op *OperationType, preferences interface{}) lex.Action { +func opAssignableToken(opType *OperationType, assignOpType *OperationType) lex.Action { + return opTokenWithPrefs(opType, assignOpType, nil) +} + +func opTokenWithPrefs(op *OperationType, assignOpType *OperationType, preferences interface{}) lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { value := string(m.Bytes) op := &Operation{OperationType: op, Value: op.Type, StringValue: value, Preferences: preferences} - return &Token{TokenType: OperationToken, Operation: op}, nil + var assign *Operation + if assignOpType != nil { + assign = &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, Preferences: preferences} + } + return &Token{TokenType: OperationToken, Operation: op, AssignOperation: assign}, nil } } @@ -191,23 +200,22 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`documentIndex`), opToken(GetDocumentIndex)) - lexer.Add([]byte(`style\s*=`), opToken(AssignStyle)) - lexer.Add([]byte(`style`), opToken(GetStyle)) + lexer.Add([]byte(`style`), opAssignableToken(GetStyle, AssignStyle)) - lexer.Add([]byte(`lineComment\s*=`), opTokenWithPrefs(AssignComment, &CommentOpPreferences{LineComment: true})) - lexer.Add([]byte(`lineComment`), opTokenWithPrefs(GetComment, &CommentOpPreferences{LineComment: true})) + lexer.Add([]byte(`tag`), opAssignableToken(GetTag, AssignTag)) - lexer.Add([]byte(`headComment\s*=`), opTokenWithPrefs(AssignComment, &CommentOpPreferences{HeadComment: true})) - lexer.Add([]byte(`headComment`), opTokenWithPrefs(GetComment, &CommentOpPreferences{HeadComment: true})) + lexer.Add([]byte(`lineComment`), opTokenWithPrefs(GetComment, AssignComment, &CommentOpPreferences{LineComment: true})) - lexer.Add([]byte(`footComment\s*=`), opTokenWithPrefs(AssignComment, &CommentOpPreferences{FootComment: true})) - lexer.Add([]byte(`footComment`), opTokenWithPrefs(GetComment, &CommentOpPreferences{FootComment: true})) + lexer.Add([]byte(`headComment`), opTokenWithPrefs(GetComment, AssignComment, &CommentOpPreferences{HeadComment: true})) - lexer.Add([]byte(`comments\s*=`), opTokenWithPrefs(AssignComment, &CommentOpPreferences{LineComment: true, HeadComment: true, FootComment: true})) + lexer.Add([]byte(`footComment`), opTokenWithPrefs(GetComment, AssignComment, &CommentOpPreferences{FootComment: true})) + + lexer.Add([]byte(`comments\s*=`), opTokenWithPrefs(AssignComment, nil, &CommentOpPreferences{LineComment: true, HeadComment: true, FootComment: true})) lexer.Add([]byte(`collect`), opToken(Collect)) lexer.Add([]byte(`\s*==\s*`), opToken(Equals)) + lexer.Add([]byte(`\s*=\s*`), opToken(PlainAssign)) lexer.Add([]byte(`del`), opToken(DeleteChild)) @@ -286,15 +294,28 @@ func (p *pathTokeniser) Tokenise(path string) ([]*Token, error) { } var postProcessedTokens = make([]*Token, 0) + skipNextToken := false + for index, token := range tokens { + if skipNextToken { + skipNextToken = false + } else { - postProcessedTokens = append(postProcessedTokens, token) + if index != len(tokens)-1 && token.AssignOperation != nil && + tokens[index+1].TokenType == OperationToken && + tokens[index+1].Operation.OperationType == PlainAssign { + token.Operation = token.AssignOperation + skipNextToken = true + } - if index != len(tokens)-1 && token.CheckForPostTraverse && - tokens[index+1].TokenType == OperationToken && - tokens[index+1].Operation.OperationType == TraversePath { - op := &Operation{OperationType: Pipe, Value: "PIPE"} - postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op}) + postProcessedTokens = append(postProcessedTokens, token) + + if index != len(tokens)-1 && token.CheckForPostTraverse && + tokens[index+1].TokenType == OperationToken && + tokens[index+1].Operation.OperationType == TraversePath { + op := &Operation{OperationType: Pipe, Value: "PIPE"} + postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op}) + } } }