From e515b8c2db6988c024942faaeb8d3368e0605452 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Mon, 2 Nov 2020 11:20:38 +1100 Subject: [PATCH] got style --- cmd/utils.go | 98 --------------- pkg/yqlib/treeops/lib.go | 20 +++- ...or_assign.go => operator_assign_update.go} | 2 +- ...test.go => operator_assign_update_test.go} | 0 pkg/yqlib/treeops/operator_explode.go | 112 ++++++++++++++++++ pkg/yqlib/treeops/operator_explode_test.go | 35 ++++++ pkg/yqlib/treeops/operatory_style.go | 77 ++++++++++++ pkg/yqlib/treeops/operatory_style_test.go | 45 +++++++ pkg/yqlib/treeops/path_parse_test.go | 20 ++++ pkg/yqlib/treeops/path_tokeniser.go | 7 +- 10 files changed, 310 insertions(+), 106 deletions(-) rename pkg/yqlib/treeops/{operator_assign.go => operator_assign_update.go} (90%) rename pkg/yqlib/treeops/{operator_assign_test.go => operator_assign_update_test.go} (100%) create mode 100644 pkg/yqlib/treeops/operator_explode.go create mode 100644 pkg/yqlib/treeops/operator_explode_test.go create mode 100644 pkg/yqlib/treeops/operatory_style.go create mode 100644 pkg/yqlib/treeops/operatory_style_test.go diff --git a/cmd/utils.go b/cmd/utils.go index 920adbad..2b24c231 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -121,104 +121,6 @@ func writeString(writer io.Writer, txt string) error { return errorWriting } -func setIfNotThere(node *yaml.Node, key string, value *yaml.Node) { - for index := 0; index < len(node.Content); index = index + 2 { - keyNode := node.Content[index] - if keyNode.Value == key { - return - } - } - // need to add it to the map - mapEntryKey := yaml.Node{Value: key, Kind: yaml.ScalarNode} - node.Content = append(node.Content, &mapEntryKey) - node.Content = append(node.Content, value) -} - -func applyAlias(node *yaml.Node, alias *yaml.Node) { - if alias == nil { - return - } - for index := 0; index < len(alias.Content); index = index + 2 { - keyNode := alias.Content[index] - log.Debugf("applying alias key %v", keyNode.Value) - valueNode := alias.Content[index+1] - setIfNotThere(node, keyNode.Value, valueNode) - } -} - -func explodeNode(node *yaml.Node) error { - node.Anchor = "" - switch node.Kind { - case yaml.SequenceNode, yaml.DocumentNode: - for index, contentNode := range node.Content { - log.Debugf("exploding index %v", index) - errorInContent := explodeNode(contentNode) - if errorInContent != nil { - return errorInContent - } - } - return nil - case yaml.AliasNode: - log.Debugf("its an alias!") - if node.Alias != nil { - node.Kind = node.Alias.Kind - node.Style = node.Alias.Style - node.Tag = node.Alias.Tag - node.Content = node.Alias.Content - node.Value = node.Alias.Value - node.Alias = nil - } - return nil - case yaml.MappingNode: - for index := 0; index < len(node.Content); index = index + 2 { - keyNode := node.Content[index] - valueNode := node.Content[index+1] - log.Debugf("traversing %v", keyNode.Value) - if keyNode.Value != "<<" { - errorInContent := explodeNode(valueNode) - if errorInContent != nil { - return errorInContent - } - errorInContent = explodeNode(keyNode) - if errorInContent != nil { - return errorInContent - } - } else { - if valueNode.Kind == yaml.SequenceNode { - log.Debugf("an alias merge list!") - for index := len(valueNode.Content) - 1; index >= 0; index = index - 1 { - aliasNode := valueNode.Content[index] - applyAlias(node, aliasNode.Alias) - } - } else { - log.Debugf("an alias merge!") - applyAlias(node, valueNode.Alias) - } - node.Content = append(node.Content[:index], node.Content[index+2:]...) - //replay that index, since the array is shorter now. - index = index - 2 - } - } - - return nil - default: - return nil - } -} - -func explode(matchingNodes *list.List) error { - log.Debug("exploding nodes") - for el := matchingNodes.Front(); el != nil; el = el.Next() { - nodeContext := el.Value.(*treeops.CandidateNode) - log.Debugf("exploding %v", nodeContext.GetKey()) - errorExplodingNode := explodeNode(nodeContext.Node) - if errorExplodingNode != nil { - return errorExplodingNode - } - } - return nil -} - func printResults(matchingNodes *list.List, writer io.Writer) error { if prettyPrint { setStyle(matchingNodes, 0) diff --git a/pkg/yqlib/treeops/lib.go b/pkg/yqlib/treeops/lib.go index 4f9e684f..51d3500b 100644 --- a/pkg/yqlib/treeops/lib.go +++ b/pkg/yqlib/treeops/lib.go @@ -19,23 +19,27 @@ type OperationType struct { } // operators TODO: -// - stripComments (recursive) +// - generator doc from operator tests +// - stripComments not recursive +// - documentIndex - retrieves document index, can be used with select // - mergeAppend (merges and appends arrays) -// - mergeIfEmpty (sets only if the document is empty, do I do that now?) -// - updateStyle -// - updateTag +// - mergeEmpty (sets only if the document is empty, do I do that now?) +// - updateStyle - not recursive +// - updateTag - not recursive // - explodeAnchors // - compare ?? // - validate ?? -// - exists ?? +// - exists var Or = &OperationType{Type: "OR", NumArgs: 2, Precedence: 20, Handler: OrOperator} var And = &OperationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: AndOperator} var Union = &OperationType{Type: "UNION", NumArgs: 2, Precedence: 10, Handler: UnionOperator} -var Assign = &OperationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: AssignOperator} +var Assign = &OperationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: AssignUpdateOperator} var AssignAttributes = &OperationType{Type: "ASSIGN_ATTRIBUTES", NumArgs: 2, Precedence: 40, Handler: AssignAttributesOperator} +var AssignStyle = &OperationType{Type: "ASSIGN_STYLE", NumArgs: 2, Precedence: 40, Handler: AssignStyleOperator} + var Multiply = &OperationType{Type: "MULTIPLY", NumArgs: 2, Precedence: 40, Handler: MultiplyOperator} var Equals = &OperationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: EqualsOperator} @@ -44,6 +48,10 @@ var Pipe = &OperationType{Type: "PIPE", NumArgs: 2, Precedence: 45, Handler: Pip var Length = &OperationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: LengthOperator} var Collect = &OperationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: CollectOperator} +var GetStyle = &OperationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: GetStyleOperator} + +var Explode = &OperationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: ExplodeOperator} + var CollectObject = &OperationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: CollectObjectOperator} var TraversePath = &OperationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator} diff --git a/pkg/yqlib/treeops/operator_assign.go b/pkg/yqlib/treeops/operator_assign_update.go similarity index 90% rename from pkg/yqlib/treeops/operator_assign.go rename to pkg/yqlib/treeops/operator_assign_update.go index 4e37c916..e124b7be 100644 --- a/pkg/yqlib/treeops/operator_assign.go +++ b/pkg/yqlib/treeops/operator_assign_update.go @@ -2,7 +2,7 @@ package treeops import "container/list" -func AssignOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) { +func AssignUpdateOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) { lhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Lhs) if err != nil { return nil, err diff --git a/pkg/yqlib/treeops/operator_assign_test.go b/pkg/yqlib/treeops/operator_assign_update_test.go similarity index 100% rename from pkg/yqlib/treeops/operator_assign_test.go rename to pkg/yqlib/treeops/operator_assign_update_test.go diff --git a/pkg/yqlib/treeops/operator_explode.go b/pkg/yqlib/treeops/operator_explode.go new file mode 100644 index 00000000..77ad39d6 --- /dev/null +++ b/pkg/yqlib/treeops/operator_explode.go @@ -0,0 +1,112 @@ +package treeops + +import ( + "container/list" + + "gopkg.in/yaml.v3" +) + +func ExplodeOperator(d *dataTreeNavigator, matchMap *list.List, pathNode *PathTreeNode) (*list.List, error) { + log.Debugf("-- ExplodeOperation") + + for el := matchMap.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + + rhs, err := d.GetMatchingNodes(nodeToMap(candidate), pathNode.Rhs) + + if err != nil { + return nil, err + } + for childEl := rhs.Front(); childEl != nil; childEl = childEl.Next() { + explodeNode(childEl.Value.(*CandidateNode).Node) + } + + } + + return matchMap, nil +} + +func explodeNode(node *yaml.Node) error { + node.Anchor = "" + switch node.Kind { + case yaml.SequenceNode, yaml.DocumentNode: + for index, contentNode := range node.Content { + log.Debugf("exploding index %v", index) + errorInContent := explodeNode(contentNode) + if errorInContent != nil { + return errorInContent + } + } + return nil + case yaml.AliasNode: + log.Debugf("its an alias!") + if node.Alias != nil { + node.Kind = node.Alias.Kind + node.Style = node.Alias.Style + node.Tag = node.Alias.Tag + node.Content = node.Alias.Content + node.Value = node.Alias.Value + node.Alias = nil + } + return nil + case yaml.MappingNode: + for index := 0; index < len(node.Content); index = index + 2 { + keyNode := node.Content[index] + valueNode := node.Content[index+1] + log.Debugf("traversing %v", keyNode.Value) + if keyNode.Value != "<<" { + errorInContent := explodeNode(valueNode) + if errorInContent != nil { + return errorInContent + } + errorInContent = explodeNode(keyNode) + if errorInContent != nil { + return errorInContent + } + } else { + if valueNode.Kind == yaml.SequenceNode { + log.Debugf("an alias merge list!") + for index := len(valueNode.Content) - 1; index >= 0; index = index - 1 { + aliasNode := valueNode.Content[index] + applyAlias(node, aliasNode.Alias) + } + } else { + log.Debugf("an alias merge!") + applyAlias(node, valueNode.Alias) + } + node.Content = append(node.Content[:index], node.Content[index+2:]...) + //replay that index, since the array is shorter now. + index = index - 2 + } + } + + return nil + default: + return nil + } +} + +func applyAlias(node *yaml.Node, alias *yaml.Node) { + if alias == nil { + return + } + for index := 0; index < len(alias.Content); index = index + 2 { + keyNode := alias.Content[index] + log.Debugf("applying alias key %v", keyNode.Value) + valueNode := alias.Content[index+1] + setIfNotThere(node, keyNode.Value, valueNode) + } +} + +func setIfNotThere(node *yaml.Node, key string, value *yaml.Node) { + for index := 0; index < len(node.Content); index = index + 2 { + keyNode := node.Content[index] + if keyNode.Value == key { + return + } + } + // need to add it to the map + mapEntryKey := yaml.Node{Value: key, Kind: yaml.ScalarNode} + node.Content = append(node.Content, &mapEntryKey) + node.Content = append(node.Content, value) +} diff --git a/pkg/yqlib/treeops/operator_explode_test.go b/pkg/yqlib/treeops/operator_explode_test.go new file mode 100644 index 00000000..ed283636 --- /dev/null +++ b/pkg/yqlib/treeops/operator_explode_test.go @@ -0,0 +1,35 @@ +package treeops + +import ( + "testing" +) + +var explodeTest = []expressionScenario{ + { + document: `{a: mike}`, + expression: `explode(.a)`, + expected: []string{ + "D0, P[], (doc)::{a: mike}\n", + }, + }, + { + document: `{f : {a: &a cat, b: *a}}`, + expression: `explode(.f)`, + expected: []string{ + "D0, P[], (doc)::{f: {a: cat, b: cat}}\n", + }, + }, + { + document: mergeDocSample, + expression: `.foo* | explode(.)`, + expected: []string{ + "D0, P[], (doc)::{f: {a: cat, b: cat}}\n", + }, + }, +} + +func TestExplodeOperatorScenarios(t *testing.T) { + for _, tt := range explodeTest { + testScenario(t, &tt) + } +} diff --git a/pkg/yqlib/treeops/operatory_style.go b/pkg/yqlib/treeops/operatory_style.go new file mode 100644 index 00000000..68209463 --- /dev/null +++ b/pkg/yqlib/treeops/operatory_style.go @@ -0,0 +1,77 @@ +package treeops + +import ( + "container/list" + "fmt" + + "gopkg.in/yaml.v3" +) + +func AssignStyleOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) { + customStyle := pathNode.Rhs.Operation.StringValue + log.Debugf("AssignStyleOperator: %v", customStyle) + + var style yaml.Style + if customStyle == "tagged" { + style = yaml.TaggedStyle + } else if customStyle == "double" { + style = yaml.DoubleQuotedStyle + } else if customStyle == "single" { + style = yaml.SingleQuotedStyle + } else if customStyle == "literal" { + style = yaml.LiteralStyle + } else if customStyle == "folded" { + style = yaml.FoldedStyle + } else if customStyle == "flow" { + style = yaml.FlowStyle + } else if customStyle != "" { + return nil, fmt.Errorf("Unknown style %v", customStyle) + } + lhs, err := d.GetMatchingNodes(matchingNodes, pathNode.Lhs) + + if err != nil { + return nil, err + } + + for el := lhs.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + log.Debugf("Setting style of : %v", candidate.GetKey()) + candidate.Node.Style = style + } + + return matchingNodes, nil +} + +func GetStyleOperator(d *dataTreeNavigator, matchingNodes *list.List, pathNode *PathTreeNode) (*list.List, error) { + log.Debugf("GetStyleOperator") + + var results = list.New() + + for el := matchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + var style = "" + switch candidate.Node.Style { + case yaml.TaggedStyle: + style = "tagged" + case yaml.DoubleQuotedStyle: + style = "double" + case yaml.SingleQuotedStyle: + style = "single" + case yaml.LiteralStyle: + style = "literal" + case yaml.FoldedStyle: + style = "folded" + case yaml.FlowStyle: + style = "flow" + case 0: + style = "" + default: + style = "" + } + node := &yaml.Node{Kind: yaml.ScalarNode, Value: style, Tag: "!!str"} + lengthCand := &CandidateNode{Node: node, Document: candidate.Document, Path: candidate.Path} + results.PushBack(lengthCand) + } + + return results, nil +} diff --git a/pkg/yqlib/treeops/operatory_style_test.go b/pkg/yqlib/treeops/operatory_style_test.go new file mode 100644 index 00000000..89e26498 --- /dev/null +++ b/pkg/yqlib/treeops/operatory_style_test.go @@ -0,0 +1,45 @@ +package treeops + +import ( + "testing" +) + +var styleOperatorScenarios = []expressionScenario{ + { + document: `{a: cat}`, + expression: `.a style="single"`, + expected: []string{ + "D0, P[], (doc)::{a: 'cat'}\n", + }, + }, + { + document: `{a: "cat", b: 'dog'}`, + expression: `.. style=""`, + expected: []string{ + "D0, P[], (!!map)::a: cat\nb: dog\n", + }, + }, + { + document: `{a: "cat", b: 'thing'}`, + expression: `.. | style`, + expected: []string{ + "D0, P[], (!!str)::flow\n", + "D0, P[a], (!!str)::double\n", + "D0, P[b], (!!str)::single\n", + }, + }, + { + document: `a: cat`, + expression: `.. | style`, + expected: []string{ + "D0, P[], (!!str)::\"\"\n", + "D0, P[a], (!!str)::\"\"\n", + }, + }, +} + +func TestStyleOperatorScenarios(t *testing.T) { + for _, tt := range styleOperatorScenarios { + testScenario(t, &tt) + } +} diff --git a/pkg/yqlib/treeops/path_parse_test.go b/pkg/yqlib/treeops/path_parse_test.go index ed12524b..3bd3ee68 100644 --- a/pkg/yqlib/treeops/path_parse_test.go +++ b/pkg/yqlib/treeops/path_parse_test.go @@ -89,6 +89,26 @@ var pathTests = []struct { append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "c", "UNION", "b", "PIPE", "[]", "CREATE_MAP", "f", "PIPE", "g", "PIPE", "[]", "}"), append(make([]interface{}, 0), "a", "c", "CREATE_MAP", "b", "[]", "PIPE", "f", "g", "PIPE", "[]", "PIPE", "CREATE_MAP", "UNION", "COLLECT_OBJECT", "PIPE"), }, + { + `explode(.a.b)`, + append(make([]interface{}, 0), "EXPLODE", "(", "a", "PIPE", "b", ")"), + append(make([]interface{}, 0), "a", "b", "PIPE", "EXPLODE"), + }, + { + `.a.b style="folded"`, + append(make([]interface{}, 0), "a", "PIPE", "b", "ASSIGN_STYLE", "folded (string)"), + append(make([]interface{}, 0), "a", "b", "PIPE", "folded (string)", "ASSIGN_STYLE"), + }, + // { + // `.a.b tag="!!str"`, + // append(make([]interface{}, 0), "EXPLODE", "(", "a", "PIPE", "b", ")"), + // append(make([]interface{}, 0), "a", "b", "PIPE", "EXPLODE"), + // }, + { + `""`, + append(make([]interface{}, 0), " (string)"), + append(make([]interface{}, 0), " (string)"), + }, // {".animals | .==cat", append(make([]interface{}, 0), "animals", "TRAVERSE", "SELF", "EQUALS", "cat")}, // {".animals | (. == cat)", append(make([]interface{}, 0), "animals", "TRAVERSE", "(", "SELF", "EQUALS", "cat", ")")}, diff --git a/pkg/yqlib/treeops/path_tokeniser.go b/pkg/yqlib/treeops/path_tokeniser.go index 59953f85..189b50f9 100644 --- a/pkg/yqlib/treeops/path_tokeniser.go +++ b/pkg/yqlib/treeops/path_tokeniser.go @@ -183,8 +183,13 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`:\s*`), opToken(CreateMap)) lexer.Add([]byte(`length`), opToken(Length)) lexer.Add([]byte(`select`), opToken(Select)) + lexer.Add([]byte(`explode`), opToken(Explode)) lexer.Add([]byte(`or`), opToken(Or)) lexer.Add([]byte(`not`), opToken(Not)) + + lexer.Add([]byte(`style\s*=`), opToken(AssignStyle)) + lexer.Add([]byte(`style`), opToken(GetStyle)) + // lexer.Add([]byte(`and`), opToken()) lexer.Add([]byte(`collect`), opToken(Collect)) @@ -217,7 +222,7 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`[Nn][Uu][Ll][Ll]`), nullValue()) lexer.Add([]byte(`~`), nullValue()) - lexer.Add([]byte(`"[^ "]+"`), stringValue(true)) + lexer.Add([]byte(`"[^ "]*"`), stringValue(true)) lexer.Add([]byte(`\[`), literalToken(OpenCollect, false)) lexer.Add([]byte(`\]`), literalToken(CloseCollect, true))