From 60511f5f92a67040a42165a8cd7c090cde315168 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Sat, 17 Oct 2020 22:10:47 +1100 Subject: [PATCH] refactoring, fixing --- pkg/yqlib/treeops/candidate_node.go | 2 +- pkg/yqlib/treeops/data_tree_navigator.go | 8 + pkg/yqlib/treeops/data_tree_navigator_test.go | 248 +----------------- pkg/yqlib/treeops/leaf_traverser.go | 2 +- pkg/yqlib/treeops/lib.go | 24 +- pkg/yqlib/treeops/operator_booleans.go | 72 +++++ pkg/yqlib/treeops/operator_booleans_test.go | 35 +++ pkg/yqlib/treeops/operator_collect.go | 32 +++ pkg/yqlib/treeops/operator_collect_test.go | 45 ++++ ...{delete_operator.go => operator_delete.go} | 0 ...{equals_operator.go => operator_equals.go} | 12 +- pkg/yqlib/treeops/operator_equals_test.go | 43 +++ pkg/yqlib/treeops/operator_select.go | 37 +++ pkg/yqlib/treeops/operator_select_test.go | 45 ++++ pkg/yqlib/treeops/operator_union.go | 19 ++ pkg/yqlib/treeops/operator_union_test.go | 31 +++ pkg/yqlib/treeops/operators.go | 61 +---- pkg/yqlib/treeops/operators_test.go | 28 ++ ...h_tokeniser_test.go => path_parse_test.go} | 44 +++- pkg/yqlib/treeops/path_postfix.go | 13 +- pkg/yqlib/treeops/path_postfix_test.go | 3 +- pkg/yqlib/treeops/path_tokeniser.go | 7 +- pkg/yqlib/treeops/path_tree.go | 2 +- 23 files changed, 484 insertions(+), 329 deletions(-) create mode 100644 pkg/yqlib/treeops/operator_booleans.go create mode 100644 pkg/yqlib/treeops/operator_booleans_test.go create mode 100644 pkg/yqlib/treeops/operator_collect.go create mode 100644 pkg/yqlib/treeops/operator_collect_test.go rename pkg/yqlib/treeops/{delete_operator.go => operator_delete.go} (100%) rename pkg/yqlib/treeops/{equals_operator.go => operator_equals.go} (82%) create mode 100644 pkg/yqlib/treeops/operator_equals_test.go create mode 100644 pkg/yqlib/treeops/operator_select.go create mode 100644 pkg/yqlib/treeops/operator_select_test.go create mode 100644 pkg/yqlib/treeops/operator_union.go create mode 100644 pkg/yqlib/treeops/operator_union_test.go create mode 100644 pkg/yqlib/treeops/operators_test.go rename pkg/yqlib/treeops/{path_tokeniser_test.go => path_parse_test.go} (81%) diff --git a/pkg/yqlib/treeops/candidate_node.go b/pkg/yqlib/treeops/candidate_node.go index aa46a495..961e9e29 100644 --- a/pkg/yqlib/treeops/candidate_node.go +++ b/pkg/yqlib/treeops/candidate_node.go @@ -15,7 +15,7 @@ type CandidateNode struct { } func (n *CandidateNode) GetKey() string { - return fmt.Sprintf("%v - %v", n.Document, n.Path) + return fmt.Sprintf("%v - %v - %v", n.Document, n.Path, n.Node.Value) } // updates this candidate from the given candidate node diff --git a/pkg/yqlib/treeops/data_tree_navigator.go b/pkg/yqlib/treeops/data_tree_navigator.go index 08f0edeb..4a798ead 100644 --- a/pkg/yqlib/treeops/data_tree_navigator.go +++ b/pkg/yqlib/treeops/data_tree_navigator.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/elliotchance/orderedmap" + "gopkg.in/op/go-logging.v1" ) type dataTreeNavigator struct { @@ -68,6 +69,13 @@ func (d *dataTreeNavigator) getMatchingNodes(matchingNodes *orderedmap.OrderedMa return matchingNodes, nil } log.Debugf("Processing Path: %v", pathNode.PathElement.toString()) + if log.IsEnabledFor(logging.DEBUG) { + for el := matchingNodes.Front(); el != nil; el = el.Next() { + log.Debug(NodeToString(el.Value.(*CandidateNode))) + } + } + log.Debug(">>") + if pathNode.PathElement.PathElementType == SelfReference { return matchingNodes, nil } else if pathNode.PathElement.PathElementType == PathKey { diff --git a/pkg/yqlib/treeops/data_tree_navigator_test.go b/pkg/yqlib/treeops/data_tree_navigator_test.go index eaa7eee8..91117aa7 100644 --- a/pkg/yqlib/treeops/data_tree_navigator_test.go +++ b/pkg/yqlib/treeops/data_tree_navigator_test.go @@ -12,6 +12,9 @@ var treeNavigator = NewDataTreeNavigator(NavigationPrefs{}) var treeCreator = NewPathTreeCreator() func readDoc(t *testing.T, content string) []*CandidateNode { + if content == "" { + return []*CandidateNode{} + } decoder := yaml.NewDecoder(strings.NewReader(content)) var dataBucket yaml.Node err := decoder.Decode(&dataBucket) @@ -21,10 +24,10 @@ func readDoc(t *testing.T, content string) []*CandidateNode { return []*CandidateNode{&CandidateNode{Node: dataBucket.Content[0], Document: 0}} } -func resultsToString(results []*CandidateNode) string { - var pretty string = "" +func resultsToString(results []*CandidateNode) []string { + var pretty []string = make([]string, 0) for _, n := range results { - pretty = pretty + "\n" + NodeToString(n) + pretty = append(pretty, NodeToString(n)) } return pretty } @@ -1052,242 +1055,3 @@ func TestDataTreeNavigatorAnd(t *testing.T) { test.AssertResult(t, expected, resultsToString(results)) } - -func TestDataTreeNavigatorEqualsSimple(t *testing.T) { - - nodes := readDoc(t, `a: - cat: {b: apple, c: yes} - pat: {b: banana} -`) - - path, errPath := treeCreator.ParsePath(".a | (.[].b == \"apple\")") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [a cat] - Tag: !!map, Kind: MappingNode, Anchor: - {b: apple, c: yes} -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorEqualsSelf(t *testing.T) { - - nodes := readDoc(t, `a: frog -b: cat -c: frog`) - - path, errPath := treeCreator.ParsePath("(a or b).(. == frog)") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [a] - Tag: !!str, Kind: ScalarNode, Anchor: - frog -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorEqualsNested(t *testing.T) { - - nodes := readDoc(t, `a: {t: frog} -b: {t: cat} -c: {t: frog}`) - - path, errPath := treeCreator.ParsePath("(t == frog)") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [a] - Tag: !!map, Kind: MappingNode, Anchor: - {t: frog} - --- Node -- - Document 0, path: [c] - Tag: !!map, Kind: MappingNode, Anchor: - {t: frog} -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorArrayEqualsSelf(t *testing.T) { - - nodes := readDoc(t, `- cat -- dog -- frog`) - - path, errPath := treeCreator.ParsePath("(. == *og)") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [1] - Tag: !!str, Kind: ScalarNode, Anchor: - dog - --- Node -- - Document 0, path: [2] - Tag: !!str, Kind: ScalarNode, Anchor: - frog -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorArrayEqualsSelfSplatFirst(t *testing.T) { - - nodes := readDoc(t, `- cat -- dog -- frog`) - - path, errPath := treeCreator.ParsePath("*(. == *og)") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [1] - Tag: !!str, Kind: ScalarNode, Anchor: - dog - --- Node -- - Document 0, path: [2] - Tag: !!str, Kind: ScalarNode, Anchor: - frog -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorArrayEquals(t *testing.T) { - - nodes := readDoc(t, `- { b: apple, animal: rabbit } -- { b: banana, animal: cat } -- { b: corn, animal: dog }`) - - path, errPath := treeCreator.ParsePath("(b == apple or animal == dog)") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [0] - Tag: !!map, Kind: MappingNode, Anchor: - {b: apple, animal: rabbit} - --- Node -- - Document 0, path: [2] - Tag: !!map, Kind: MappingNode, Anchor: - {b: corn, animal: dog} -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorArrayEqualsDeep(t *testing.T) { - - nodes := readDoc(t, `apples: - - { b: apple, animal: {legs: 2} } - - { b: banana, animal: {legs: 4} } - - { b: corn, animal: {legs: 6} } -`) - - path, errPath := treeCreator.ParsePath("apples(animal.legs == 4)") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [apples 1] - Tag: !!map, Kind: MappingNode, Anchor: - {b: banana, animal: {legs: 4}} -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func xTestDataTreeNavigatorEqualsTrickey(t *testing.T) { - - nodes := readDoc(t, `a: - cat: {b: apso, c: {d : yes}} - pat: {b: apple, c: {d : no}} - sat: {b: apsy, c: {d : yes}} - fat: {b: apple} -`) - - path, errPath := treeCreator.ParsePath(".a(.b == ap* and .c.d == yes)") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [a cat] - Tag: !!map, Kind: MappingNode, Anchor: - {b: apso, c: {d: yes}} - --- Node -- - Document 0, path: [a sat] - Tag: !!map, Kind: MappingNode, Anchor: - {b: apsy, c: {d: yes}} -` - - test.AssertResult(t, expected, resultsToString(results)) -} diff --git a/pkg/yqlib/treeops/leaf_traverser.go b/pkg/yqlib/treeops/leaf_traverser.go index 679a08b2..3c19dc76 100644 --- a/pkg/yqlib/treeops/leaf_traverser.go +++ b/pkg/yqlib/treeops/leaf_traverser.go @@ -19,7 +19,7 @@ func NewLeafTraverser(navigationPrefs NavigationPrefs) LeafTraverser { } func (t *traverser) keyMatches(key *yaml.Node, pathNode *PathElement) bool { - return Match(key.Value, pathNode.StringValue) + return pathNode.Value == "[]" || Match(key.Value, pathNode.StringValue) } func (t *traverser) traverseMap(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) { diff --git a/pkg/yqlib/treeops/lib.go b/pkg/yqlib/treeops/lib.go index 46d9ba2d..01d62a86 100644 --- a/pkg/yqlib/treeops/lib.go +++ b/pkg/yqlib/treeops/lib.go @@ -33,19 +33,24 @@ type OperationType struct { var None = &OperationType{Type: "NONE", NumArgs: 0, Precedence: 0} -var Or = &OperationType{Type: "OR", NumArgs: 2, Precedence: 10, Handler: UnionOperator} -var And = &OperationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: IntersectionOperator} +var Or = &OperationType{Type: "OR", NumArgs: 2, Precedence: 20, Handler: OrOperator} +var And = &OperationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: AndOperator} + +var Union = &OperationType{Type: "UNION", NumArgs: 2, Precedence: 10, Handler: UnionOperator} +var Intersection = &OperationType{Type: "INTERSECTION", NumArgs: 2, Precedence: 20, Handler: IntersectionOperator} var Assign = &OperationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: AssignOperator} var Equals = &OperationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: EqualsOperator} var Pipe = &OperationType{Type: "PIPE", NumArgs: 2, Precedence: 45, Handler: PipeOperator} var Length = &OperationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: LengthOperator} +var Collect = &OperationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: CollectOperator} // not sure yet +var Select = &OperationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Handler: SelectOperator} + var DeleteChild = &OperationType{Type: "DELETE", NumArgs: 2, Precedence: 40, Handler: DeleteChildOperator} -var Collect = &OperationType{Type: "COLLECT", NumArgs: 1, Precedence: 40, Handler: CollectOperator} // var Splat = &OperationType{Type: "SPLAT", NumArgs: 0, Precedence: 40, Handler: SplatOperator} @@ -64,13 +69,13 @@ func (p *PathElement) toString() string { var result string = `` switch p.PathElementType { case PathKey: - result = result + fmt.Sprintf("PathKey - %v", p.Value) + result = result + fmt.Sprintf("%v", p.Value) case SelfReference: result = result + fmt.Sprintf("SELF") case Operation: - result = result + fmt.Sprintf("Operation - %v", p.OperationType.Type) + result = result + fmt.Sprintf("%v", p.OperationType.Type) case Value: - result = result + fmt.Sprintf("Value - %v (%T)", p.Value, p.Value) + result = result + fmt.Sprintf("%v (%T)", p.Value, p.Value) default: result = result + "I HAVENT GOT A STRATEGY" } @@ -124,7 +129,7 @@ func NodeToString(node *CandidateNode) string { } value := node.Node if value == nil { - return "-- node is nil --" + return "-- nil --" } buf := new(bytes.Buffer) encoder := yaml.NewEncoder(buf) @@ -133,10 +138,7 @@ func NodeToString(node *CandidateNode) string { log.Error("Error debugging node, %v", errorEncoding.Error()) } encoder.Close() - return fmt.Sprintf(`-- Node -- - Document %v, path: %v - Tag: %v, Kind: %v, Anchor: %v - %v`, node.Document, node.Path, value.Tag, KindString(value.Kind), value.Anchor, buf.String()) + return fmt.Sprintf(`D%v, P%v, (%v)::%v`, node.Document, node.Path, value.Tag, buf.String()) } func KindString(kind yaml.Kind) string { diff --git a/pkg/yqlib/treeops/operator_booleans.go b/pkg/yqlib/treeops/operator_booleans.go new file mode 100644 index 00000000..45acd163 --- /dev/null +++ b/pkg/yqlib/treeops/operator_booleans.go @@ -0,0 +1,72 @@ +package treeops + +import ( + "github.com/elliotchance/orderedmap" + "gopkg.in/yaml.v3" +) + +func isTruthy(c *CandidateNode) (bool, error) { + node := c.Node + value := true + if node.Kind == yaml.ScalarNode && node.Tag == "!!bool" { + errDecoding := node.Decode(&value) + if errDecoding != nil { + return false, errDecoding + } + + } + return value, nil +} + +type boolOp func(bool, bool) bool + +func booleanOp(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode, op boolOp) (*orderedmap.OrderedMap, error) { + var results = orderedmap.NewOrderedMap() + + for el := matchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + lhs, err := d.getMatchingNodes(nodeToMap(candidate), pathNode.Lhs) + if err != nil { + return nil, err + } + rhs, err := d.getMatchingNodes(nodeToMap(candidate), pathNode.Rhs) + if err != nil { + return nil, err + } + + for lhsChild := lhs.Front(); lhsChild != nil; lhsChild = lhsChild.Next() { + lhsCandidate := lhsChild.Value.(*CandidateNode) + lhsTrue, errDecoding := isTruthy(lhsCandidate) + if errDecoding != nil { + return nil, errDecoding + } + + for rhsChild := rhs.Front(); rhsChild != nil; rhsChild = rhsChild.Next() { + rhsCandidate := rhsChild.Value.(*CandidateNode) + rhsTrue, errDecoding := isTruthy(rhsCandidate) + if errDecoding != nil { + return nil, errDecoding + } + boolResult := createBooleanCandidate(lhsCandidate, op(lhsTrue, rhsTrue)) + + results.Set(boolResult.GetKey(), boolResult) + } + } + + } + return results, nil +} + +func OrOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { + log.Debugf("-- orOp") + return booleanOp(d, matchingNodes, pathNode, func(b1 bool, b2 bool) bool { + return b1 || b2 + }) +} + +func AndOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { + log.Debugf("-- AndOp") + return booleanOp(d, matchingNodes, pathNode, func(b1 bool, b2 bool) bool { + return b1 && b2 + }) +} diff --git a/pkg/yqlib/treeops/operator_booleans_test.go b/pkg/yqlib/treeops/operator_booleans_test.go new file mode 100644 index 00000000..59f98953 --- /dev/null +++ b/pkg/yqlib/treeops/operator_booleans_test.go @@ -0,0 +1,35 @@ +package treeops + +import ( + "testing" +) + +var booleanOperatorScenarios = []expressionScenario{ + { + document: `{}`, + expression: `true or false`, + expected: []string{ + "D0, P[], (!!bool)::true\n", + }, + }, { + document: `{}`, + expression: `false or false`, + expected: []string{ + "D0, P[], (!!bool)::false\n", + }, + }, { + document: `{a: true, b: false}`, + expression: `.[] or (false, true)`, + expected: []string{ + "D0, P[a], (!!bool)::true\n", + "D0, P[b], (!!bool)::false\n", + "D0, P[b], (!!bool)::true\n", + }, + }, +} + +func TestBooleanOperatorScenarios(t *testing.T) { + for _, tt := range booleanOperatorScenarios { + testScenario(t, &tt) + } +} diff --git a/pkg/yqlib/treeops/operator_collect.go b/pkg/yqlib/treeops/operator_collect.go new file mode 100644 index 00000000..a1197faf --- /dev/null +++ b/pkg/yqlib/treeops/operator_collect.go @@ -0,0 +1,32 @@ +package treeops + +import ( + "github.com/elliotchance/orderedmap" + "gopkg.in/yaml.v3" +) + +func CollectOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { + log.Debugf("-- collectOperation") + + var results = orderedmap.NewOrderedMap() + + node := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"} + + var document uint = 0 + var path []interface{} + + for el := matchMap.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + log.Debugf("Collecting %v", NodeToString(candidate)) + if path == nil && candidate.Path != nil && len(candidate.Path) > 1 { + path = candidate.Path[:len(candidate.Path)-1] + document = candidate.Document + } + node.Content = append(node.Content, candidate.Node) + } + + collectC := &CandidateNode{Node: node, Document: document, Path: path} + results.Set(collectC.GetKey(), collectC) + + return results, nil +} diff --git a/pkg/yqlib/treeops/operator_collect_test.go b/pkg/yqlib/treeops/operator_collect_test.go new file mode 100644 index 00000000..3fe7535c --- /dev/null +++ b/pkg/yqlib/treeops/operator_collect_test.go @@ -0,0 +1,45 @@ +package treeops + +import ( + "testing" +) + +var collectOperatorScenarios = []expressionScenario{ + { + document: `{}`, + expression: `["cat"]`, + expected: []string{ + "D0, P[], (!!seq)::- cat\n", + }, + }, { + document: `{}`, + expression: `["cat", "dog"]`, + expected: []string{ + "D0, P[], (!!seq)::- cat\n- dog\n", + }, + }, { + document: `{}`, + expression: `1 | collect`, + expected: []string{ + "D0, P[], (!!seq)::- 1\n", + }, + }, { + document: `[1,2,3]`, + expression: `[.[]]`, + expected: []string{ + "D0, P[], (!!seq)::- 1\n- 2\n- 3\n", + }, + }, { + document: `a: {b: [1,2,3]}`, + expression: `[.a.b[]]`, + expected: []string{ + "D0, P[a b], (!!seq)::- 1\n- 2\n- 3\n", + }, + }, +} + +func TestCollectOperatorScenarios(t *testing.T) { + for _, tt := range collectOperatorScenarios { + testScenario(t, &tt) + } +} diff --git a/pkg/yqlib/treeops/delete_operator.go b/pkg/yqlib/treeops/operator_delete.go similarity index 100% rename from pkg/yqlib/treeops/delete_operator.go rename to pkg/yqlib/treeops/operator_delete.go diff --git a/pkg/yqlib/treeops/equals_operator.go b/pkg/yqlib/treeops/operator_equals.go similarity index 82% rename from pkg/yqlib/treeops/equals_operator.go rename to pkg/yqlib/treeops/operator_equals.go index 4a3aee42..18dfa1c6 100644 --- a/pkg/yqlib/treeops/equals_operator.go +++ b/pkg/yqlib/treeops/operator_equals.go @@ -2,7 +2,6 @@ package treeops import ( "github.com/elliotchance/orderedmap" - "gopkg.in/yaml.v3" ) func EqualsOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { @@ -18,15 +17,8 @@ func EqualsOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathN return nil, errInChild } - matchString := "true" - if !matches { - matchString = "false" - } - - node := &yaml.Node{Kind: yaml.ScalarNode, Value: matchString, Tag: "!!bool"} - lengthCand := &CandidateNode{Node: node, Document: candidate.Document, Path: candidate.Path} - results.Set(candidate.GetKey(), lengthCand) - + equalsCandidate := createBooleanCandidate(candidate, matches) + results.Set(equalsCandidate.GetKey(), equalsCandidate) } return results, nil diff --git a/pkg/yqlib/treeops/operator_equals_test.go b/pkg/yqlib/treeops/operator_equals_test.go new file mode 100644 index 00000000..e6f210a2 --- /dev/null +++ b/pkg/yqlib/treeops/operator_equals_test.go @@ -0,0 +1,43 @@ +package treeops + +import ( + "testing" +) + +var equalsOperatorScenarios = []expressionScenario{ + { + document: `[cat,goat,dog]`, + expression: `(.[] == "*at")`, + expected: []string{ + "D0, P[], (!!bool)::true\n", + }, + }, { + document: `[cat,goat,dog]`, + expression: `.[] | (. == "*at")`, + expected: []string{ + "D0, P[0], (!!bool)::true\n", + "D0, P[1], (!!bool)::true\n", + "D0, P[2], (!!bool)::false\n", + }, + }, { + document: `[3, 4, 5]`, + expression: `.[] | (. == 4)`, + expected: []string{ + "D0, P[0], (!!bool)::false\n", + "D0, P[1], (!!bool)::true\n", + "D0, P[2], (!!bool)::false\n", + }, + }, { + document: `a: { cat: {b: apple, c: whatever}, pat: {b: banana} }`, + expression: `.a | (.[].b == "apple")`, + expected: []string{ + "D0, P[a], (!!bool)::true\n", + }, + }, +} + +func TestEqualOperatorScenarios(t *testing.T) { + for _, tt := range equalsOperatorScenarios { + testScenario(t, &tt) + } +} diff --git a/pkg/yqlib/treeops/operator_select.go b/pkg/yqlib/treeops/operator_select.go new file mode 100644 index 00000000..fb92a90e --- /dev/null +++ b/pkg/yqlib/treeops/operator_select.go @@ -0,0 +1,37 @@ +package treeops + +import ( + "github.com/elliotchance/orderedmap" +) + +func SelectOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { + + log.Debugf("-- selectOperation") + var results = orderedmap.NewOrderedMap() + + for el := matchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + + rhs, err := d.getMatchingNodes(nodeToMap(candidate), pathNode.Rhs) + + if err != nil { + return nil, err + } + + // grab the first value + first := rhs.Front() + + if first != nil { + result := first.Value.(*CandidateNode) + includeResult, errDecoding := isTruthy(result) + if errDecoding != nil { + return nil, errDecoding + } + + if includeResult { + results.Set(candidate.GetKey(), candidate) + } + } + } + return results, nil +} diff --git a/pkg/yqlib/treeops/operator_select_test.go b/pkg/yqlib/treeops/operator_select_test.go new file mode 100644 index 00000000..70536768 --- /dev/null +++ b/pkg/yqlib/treeops/operator_select_test.go @@ -0,0 +1,45 @@ +package treeops + +import ( + "testing" +) + +var selectOperatorScenarios = []expressionScenario{ + { + document: `[cat,goat,dog]`, + expression: `.[] | select(. == "*at")`, + expected: []string{ + "D0, P[0], (!!str)::cat\n", + "D0, P[1], (!!str)::goat\n", + }, + }, { + document: `[hot, fot, dog]`, + expression: `.[] | select(. == "*at")`, + expected: []string{}, + }, { + document: `a: [cat,goat,dog]`, + expression: `.a[] | select(. == "*at")`, + expected: []string{ + "D0, P[a 0], (!!str)::cat\n", + "D0, P[a 1], (!!str)::goat\n"}, + }, { + document: `a: { things: cat, bob: goat, horse: dog }`, + expression: `.a[] | select(. == "*at")`, + expected: []string{ + "D0, P[a things], (!!str)::cat\n", + "D0, P[a bob], (!!str)::goat\n"}, + }, { + document: `a: { things: {include: true}, notMe: {include: false}, andMe: {include: fold} }`, + expression: `.a[] | select(.include)`, + expected: []string{ + "D0, P[a things], (!!map)::{include: true}\n", + "D0, P[a andMe], (!!map)::{include: fold}\n", + }, + }, +} + +func TestSelectOperatorScenarios(t *testing.T) { + for _, tt := range selectOperatorScenarios { + testScenario(t, &tt) + } +} diff --git a/pkg/yqlib/treeops/operator_union.go b/pkg/yqlib/treeops/operator_union.go new file mode 100644 index 00000000..df44e9f6 --- /dev/null +++ b/pkg/yqlib/treeops/operator_union.go @@ -0,0 +1,19 @@ +package treeops + +import "github.com/elliotchance/orderedmap" + +func UnionOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { + lhs, err := d.getMatchingNodes(matchingNodes, pathNode.Lhs) + if err != nil { + return nil, err + } + rhs, err := d.getMatchingNodes(matchingNodes, pathNode.Rhs) + if err != nil { + return nil, err + } + for el := rhs.Front(); el != nil; el = el.Next() { + node := el.Value.(*CandidateNode) + lhs.Set(node.GetKey(), node) + } + return lhs, nil +} diff --git a/pkg/yqlib/treeops/operator_union_test.go b/pkg/yqlib/treeops/operator_union_test.go new file mode 100644 index 00000000..ff80f1d6 --- /dev/null +++ b/pkg/yqlib/treeops/operator_union_test.go @@ -0,0 +1,31 @@ +package treeops + +import ( + "testing" +) + +var unionOperatorScenarios = []expressionScenario{ + { + document: `{}`, + expression: `"cat", "dog"`, + expected: []string{ + "D0, P[], (!!str)::cat\n", + "D0, P[], (!!str)::dog\n", + }, + }, { + document: `{a: frog}`, + expression: `1, true, "cat", .a`, + expected: []string{ + "D0, P[], (!!int)::1\n", + "D0, P[], (!!bool)::true\n", + "D0, P[], (!!str)::cat\n", + "D0, P[a], (!!str)::frog\n", + }, + }, +} + +func TestUnionOperatorScenarios(t *testing.T) { + for _, tt := range unionOperatorScenarios { + testScenario(t, &tt) + } +} diff --git a/pkg/yqlib/treeops/operators.go b/pkg/yqlib/treeops/operators.go index 9e5afcaf..5b5068d0 100644 --- a/pkg/yqlib/treeops/operators.go +++ b/pkg/yqlib/treeops/operators.go @@ -17,6 +17,15 @@ func PipeOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pa return d.getMatchingNodes(lhs, pathNode.Rhs) } +func createBooleanCandidate(owner *CandidateNode, value bool) *CandidateNode { + valString := "true" + if !value { + valString = "false" + } + node := &yaml.Node{Kind: yaml.ScalarNode, Value: valString, Tag: "!!bool"} + return &CandidateNode{Node: node, Document: owner.Document, Path: owner.Path} +} + func nodeToMap(candidate *CandidateNode) *orderedmap.OrderedMap { elMap := orderedmap.NewOrderedMap() elMap.Set(candidate.GetKey(), candidate) @@ -47,22 +56,6 @@ func AssignOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, return lhs, nil } -func UnionOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { - lhs, err := d.getMatchingNodes(matchingNodes, pathNode.Lhs) - if err != nil { - return nil, err - } - rhs, err := d.getMatchingNodes(matchingNodes, pathNode.Rhs) - if err != nil { - return nil, err - } - for el := rhs.Front(); el != nil; el = el.Next() { - node := el.Value.(*CandidateNode) - lhs.Set(node.GetKey(), node) - } - return lhs, nil -} - func IntersectionOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { lhs, err := d.getMatchingNodes(matchingNodes, pathNode.Lhs) if err != nil { @@ -82,16 +75,6 @@ func IntersectionOperator(d *dataTreeNavigator, matchingNodes *orderedmap.Ordere return matchingNodeMap, nil } -func splatNode(d *dataTreeNavigator, candidate *CandidateNode) (*orderedmap.OrderedMap, error) { - //need to splat matching nodes, then search through them - splatter := &PathTreeNode{PathElement: &PathElement{ - PathElementType: PathKey, - Value: "*", - StringValue: "*", - }} - return d.getMatchingNodes(nodeToMap(candidate), splatter) -} - func LengthOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { log.Debugf("-- lengthOperation") var results = orderedmap.NewOrderedMap() @@ -117,29 +100,3 @@ func LengthOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathN return results, nil } - -func CollectOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { - log.Debugf("-- collectOperation") - - var results = orderedmap.NewOrderedMap() - - node := &yaml.Node{Kind: yaml.SequenceNode} - - var document uint = 0 - var path []interface{} - - for el := matchMap.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - if path == nil && candidate.Path != nil { - path = candidate.Path - document = candidate.Document - } - node.Content = append(node.Content, candidate.Node) - } - - collectC := &CandidateNode{Node: node, Document: document, Path: path} - results.Set(collectC.GetKey(), collectC) - - return results, nil - -} diff --git a/pkg/yqlib/treeops/operators_test.go b/pkg/yqlib/treeops/operators_test.go new file mode 100644 index 00000000..fedd6dd9 --- /dev/null +++ b/pkg/yqlib/treeops/operators_test.go @@ -0,0 +1,28 @@ +package treeops + +import ( + "testing" + + "github.com/mikefarah/yq/v3/test" +) + +type expressionScenario struct { + document string + expression string + expected []string +} + +func testScenario(t *testing.T, s *expressionScenario) { + + nodes := readDoc(t, s.document) + path, errPath := treeCreator.ParsePath(s.expression) + if errPath != nil { + t.Error(errPath) + } + results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + + if errNav != nil { + t.Error(errNav) + } + test.AssertResultComplexWithContext(t, s.expected, resultsToString(results), s.expression) +} diff --git a/pkg/yqlib/treeops/path_tokeniser_test.go b/pkg/yqlib/treeops/path_parse_test.go similarity index 81% rename from pkg/yqlib/treeops/path_tokeniser_test.go rename to pkg/yqlib/treeops/path_parse_test.go index 731460e3..b7d56213 100644 --- a/pkg/yqlib/treeops/path_tokeniser_test.go +++ b/pkg/yqlib/treeops/path_parse_test.go @@ -1,14 +1,16 @@ package treeops import ( + "fmt" "testing" "github.com/mikefarah/yq/v3/test" ) -var tokeniserTests = []struct { - path string - expectedTokens []interface{} +var pathTests = []struct { + path string + expectedTokens []interface{} + expectedPostFix []interface{} }{ // TODO: Ensure ALL documented examples have tests! sheesh // {"len(.)", append(make([]interface{}, 0), "LENGTH", "(", "SELF", ")")}, // {"\"len\"(.)", append(make([]interface{}, 0), "len", "TRAVERSE", "(", "SELF", ")")}, @@ -37,7 +39,21 @@ var tokeniserTests = []struct { // {`."[a", ."b]"`, append(make([]interface{}, 0), "[a", "OR", "b]")}, // {`.a[]`, append(make([]interface{}, 0), "a", "PIPE", "[]")}, // {`.[].a`, append(make([]interface{}, 0), "[]", "PIPE", "a")}, - {`.a | (.[].b == "apple")`, append(make([]interface{}, 0), "a", "PIPE", "(", "[]", "PIPE", "b", "EQUALS", "apple", ")")}, + { + `.a | (.[].b == "apple")`, + append(make([]interface{}, 0), "a", "PIPE", "(", "[]", "PIPE", "b", "EQUALS", "apple", ")"), + append(make([]interface{}, 0), "a", "[]", "b", "PIPE", "apple (string)", "EQUALS", "PIPE"), + }, + { + `.[] | select(. == "*at")`, + append(make([]interface{}, 0), "[]", "PIPE", "SELECT", "(", "SELF", "EQUALS", "*at", ")"), + append(make([]interface{}, 0), "[]", "SELF", "*at (string)", "EQUALS", "SELECT", "PIPE"), + }, + { + `[true]`, + append(make([]interface{}, 0), "[", true, "]"), + append(make([]interface{}, 0), "true (bool)", "COLLECT"), + }, // {".animals | .==cat", append(make([]interface{}, 0), "animals", "TRAVERSE", "SELF", "EQUALS", "cat")}, // {".animals | (. == cat)", append(make([]interface{}, 0), "animals", "TRAVERSE", "(", "SELF", "EQUALS", "cat", ")")}, @@ -61,8 +77,8 @@ var tokeniserTests = []struct { var tokeniser = NewPathTokeniser() -func TestTokeniser(t *testing.T) { - for _, tt := range tokeniserTests { +func TestPathParsing(t *testing.T) { + for _, tt := range pathTests { tokens, err := tokeniser.Tokenise(tt.path) if err != nil { t.Error(tt.path, err) @@ -71,6 +87,20 @@ func TestTokeniser(t *testing.T) { for _, token := range tokens { tokenValues = append(tokenValues, token.Value) } - test.AssertResultComplexWithContext(t, tt.expectedTokens, tokenValues, tt.path) + test.AssertResultComplexWithContext(t, tt.expectedTokens, tokenValues, fmt.Sprintf("tokenise: %v", tt.path)) + + results, errorP := postFixer.ConvertToPostfix(tokens) + + var readableResults []interface{} + for _, token := range results { + readableResults = append(readableResults, token.toString()) + } + + if errorP != nil { + t.Error(tt.path, err) + } + + test.AssertResultComplexWithContext(t, tt.expectedPostFix, readableResults, fmt.Sprintf("postfix: %v", tt.path)) + } } diff --git a/pkg/yqlib/treeops/path_postfix.go b/pkg/yqlib/treeops/path_postfix.go index f1c46cd4..3ed3e009 100644 --- a/pkg/yqlib/treeops/path_postfix.go +++ b/pkg/yqlib/treeops/path_postfix.go @@ -2,6 +2,8 @@ package treeops import ( "errors" + + "gopkg.in/op/go-logging.v1" ) type PathPostFixer interface { @@ -43,9 +45,10 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*PathElement, if len(opStack) == 0 { return nil, errors.New("Bad path expression, got close collect brackets without matching opening bracket") } - // now we should have ( as the last element on the opStack, get rid of it + // now we should have [] as the last element on the opStack, get rid of it opStack = opStack[0 : len(opStack)-1] //and append a collect to the opStack + opStack = append(opStack, &Token{PathElementType: Operation, OperationType: Pipe}) opStack = append(opStack, &Token{PathElementType: Operation, OperationType: Collect}) case CloseBracket: for len(opStack) > 0 && opStack[len(opStack)-1].PathElementType != OpenBracket { @@ -67,5 +70,13 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*PathElement, opStack = append(opStack, token) } } + + if log.IsEnabledFor(logging.DEBUG) { + log.Debugf("PostFix Result:") + for _, token := range result { + log.Debugf("> %v", token.toString()) + } + } + return result, nil } diff --git a/pkg/yqlib/treeops/path_postfix_test.go b/pkg/yqlib/treeops/path_postfix_test.go index 0673424d..ef5382f2 100644 --- a/pkg/yqlib/treeops/path_postfix_test.go +++ b/pkg/yqlib/treeops/path_postfix_test.go @@ -20,11 +20,12 @@ func testExpression(expression string) (string, error) { } formatted := "" for _, path := range results { - formatted = formatted + path.toString() + "\n--------\n" + formatted = formatted + path.toString() + ", " } return formatted, nil } + func TestPostFixTraverseBar(t *testing.T) { var infix = ".animals | [.]" var expectedOutput = `PathKey - animals diff --git a/pkg/yqlib/treeops/path_tokeniser.go b/pkg/yqlib/treeops/path_tokeniser.go index e9d91b8c..f00651fc 100644 --- a/pkg/yqlib/treeops/path_tokeniser.go +++ b/pkg/yqlib/treeops/path_tokeniser.go @@ -107,9 +107,12 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`\.?\[\]`), literalToken(PathKey, "[]", true)) lexer.Add([]byte(`\.\.`), literalToken(PathKey, "..", true)) - lexer.Add([]byte(`,`), opToken(Or)) + lexer.Add([]byte(`,`), opToken(Union)) lexer.Add([]byte(`length`), opToken(Length)) - lexer.Add([]byte(`([Cc][Oo][Ll][Ll][Ee][Cc][Tt])`), opToken(Collect)) + lexer.Add([]byte(`select`), opToken(Select)) + lexer.Add([]byte(`or`), opToken(Or)) + lexer.Add([]byte(`and`), opToken(And)) + lexer.Add([]byte(`collect`), opToken(Collect)) lexer.Add([]byte(`\s*==\s*`), opToken(Equals)) diff --git a/pkg/yqlib/treeops/path_tree.go b/pkg/yqlib/treeops/path_tree.go index ce7c6523..30c67ee9 100644 --- a/pkg/yqlib/treeops/path_tree.go +++ b/pkg/yqlib/treeops/path_tree.go @@ -52,7 +52,7 @@ func (p *pathTreeCreator) CreatePathTree(postFixPath []*PathElement) (*PathTreeN remaining, rhs := stack[:len(stack)-1], stack[len(stack)-1] newNode.Rhs = rhs stack = remaining - } else { + } else if numArgs == 2 { remaining, lhs, rhs := stack[:len(stack)-2], stack[len(stack)-2], stack[len(stack)-1] newNode.Lhs = lhs newNode.Rhs = rhs