From 4f574efdc4b527d85b850d41d9700cbf61d7835a Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Tue, 20 Oct 2020 15:33:20 +1100 Subject: [PATCH] simplified, refactored --- pkg/yqlib/treeops/data_tree_navigator.go | 6 +- pkg/yqlib/treeops/data_tree_navigator_test.go | 1524 +++++++---------- pkg/yqlib/treeops/lib.go | 31 +- pkg/yqlib/treeops/operator_equals.go | 2 +- pkg/yqlib/treeops/operator_multilpy.go | 12 +- .../treeops/operator_recursive_descent.go | 4 +- pkg/yqlib/treeops/operator_self.go | 7 + pkg/yqlib/treeops/operator_traverse_path.go | 10 +- .../treeops/operator_traverse_path_test.go | 54 + pkg/yqlib/treeops/operator_value.go | 7 + pkg/yqlib/treeops/operator_value_test.go | 3 +- pkg/yqlib/treeops/path_parse_test.go | 12 +- pkg/yqlib/treeops/path_postfix.go | 33 +- pkg/yqlib/treeops/path_tokeniser.go | 72 +- pkg/yqlib/treeops/path_tree.go | 22 +- pkg/yqlib/treeops/sample.yaml | 2 +- pkg/yqlib/treeops/temp.json | 8 +- pkg/yqlib/treeops/value_node_builder.go | 20 - 18 files changed, 848 insertions(+), 981 deletions(-) create mode 100644 pkg/yqlib/treeops/operator_self.go create mode 100644 pkg/yqlib/treeops/operator_value.go delete mode 100644 pkg/yqlib/treeops/value_node_builder.go diff --git a/pkg/yqlib/treeops/data_tree_navigator.go b/pkg/yqlib/treeops/data_tree_navigator.go index 7306300c..d502047e 100644 --- a/pkg/yqlib/treeops/data_tree_navigator.go +++ b/pkg/yqlib/treeops/data_tree_navigator.go @@ -48,17 +48,17 @@ func (d *dataTreeNavigator) getMatchingNodes(matchingNodes *orderedmap.OrderedMa log.Debugf("getMatchingNodes - nothing to do") return matchingNodes, nil } - log.Debugf("Processing Path: %v", pathNode.PathElement.toString()) + log.Debugf("Processing Op: %v", pathNode.Operation.toString()) if log.IsEnabledFor(logging.DEBUG) { for el := matchingNodes.Front(); el != nil; el = el.Next() { log.Debug(NodeToString(el.Value.(*CandidateNode))) } } log.Debug(">>") - handler := pathNode.PathElement.OperationType.Handler + handler := pathNode.Operation.OperationType.Handler if handler != nil { return handler(d, matchingNodes, pathNode) } - return nil, fmt.Errorf("Unknown operator %v", pathNode.PathElement.OperationType) + return nil, fmt.Errorf("Unknown operator %v", pathNode.Operation.OperationType) } diff --git a/pkg/yqlib/treeops/data_tree_navigator_test.go b/pkg/yqlib/treeops/data_tree_navigator_test.go index 234d6296..750bf000 100644 --- a/pkg/yqlib/treeops/data_tree_navigator_test.go +++ b/pkg/yqlib/treeops/data_tree_navigator_test.go @@ -4,7 +4,6 @@ import ( "strings" "testing" - "github.com/mikefarah/yq/v3/test" yaml "gopkg.in/yaml.v3" ) @@ -33,877 +32,652 @@ func resultsToString(results []*CandidateNode) []string { return pretty } -func TestDataTreeNavigatorSimple(t *testing.T) { - - nodes := readDoc(t, `a: - b: apple`) - - path, errPath := treeCreator.ParsePath("a") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [a] - Tag: !!map, Kind: MappingNode, Anchor: - b: apple -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorDeleteSimple(t *testing.T) { - - nodes := readDoc(t, `a: - b: apple - c: camel`) - - path, errPath := treeCreator.ParsePath("a .- b") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [a] - Tag: !!map, Kind: MappingNode, Anchor: - c: camel -` - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorDeleteTwice(t *testing.T) { - - nodes := readDoc(t, `a: - b: apple - c: camel - d: dingo`) - - path, errPath := treeCreator.ParsePath("a .- b OR a .- c") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [a] - Tag: !!map, Kind: MappingNode, Anchor: - d: dingo -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorDeleteWithUnion(t *testing.T) { - - nodes := readDoc(t, `a: - b: apple - c: camel - d: dingo`) - - path, errPath := treeCreator.ParsePath("a .- (b OR c)") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [a] - Tag: !!map, Kind: MappingNode, Anchor: - d: dingo -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorDeleteByIndex(t *testing.T) { - - nodes := readDoc(t, `a: - - b: apple - - b: sdfsd - - b: apple`) - - path, errPath := treeCreator.ParsePath("(a .- (0 or 1))") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [a] - Tag: !!seq, Kind: SequenceNode, Anchor: - - b: apple -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorDeleteByFind(t *testing.T) { - - nodes := readDoc(t, `a: - - b: apple - - b: sdfsd - - b: apple`) - - path, errPath := treeCreator.ParsePath("(a .- (* == apple))") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [a] - Tag: !!seq, Kind: SequenceNode, Anchor: - - b: sdfsd -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorDeleteArrayByFind(t *testing.T) { - - nodes := readDoc(t, `a: - - apple - - sdfsd - - apple`) - - path, errPath := treeCreator.ParsePath("(a .- (. == apple))") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [a] - Tag: !!seq, Kind: SequenceNode, Anchor: - - sdfsd -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorDeleteViaSelf(t *testing.T) { - - nodes := readDoc(t, `- apple -- sdfsd -- apple`) - - path, errPath := treeCreator.ParsePath(". .- (. == apple)") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [] - Tag: !!seq, Kind: SequenceNode, Anchor: - - sdfsd -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorFilterWithSplat(t *testing.T) { - - nodes := readDoc(t, `f: - a: frog - b: dally - c: log`) - - path, errPath := treeCreator.ParsePath(".f | .[] == \"frog\"") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [f] - Tag: !!int, Kind: ScalarNode, Anchor: - 2 -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorCountAndCollectWithFilterCmd(t *testing.T) { - - nodes := readDoc(t, `f: - a: frog - b: dally - c: log`) - - path, errPath := treeCreator.ParsePath(".f | .[] == *og ") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [f] - Tag: !!int, Kind: ScalarNode, Anchor: - 2 -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorCollectWithFilter(t *testing.T) { - - nodes := readDoc(t, `f: - a: frog - b: dally - c: log`) - - path, errPath := treeCreator.ParsePath("f(collect(. == *og))") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [f] - Tag: , Kind: SequenceNode, Anchor: - - frog -- log -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorCountWithFilter2(t *testing.T) { - - nodes := readDoc(t, `f: - a: frog - b: dally - c: log`) - - path, errPath := treeCreator.ParsePath("count(f(. == *og))") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [] - Tag: !!int, Kind: ScalarNode, Anchor: - 2 -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorCollectWithFilter2(t *testing.T) { - - nodes := readDoc(t, `f: - a: frog - b: dally - c: log`) - - path, errPath := treeCreator.ParsePath("collect(f(. == *og))") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [] - Tag: , Kind: SequenceNode, Anchor: - - frog -- log -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorCountMultipleMatchesInside(t *testing.T) { - - nodes := readDoc(t, `f: - a: [1,2] - b: dally - c: [3,4,5]`) - - path, errPath := treeCreator.ParsePath("f | count(a or c)") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [f] - Tag: !!int, Kind: ScalarNode, Anchor: - 2 -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorCollectMultipleMatchesInside(t *testing.T) { - - nodes := readDoc(t, `f: - a: [1,2] - b: dally - c: [3,4,5]`) - - path, errPath := treeCreator.ParsePath("f | collect(a or c)") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [f] - Tag: , Kind: SequenceNode, Anchor: - - [1, 2] -- [3, 4, 5] -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorCountMultipleMatchesInsideSplat(t *testing.T) { - - nodes := readDoc(t, `f: - a: [1,2,3] - b: [1,2,3,4] - c: [1,2,3,4,5]`) - - path, errPath := treeCreator.ParsePath("f(count( (a or c)*))") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [f] - Tag: !!int, Kind: ScalarNode, Anchor: - 8 -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorCountMultipleMatchesOutside(t *testing.T) { - - nodes := readDoc(t, `f: - a: [1,2,3] - b: [1,2,3,4] - c: [1,2,3,4,5]`) - - path, errPath := treeCreator.ParsePath("f(a or c)(count(*))") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [f a] - Tag: !!int, Kind: ScalarNode, Anchor: - 3 --- Node -- - Document 0, path: [f c] - Tag: !!int, Kind: ScalarNode, Anchor: - 5 -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorCountOfResults(t *testing.T) { - - nodes := readDoc(t, `- apple -- sdfsd -- apple`) - - path, errPath := treeCreator.ParsePath("count(*)") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [] - Tag: !!int, Kind: ScalarNode, Anchor: - 3 -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorCountNoMatches(t *testing.T) { - - nodes := readDoc(t, `- apple -- sdfsd -- apple`) - - path, errPath := treeCreator.ParsePath("count(5)") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [] - Tag: !!int, Kind: ScalarNode, Anchor: - 0 -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorDeleteAndWrite(t *testing.T) { - - nodes := readDoc(t, `a: - - b: apple - - b: sdfsd - - { b: apple, c: cat }`) - - path, errPath := treeCreator.ParsePath("(a .- (0 or 1)) or (a[0].b := frog)") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [a] - Tag: !!seq, Kind: SequenceNode, Anchor: - - {b: frog, c: cat} - --- Node -- - Document 0, path: [a 0 b] - Tag: !!str, Kind: ScalarNode, Anchor: - frog -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorDeleteArray(t *testing.T) { - - nodes := readDoc(t, `a: - - b: apple - - b: sdfsd - - b: apple`) - - path, errPath := treeCreator.ParsePath("a .- (b == a*)") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [a] - Tag: !!seq, Kind: SequenceNode, Anchor: - - b: sdfsd -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorArraySimple(t *testing.T) { - - nodes := readDoc(t, `- b: apple`) - - path, errPath := treeCreator.ParsePath("[0]") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [0] - Tag: !!map, Kind: MappingNode, Anchor: - b: apple -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorSimpleAssignByFind(t *testing.T) { - - nodes := readDoc(t, `a: - b: apple`) - - path, errPath := treeCreator.ParsePath("a(. == apple) := frog") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [a b] - Tag: !!str, Kind: ScalarNode, Anchor: - frog -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorArraySplat(t *testing.T) { - - nodes := readDoc(t, `- b: apple -- c: banana`) - - path, errPath := treeCreator.ParsePath("[*]") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [0] - Tag: !!map, Kind: MappingNode, Anchor: - b: apple - --- Node -- - Document 0, path: [1] - Tag: !!map, Kind: MappingNode, Anchor: - c: banana -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorSimpleDeep(t *testing.T) { - - nodes := readDoc(t, `a: - b: apple`) - - path, errPath := treeCreator.ParsePath("a.b") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [a b] - Tag: !!str, Kind: ScalarNode, Anchor: - apple -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorSimpleMismatch(t *testing.T) { - - nodes := readDoc(t, `a: - c: apple`) - - path, errPath := treeCreator.ParsePath("a.b") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := `` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorWild(t *testing.T) { - - nodes := readDoc(t, `a: - cat: apple - mad: things`) - - path, errPath := treeCreator.ParsePath("a.*a*") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [a cat] - Tag: !!str, Kind: ScalarNode, Anchor: - apple - --- Node -- - Document 0, path: [a mad] - Tag: !!str, Kind: ScalarNode, Anchor: - things -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorWildDeepish(t *testing.T) { - - nodes := readDoc(t, `a: - cat: {b: 3} - mad: {b: 4} - fad: {c: t}`) - - path, errPath := treeCreator.ParsePath("a.*a*.b") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [a cat b] - Tag: !!int, Kind: ScalarNode, Anchor: - 3 - --- Node -- - Document 0, path: [a mad b] - Tag: !!int, Kind: ScalarNode, Anchor: - 4 -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorOrSimple(t *testing.T) { - - nodes := readDoc(t, `a: - cat: apple - mad: things`) - - path, errPath := treeCreator.ParsePath("a.(cat or mad)") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [a cat] - Tag: !!str, Kind: ScalarNode, Anchor: - apple - --- Node -- - Document 0, path: [a mad] - Tag: !!str, Kind: ScalarNode, Anchor: - things -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorOrSimpleWithDepth(t *testing.T) { - - nodes := readDoc(t, `a: - cat: {b: 3} - mad: {b: 4} - fad: {c: t}`) - - path, errPath := treeCreator.ParsePath("a.(cat.* or fad.*)") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [a cat b] - Tag: !!int, Kind: ScalarNode, Anchor: - 3 - --- Node -- - Document 0, path: [a fad c] - Tag: !!str, Kind: ScalarNode, Anchor: - t -` - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorOrDeDupes(t *testing.T) { - - nodes := readDoc(t, `a: - cat: apple - mad: things`) - - path, errPath := treeCreator.ParsePath("a.(cat or cat)") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [a cat] - Tag: !!str, Kind: ScalarNode, Anchor: - apple -` - - test.AssertResult(t, expected, resultsToString(results)) -} - -func TestDataTreeNavigatorAnd(t *testing.T) { - - nodes := readDoc(t, `a: - cat: apple - pat: apple - cow: apple - mad: things`) - - path, errPath := treeCreator.ParsePath("a.(*t and c*)") - if errPath != nil { - t.Error(errPath) - } - results, errNav := treeNavigator.GetMatchingNodes(nodes, path) - - if errNav != nil { - t.Error(errNav) - } - - expected := ` --- Node -- - Document 0, path: [a cat] - Tag: !!str, Kind: ScalarNode, Anchor: - apple -` - - test.AssertResult(t, expected, resultsToString(results)) -} +// func TestDataTreeNavigatorDeleteSimple(t *testing.T) { + +// nodes := readDoc(t, `a: +// b: apple +// c: camel`) + +// path, errPath := treeCreator.ParsePath("a .- b") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [a] +// Tag: !!map, Kind: MappingNode, Anchor: +// c: camel +// ` +// test.AssertResult(t, expected, resultsToString(results)) +// } + +// func TestDataTreeNavigatorDeleteTwice(t *testing.T) { + +// nodes := readDoc(t, `a: +// b: apple +// c: camel +// d: dingo`) + +// path, errPath := treeCreator.ParsePath("a .- b OR a .- c") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [a] +// Tag: !!map, Kind: MappingNode, Anchor: +// d: dingo +// ` + +// test.AssertResult(t, expected, resultsToString(results)) +// } + +// func TestDataTreeNavigatorDeleteWithUnion(t *testing.T) { + +// nodes := readDoc(t, `a: +// b: apple +// c: camel +// d: dingo`) + +// path, errPath := treeCreator.ParsePath("a .- (b OR c)") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [a] +// Tag: !!map, Kind: MappingNode, Anchor: +// d: dingo +// ` + +// test.AssertResult(t, expected, resultsToString(results)) +// } + +// func TestDataTreeNavigatorDeleteByIndex(t *testing.T) { + +// nodes := readDoc(t, `a: +// - b: apple +// - b: sdfsd +// - b: apple`) + +// path, errPath := treeCreator.ParsePath("(a .- (0 or 1))") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [a] +// Tag: !!seq, Kind: SequenceNode, Anchor: +// - b: apple +// ` + +// test.AssertResult(t, expected, resultsToString(results)) +// } + +// func TestDataTreeNavigatorDeleteByFind(t *testing.T) { + +// nodes := readDoc(t, `a: +// - b: apple +// - b: sdfsd +// - b: apple`) + +// path, errPath := treeCreator.ParsePath("(a .- (* == apple))") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [a] +// Tag: !!seq, Kind: SequenceNode, Anchor: +// - b: sdfsd +// ` + +// test.AssertResult(t, expected, resultsToString(results)) +// } + +// func TestDataTreeNavigatorDeleteArrayByFind(t *testing.T) { + +// nodes := readDoc(t, `a: +// - apple +// - sdfsd +// - apple`) + +// path, errPath := treeCreator.ParsePath("(a .- (. == apple))") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [a] +// Tag: !!seq, Kind: SequenceNode, Anchor: +// - sdfsd +// ` + +// test.AssertResult(t, expected, resultsToString(results)) +// } + +// func TestDataTreeNavigatorDeleteViaSelf(t *testing.T) { + +// nodes := readDoc(t, `- apple +// - sdfsd +// - apple`) + +// path, errPath := treeCreator.ParsePath(". .- (. == apple)") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [] +// Tag: !!seq, Kind: SequenceNode, Anchor: +// - sdfsd +// ` + +// test.AssertResult(t, expected, resultsToString(results)) +// } + +// func TestDataTreeNavigatorFilterWithSplat(t *testing.T) { + +// nodes := readDoc(t, `f: +// a: frog +// b: dally +// c: log`) + +// path, errPath := treeCreator.ParsePath(".f | .[] == \"frog\"") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [f] +// Tag: !!int, Kind: ScalarNode, Anchor: +// 2 +// ` + +// test.AssertResult(t, expected, resultsToString(results)) +// } + +// func TestDataTreeNavigatorCountAndCollectWithFilterCmd(t *testing.T) { + +// nodes := readDoc(t, `f: +// a: frog +// b: dally +// c: log`) + +// path, errPath := treeCreator.ParsePath(".f | .[] == *og ") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [f] +// Tag: !!int, Kind: ScalarNode, Anchor: +// 2 +// ` + +// test.AssertResult(t, expected, resultsToString(results)) +// } + +// func TestDataTreeNavigatorCollectWithFilter(t *testing.T) { + +// nodes := readDoc(t, `f: +// a: frog +// b: dally +// c: log`) + +// path, errPath := treeCreator.ParsePath("f(collect(. == *og))") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [f] +// Tag: , Kind: SequenceNode, Anchor: +// - frog +// - log +// ` + +// test.AssertResult(t, expected, resultsToString(results)) +// } + +// func TestDataTreeNavigatorCountWithFilter2(t *testing.T) { + +// nodes := readDoc(t, `f: +// a: frog +// b: dally +// c: log`) + +// path, errPath := treeCreator.ParsePath("count(f(. == *og))") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [] +// Tag: !!int, Kind: ScalarNode, Anchor: +// 2 +// ` + +// test.AssertResult(t, expected, resultsToString(results)) +// } + +// func TestDataTreeNavigatorCollectWithFilter2(t *testing.T) { + +// nodes := readDoc(t, `f: +// a: frog +// b: dally +// c: log`) + +// path, errPath := treeCreator.ParsePath("collect(f(. == *og))") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [] +// Tag: , Kind: SequenceNode, Anchor: +// - frog +// - log +// ` + +// test.AssertResult(t, expected, resultsToString(results)) +// } + +// func TestDataTreeNavigatorCountMultipleMatchesInside(t *testing.T) { + +// nodes := readDoc(t, `f: +// a: [1,2] +// b: dally +// c: [3,4,5]`) + +// path, errPath := treeCreator.ParsePath("f | count(a or c)") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [f] +// Tag: !!int, Kind: ScalarNode, Anchor: +// 2 +// ` + +// test.AssertResult(t, expected, resultsToString(results)) +// } + +// func TestDataTreeNavigatorCollectMultipleMatchesInside(t *testing.T) { + +// nodes := readDoc(t, `f: +// a: [1,2] +// b: dally +// c: [3,4,5]`) + +// path, errPath := treeCreator.ParsePath("f | collect(a or c)") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [f] +// Tag: , Kind: SequenceNode, Anchor: +// - [1, 2] +// - [3, 4, 5] +// ` + +// test.AssertResult(t, expected, resultsToString(results)) +// } + +// func TestDataTreeNavigatorCountMultipleMatchesInsideSplat(t *testing.T) { + +// nodes := readDoc(t, `f: +// a: [1,2,3] +// b: [1,2,3,4] +// c: [1,2,3,4,5]`) + +// path, errPath := treeCreator.ParsePath("f(count( (a or c)*))") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [f] +// Tag: !!int, Kind: ScalarNode, Anchor: +// 8 +// ` + +// test.AssertResult(t, expected, resultsToString(results)) +// } + +// func TestDataTreeNavigatorCountMultipleMatchesOutside(t *testing.T) { + +// nodes := readDoc(t, `f: +// a: [1,2,3] +// b: [1,2,3,4] +// c: [1,2,3,4,5]`) + +// path, errPath := treeCreator.ParsePath("f(a or c)(count(*))") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [f a] +// Tag: !!int, Kind: ScalarNode, Anchor: +// 3 +// -- Node -- +// Document 0, path: [f c] +// Tag: !!int, Kind: ScalarNode, Anchor: +// 5 +// ` + +// test.AssertResult(t, expected, resultsToString(results)) +// } + +// func TestDataTreeNavigatorCountOfResults(t *testing.T) { + +// nodes := readDoc(t, `- apple +// - sdfsd +// - apple`) + +// path, errPath := treeCreator.ParsePath("count(*)") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [] +// Tag: !!int, Kind: ScalarNode, Anchor: +// 3 +// ` + +// test.AssertResult(t, expected, resultsToString(results)) +// } + +// func TestDataTreeNavigatorCountNoMatches(t *testing.T) { + +// nodes := readDoc(t, `- apple +// - sdfsd +// - apple`) + +// path, errPath := treeCreator.ParsePath("count(5)") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [] +// Tag: !!int, Kind: ScalarNode, Anchor: +// 0 +// ` + +// test.AssertResult(t, expected, resultsToString(results)) +// } + +// func TestDataTreeNavigatorDeleteAndWrite(t *testing.T) { + +// nodes := readDoc(t, `a: +// - b: apple +// - b: sdfsd +// - { b: apple, c: cat }`) + +// path, errPath := treeCreator.ParsePath("(a .- (0 or 1)) or (a[0].b := frog)") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [a] +// Tag: !!seq, Kind: SequenceNode, Anchor: +// - {b: frog, c: cat} + +// -- Node -- +// Document 0, path: [a 0 b] +// Tag: !!str, Kind: ScalarNode, Anchor: +// frog +// ` + +// test.AssertResult(t, expected, resultsToString(results)) +// } + +// func TestDataTreeNavigatorDeleteArray(t *testing.T) { + +// nodes := readDoc(t, `a: +// - b: apple +// - b: sdfsd +// - b: apple`) + +// path, errPath := treeCreator.ParsePath("a .- (b == a*)") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [a] +// Tag: !!seq, Kind: SequenceNode, Anchor: +// - b: sdfsd +// ` + +// test.AssertResult(t, expected, resultsToString(results)) +// } + +// func TestDataTreeNavigatorArraySimple(t *testing.T) { + +// nodes := readDoc(t, `- b: apple`) + +// path, errPath := treeCreator.ParsePath("[0]") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [0] +// Tag: !!map, Kind: MappingNode, Anchor: +// b: apple +// ` + +// test.AssertResult(t, expected, resultsToString(results)) +// } + +// func TestDataTreeNavigatorSimpleAssignByFind(t *testing.T) { + +// nodes := readDoc(t, `a: +// b: apple`) + +// path, errPath := treeCreator.ParsePath("a(. == apple) := frog") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [a b] +// Tag: !!str, Kind: ScalarNode, Anchor: +// frog +// ` + +// test.AssertResult(t, expected, resultsToString(results)) +// } + +// func TestDataTreeNavigatorOrDeDupes(t *testing.T) { + +// nodes := readDoc(t, `a: +// cat: apple +// mad: things`) + +// path, errPath := treeCreator.ParsePath("a.(cat or cat)") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [a cat] +// Tag: !!str, Kind: ScalarNode, Anchor: +// apple +// ` + +// test.AssertResult(t, expected, resultsToString(results)) +// } + +// func TestDataTreeNavigatorAnd(t *testing.T) { + +// nodes := readDoc(t, `a: +// cat: apple +// pat: apple +// cow: apple +// mad: things`) + +// path, errPath := treeCreator.ParsePath("a.(*t and c*)") +// if errPath != nil { +// t.Error(errPath) +// } +// results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + +// if errNav != nil { +// t.Error(errNav) +// } + +// expected := ` +// -- Node -- +// Document 0, path: [a cat] +// Tag: !!str, Kind: ScalarNode, Anchor: +// apple +// ` + +// test.AssertResult(t, expected, resultsToString(results)) +// } diff --git a/pkg/yqlib/treeops/lib.go b/pkg/yqlib/treeops/lib.go index 28c2f459..b60d055e 100644 --- a/pkg/yqlib/treeops/lib.go +++ b/pkg/yqlib/treeops/lib.go @@ -36,8 +36,8 @@ var Collect = &OperationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handle var TraversePath = &OperationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator} var DocumentFilter = &OperationType{Type: "DOCUMENT_FILTER", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator} -var SelfReference = &OperationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator} -var ValueOp = &OperationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator} +var SelfReference = &OperationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: SelfOperator} +var ValueOp = &OperationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: ValueOperator} var RecursiveDescent = &OperationType{Type: "RECURSIVE_DESCENT", NumArgs: 0, Precedence: 50, Handler: RecursiveDescentOperator} @@ -52,15 +52,38 @@ var DeleteChild = &OperationType{Type: "DELETE", NumArgs: 2, Precedence: 40, Han // var Exists = &OperationType{Type: "Length", NumArgs: 2, Precedence: 35} // filters matches if they have the existing path -type PathElement struct { +type Operation struct { OperationType *OperationType Value interface{} StringValue string CandidateNode *CandidateNode // used for Value Path elements } +func CreateValueOperation(value interface{}, stringValue string) *Operation { + var node yaml.Node = yaml.Node{Kind: yaml.ScalarNode} + node.Value = stringValue + + switch value.(type) { + case float32, float64: + node.Tag = "!!float" + case int, int64, int32: + node.Tag = "!!int" + case bool: + node.Tag = "!!bool" + case string: + node.Tag = "!!str" + } + + return &Operation{ + OperationType: ValueOp, + Value: value, + StringValue: stringValue, + CandidateNode: &CandidateNode{Node: &node}, + } +} + // debugging purposes only -func (p *PathElement) toString() string { +func (p *Operation) toString() string { if p.OperationType == TraversePath { return fmt.Sprintf("%v", p.Value) } else if p.OperationType == DocumentFilter { diff --git a/pkg/yqlib/treeops/operator_equals.go b/pkg/yqlib/treeops/operator_equals.go index 18dfa1c6..a2a4aa2d 100644 --- a/pkg/yqlib/treeops/operator_equals.go +++ b/pkg/yqlib/treeops/operator_equals.go @@ -34,7 +34,7 @@ func hasMatch(d *dataTreeNavigator, candidate *CandidateNode, lhs *PathTreeNode, } // TODO = handle other RHS types - return containsMatchingValue(childMatches, rhs.PathElement.StringValue), nil + return containsMatchingValue(childMatches, rhs.Operation.StringValue), nil } func containsMatchingValue(matchMap *orderedmap.OrderedMap, valuePattern string) bool { diff --git a/pkg/yqlib/treeops/operator_multilpy.go b/pkg/yqlib/treeops/operator_multilpy.go index 57b73299..e0efff67 100644 --- a/pkg/yqlib/treeops/operator_multilpy.go +++ b/pkg/yqlib/treeops/operator_multilpy.go @@ -61,12 +61,12 @@ func multiply(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*Ca func createTraversalTree(path []interface{}) *PathTreeNode { if len(path) == 0 { - return &PathTreeNode{PathElement: &PathElement{OperationType: SelfReference}} + return &PathTreeNode{Operation: &Operation{OperationType: SelfReference}} } else if len(path) == 1 { - return &PathTreeNode{PathElement: &PathElement{OperationType: TraversePath, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}} + return &PathTreeNode{Operation: &Operation{OperationType: TraversePath, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}} } return &PathTreeNode{ - PathElement: &PathElement{OperationType: Pipe}, + Operation: &Operation{OperationType: Pipe}, Lhs: createTraversalTree(path[0:1]), Rhs: createTraversalTree(path[1:])} @@ -77,13 +77,13 @@ func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *Candid lhsPath := rhs.Path[pathIndexToStartFrom:] - assignmentOp := &PathElement{OperationType: AssignAttributes} + assignmentOp := &Operation{OperationType: AssignAttributes} if rhs.Node.Kind == yaml.ScalarNode { assignmentOp.OperationType = Assign } - rhsOp := &PathElement{OperationType: ValueOp, CandidateNode: rhs} + rhsOp := &Operation{OperationType: ValueOp, CandidateNode: rhs} - assignmentOpNode := &PathTreeNode{PathElement: assignmentOp, Lhs: createTraversalTree(lhsPath), Rhs: &PathTreeNode{PathElement: rhsOp}} + assignmentOpNode := &PathTreeNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath), Rhs: &PathTreeNode{Operation: rhsOp}} _, err := d.getMatchingNodes(nodeToMap(lhs), assignmentOpNode) diff --git a/pkg/yqlib/treeops/operator_recursive_descent.go b/pkg/yqlib/treeops/operator_recursive_descent.go index 583eb9f9..38513b7a 100644 --- a/pkg/yqlib/treeops/operator_recursive_descent.go +++ b/pkg/yqlib/treeops/operator_recursive_descent.go @@ -16,8 +16,8 @@ func RecursiveDescentOperator(d *dataTreeNavigator, matchMap *orderedmap.Ordered } func recursiveDecent(d *dataTreeNavigator, results *orderedmap.OrderedMap, matchMap *orderedmap.OrderedMap) error { - splatPathElement := &PathElement{OperationType: TraversePath, Value: "[]"} - splatTreeNode := &PathTreeNode{PathElement: splatPathElement} + splatOperation := &Operation{OperationType: TraversePath, Value: "[]"} + splatTreeNode := &PathTreeNode{Operation: splatOperation} for el := matchMap.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) diff --git a/pkg/yqlib/treeops/operator_self.go b/pkg/yqlib/treeops/operator_self.go new file mode 100644 index 00000000..170ef2c6 --- /dev/null +++ b/pkg/yqlib/treeops/operator_self.go @@ -0,0 +1,7 @@ +package treeops + +import "github.com/elliotchance/orderedmap" + +func SelfOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { + return matchMap, nil +} diff --git a/pkg/yqlib/treeops/operator_traverse_path.go b/pkg/yqlib/treeops/operator_traverse_path.go index 14803d72..47e396df 100644 --- a/pkg/yqlib/treeops/operator_traverse_path.go +++ b/pkg/yqlib/treeops/operator_traverse_path.go @@ -14,7 +14,7 @@ func TraversePathOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, var err error for el := matchMap.Front(); el != nil; el = el.Next() { - newNodes, err = traverse(d, el.Value.(*CandidateNode), pathNode.PathElement) + newNodes, err = traverse(d, el.Value.(*CandidateNode), pathNode.Operation) if err != nil { return nil, err } @@ -26,7 +26,7 @@ func TraversePathOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, return matchingNodeMap, nil } -func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) { +func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, pathNode *Operation) ([]*CandidateNode, error) { log.Debug("Traversing %v", NodeToString(matchingNode)) value := matchingNode.Node @@ -78,11 +78,11 @@ func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, pathNode *PathE } } -func keyMatches(key *yaml.Node, pathNode *PathElement) bool { +func keyMatches(key *yaml.Node, pathNode *Operation) bool { return pathNode.Value == "[]" || Match(key.Value, pathNode.StringValue) } -func traverseMap(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) { +func traverseMap(candidate *CandidateNode, pathNode *Operation) ([]*CandidateNode, error) { // value.Content is a concatenated array of key, value, // so keys are in the even indexes, values in odd. // merge aliases are defined first, but we only want to traverse them @@ -123,7 +123,7 @@ func traverseMap(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateN return newMatches, nil } -func traverseArray(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) { +func traverseArray(candidate *CandidateNode, pathNode *Operation) ([]*CandidateNode, error) { log.Debug("pathNode Value %v", pathNode.Value) if pathNode.Value == "[]" { diff --git a/pkg/yqlib/treeops/operator_traverse_path_test.go b/pkg/yqlib/treeops/operator_traverse_path_test.go index 14217970..4199d0a2 100644 --- a/pkg/yqlib/treeops/operator_traverse_path_test.go +++ b/pkg/yqlib/treeops/operator_traverse_path_test.go @@ -12,6 +12,60 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[a], (!!map)::{b: apple}\n", }, }, + { + document: `[{b: apple}, {c: banana}]`, + expression: `.[]`, + expected: []string{ + "D0, P[0], (!!map)::{b: apple}\n", + "D0, P[1], (!!map)::{c: banana}\n", + }, + }, + { + document: `{}`, + expression: `.a.b`, + expected: []string{ + "D0, P[a b], ()::null\n", + }, + }, + { + document: `{}`, + expression: `.[1].a`, + expected: []string{ + "D0, P[1 a], ()::null\n", + }, + }, + { + document: `{}`, + expression: `.a.[1]`, + expected: []string{ + "D0, P[a 1], ()::null\n", + }, + }, + { + document: `{a: {cat: apple, mad: things}}`, + expression: `.a."*a*"`, + expected: []string{ + "D0, P[a cat], (!!str)::apple\n", + "D0, P[a mad], (!!str)::things\n", + }, + }, + { + document: `{a: {cat: {b: 3}, mad: {b: 4}, fad: {c: t}}}`, + expression: `.a."*a*".b`, + expected: []string{ + "D0, P[a cat b], (!!int)::3\n", + "D0, P[a mad b], (!!int)::4\n", + "D0, P[a fad b], ()::null\n", + }, + }, + { + document: `{a: {cat: apple, mad: things}}`, + expression: `.a | (.cat, .mad)`, + expected: []string{ + "D0, P[a cat], (!!str)::apple\n", + "D0, P[a mad], (!!str)::things\n", + }, + }, } func TestTraversePathOperatorScenarios(t *testing.T) { diff --git a/pkg/yqlib/treeops/operator_value.go b/pkg/yqlib/treeops/operator_value.go new file mode 100644 index 00000000..caabd354 --- /dev/null +++ b/pkg/yqlib/treeops/operator_value.go @@ -0,0 +1,7 @@ +package treeops + +import "github.com/elliotchance/orderedmap" + +func ValueOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { + return nodeToMap(pathNode.Operation.CandidateNode), nil +} diff --git a/pkg/yqlib/treeops/operator_value_test.go b/pkg/yqlib/treeops/operator_value_test.go index b722a4b4..c7512f61 100644 --- a/pkg/yqlib/treeops/operator_value_test.go +++ b/pkg/yqlib/treeops/operator_value_test.go @@ -11,7 +11,8 @@ var valueOperatorScenarios = []expressionScenario{ expected: []string{ "D0, P[], (!!int)::1\n", }, - }, { + }, + { document: ``, expression: `-1`, expected: []string{ diff --git a/pkg/yqlib/treeops/path_parse_test.go b/pkg/yqlib/treeops/path_parse_test.go index 7e2fe345..16bcf245 100644 --- a/pkg/yqlib/treeops/path_parse_test.go +++ b/pkg/yqlib/treeops/path_parse_test.go @@ -41,22 +41,22 @@ var pathTests = []struct { // {`.[].a`, append(make([]interface{}, 0), "[]", "PIPE", "a")}, { `d0.a`, - append(make([]interface{}, 0), int64(0), "PIPE", "a"), - append(make([]interface{}, 0), "D0", "a", "PIPE"), + append(make([]interface{}, 0), "d0", "PIPE", "a"), + append(make([]interface{}, 0), "d0", "a", "PIPE"), }, { `.a | (.[].b == "apple")`, - append(make([]interface{}, 0), "a", "PIPE", "(", "[]", "PIPE", "b", "EQUALS", "apple", ")"), + append(make([]interface{}, 0), "a", "PIPE", "(", "[]", "PIPE", "b", "EQUALS", "apple (string)", ")"), append(make([]interface{}, 0), "a", "[]", "b", "PIPE", "apple (string)", "EQUALS", "PIPE"), }, { `.[] | select(. == "*at")`, - append(make([]interface{}, 0), "[]", "PIPE", "SELECT", "(", "SELF", "EQUALS", "*at", ")"), + append(make([]interface{}, 0), "[]", "PIPE", "SELECT", "(", "SELF", "EQUALS", "*at (string)", ")"), append(make([]interface{}, 0), "[]", "SELF", "*at (string)", "EQUALS", "SELECT", "PIPE"), }, { `[true]`, - append(make([]interface{}, 0), "[", true, "]"), + append(make([]interface{}, 0), "[", "true (bool)", "]"), append(make([]interface{}, 0), "true (bool)", "COLLECT", "PIPE"), }, @@ -91,7 +91,7 @@ func TestPathParsing(t *testing.T) { } var tokenValues []interface{} for _, token := range tokens { - tokenValues = append(tokenValues, token.Value) + tokenValues = append(tokenValues, token.toString()) } test.AssertResultComplexWithContext(t, tt.expectedTokens, tokenValues, fmt.Sprintf("tokenise: %v", tt.path)) diff --git a/pkg/yqlib/treeops/path_postfix.go b/pkg/yqlib/treeops/path_postfix.go index 934f22bb..850307d4 100644 --- a/pkg/yqlib/treeops/path_postfix.go +++ b/pkg/yqlib/treeops/path_postfix.go @@ -7,7 +7,7 @@ import ( ) type PathPostFixer interface { - ConvertToPostfix([]*Token) ([]*PathElement, error) + ConvertToPostfix([]*Token) ([]*Operation, error) } type pathPostFixer struct { @@ -17,27 +17,20 @@ func NewPathPostFixer() PathPostFixer { return &pathPostFixer{} } -func popOpToResult(opStack []*Token, result []*PathElement) ([]*Token, []*PathElement) { +func popOpToResult(opStack []*Token, result []*Operation) ([]*Token, []*Operation) { var newOp *Token opStack, newOp = opStack[0:len(opStack)-1], opStack[len(opStack)-1] - var pathElement = PathElement{OperationType: newOp.OperationType, Value: newOp.Value, StringValue: newOp.StringValue} - - if newOp.OperationType == ValueOp { - var candidateNode = BuildCandidateNodeFrom(newOp) - pathElement.CandidateNode = candidateNode - } - - return opStack, append(result, &pathElement) + return opStack, append(result, newOp.Operation) } -func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*PathElement, error) { - var result []*PathElement +func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*Operation, error) { + var result []*Operation // surround the whole thing with quotes - var opStack = []*Token{&Token{TokenType: OpenBracket, Value: "("}} - var tokens = append(infixTokens, &Token{TokenType: CloseBracket, Value: ")"}) + var opStack = []*Token{&Token{TokenType: OpenBracket}} + var tokens = append(infixTokens, &Token{TokenType: CloseBracket}) for _, token := range tokens { - log.Debugf("postfix processing token %v", token.Value) + log.Debugf("postfix processing token %v, %v", token.toString(), token.Operation) switch token.TokenType { case OpenBracket, OpenCollect: opStack = append(opStack, token) @@ -51,8 +44,8 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*PathElement, // now we should have [] as the last element on the opStack, get rid of it opStack = opStack[0 : len(opStack)-1] //and append a collect to the opStack - opStack = append(opStack, &Token{TokenType: Operation, OperationType: Pipe}) - opStack = append(opStack, &Token{TokenType: Operation, OperationType: Collect}) + opStack = append(opStack, &Token{TokenType: OperationToken, Operation: &Operation{OperationType: Pipe}}) + opStack = append(opStack, &Token{TokenType: OperationToken, Operation: &Operation{OperationType: Collect}}) case CloseBracket: for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != OpenBracket { opStack, result = popOpToResult(opStack, result) @@ -64,9 +57,11 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*PathElement, opStack = opStack[0 : len(opStack)-1] default: - var currentPrecedence = token.OperationType.Precedence + var currentPrecedence = token.Operation.OperationType.Precedence // pop off higher precedent operators onto the result - for len(opStack) > 0 && opStack[len(opStack)-1].OperationType.Precedence >= currentPrecedence { + for len(opStack) > 0 && + opStack[len(opStack)-1].TokenType == OperationToken && + opStack[len(opStack)-1].Operation.OperationType.Precedence >= currentPrecedence { opStack, result = popOpToResult(opStack, result) } // add this operator to the opStack diff --git a/pkg/yqlib/treeops/path_tokeniser.go b/pkg/yqlib/treeops/path_tokeniser.go index 88384dfe..ca302f78 100644 --- a/pkg/yqlib/treeops/path_tokeniser.go +++ b/pkg/yqlib/treeops/path_tokeniser.go @@ -1,6 +1,7 @@ package treeops import ( + "fmt" "strconv" lex "github.com/timtadh/lexmachine" @@ -14,7 +15,7 @@ func skip(*lex.Scanner, *machines.Match) (interface{}, error) { type TokenType uint32 const ( - Operation = 1 << iota + OperationToken = 1 << iota OpenBracket CloseBracket OpenCollect @@ -22,14 +23,28 @@ const ( ) type Token struct { - TokenType TokenType - OperationType *OperationType - Value interface{} - StringValue string + TokenType TokenType + Operation *Operation CheckForPostTraverse bool // e.g. [1]cat should really be [1].cat } +func (t *Token) toString() string { + if t.TokenType == OperationToken { + return t.Operation.toString() + } else if t.TokenType == OpenBracket { + return "(" + } else if t.TokenType == CloseBracket { + return ")" + } else if t.TokenType == OpenCollect { + return "[" + } else if t.TokenType == CloseCollect { + return "]" + } else { + return fmt.Sprintf("NFI") + } +} + func pathToken(wrapped bool) lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { value := string(m.Bytes) @@ -37,13 +52,15 @@ func pathToken(wrapped bool) lex.Action { if wrapped { value = unwrap(value) } - return &Token{TokenType: Operation, OperationType: TraversePath, Value: value, StringValue: value, CheckForPostTraverse: true}, nil + op := &Operation{OperationType: TraversePath, Value: value, StringValue: value} + return &Token{TokenType: OperationToken, Operation: op, CheckForPostTraverse: true}, nil } } func literalPathToken(value string) lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { - return &Token{TokenType: Operation, OperationType: TraversePath, Value: value, StringValue: value, CheckForPostTraverse: true}, nil + op := &Operation{OperationType: TraversePath, Value: value, StringValue: value} + return &Token{TokenType: OperationToken, Operation: op, CheckForPostTraverse: true}, nil } } @@ -55,20 +72,22 @@ func documentToken() lex.Action { if errParsingInt != nil { return nil, errParsingInt } - return &Token{TokenType: Operation, OperationType: DocumentFilter, Value: number, StringValue: numberString, CheckForPostTraverse: true}, nil + op := &Operation{OperationType: DocumentFilter, Value: number, StringValue: numberString} + return &Token{TokenType: OperationToken, Operation: op, CheckForPostTraverse: true}, nil } } func opToken(op *OperationType) lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { value := string(m.Bytes) - return &Token{TokenType: Operation, OperationType: op, Value: op.Type, StringValue: value}, nil + op := &Operation{OperationType: op, Value: op.Type, StringValue: value} + return &Token{TokenType: OperationToken, Operation: op}, nil } } -func literalToken(pType TokenType, literal string, checkForPost bool) lex.Action { +func literalToken(pType TokenType, checkForPost bool) lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { - return &Token{TokenType: Operation, OperationType: ValueOp, Value: literal, StringValue: literal, CheckForPostTraverse: checkForPost}, nil + return &Token{TokenType: pType, CheckForPostTraverse: checkForPost}, nil } } @@ -88,7 +107,8 @@ func arrayIndextoken(precedingDot bool) lex.Action { if errParsingInt != nil { return nil, errParsingInt } - return &Token{TokenType: Operation, OperationType: TraversePath, Value: number, StringValue: numberString, CheckForPostTraverse: true}, nil + op := &Operation{OperationType: TraversePath, Value: number, StringValue: numberString} + return &Token{TokenType: OperationToken, Operation: op, CheckForPostTraverse: true}, nil } } @@ -99,7 +119,8 @@ func numberValue() lex.Action { if errParsingInt != nil { return nil, errParsingInt } - return &Token{TokenType: Operation, OperationType: ValueOp, Value: number, StringValue: numberString}, nil + + return &Token{TokenType: OperationToken, Operation: CreateValueOperation(number, numberString)}, nil } } @@ -110,13 +131,13 @@ func floatValue() lex.Action { if errParsingInt != nil { return nil, errParsingInt } - return &Token{TokenType: Operation, OperationType: ValueOp, Value: number, StringValue: numberString}, nil + return &Token{TokenType: OperationToken, Operation: CreateValueOperation(number, numberString)}, nil } } func booleanValue(val bool) lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { - return &Token{TokenType: Operation, OperationType: ValueOp, Value: val, StringValue: string(m.Bytes)}, nil + return &Token{TokenType: OperationToken, Operation: CreateValueOperation(val, string(m.Bytes))}, nil } } @@ -126,21 +147,22 @@ func stringValue(wrapped bool) lex.Action { if wrapped { value = unwrap(value) } - return &Token{TokenType: Operation, OperationType: ValueOp, Value: value, StringValue: value}, nil + return &Token{TokenType: OperationToken, Operation: CreateValueOperation(value, value)}, nil } } func selfToken() lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { - return &Token{TokenType: Operation, OperationType: SelfReference}, nil + op := &Operation{OperationType: SelfReference} + return &Token{TokenType: OperationToken, Operation: op}, nil } } // Creates the lexer object and compiles the NFA. func initLexer() (*lex.Lexer, error) { lexer := lex.NewLexer() - lexer.Add([]byte(`\(`), literalToken(OpenBracket, "(", false)) - lexer.Add([]byte(`\)`), literalToken(CloseBracket, ")", true)) + lexer.Add([]byte(`\(`), literalToken(OpenBracket, false)) + lexer.Add([]byte(`\)`), literalToken(CloseBracket, true)) lexer.Add([]byte(`\.?\[\]`), literalPathToken("[]")) lexer.Add([]byte(`\.\.`), opToken(RecursiveDescent)) @@ -180,8 +202,8 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`"[^ "]+"`), stringValue(true)) - lexer.Add([]byte(`\[`), literalToken(OpenCollect, "[", false)) - lexer.Add([]byte(`\]`), literalToken(CloseCollect, "]", true)) + lexer.Add([]byte(`\[`), literalToken(OpenCollect, false)) + lexer.Add([]byte(`\]`), literalToken(CloseCollect, true)) lexer.Add([]byte(`\*`), opToken(Multiply)) // lexer.Add([]byte(`[^ \,\|\.\[\(\)=]+`), stringValue(false)) @@ -219,7 +241,7 @@ func (p *pathTokeniser) Tokenise(path string) ([]*Token, error) { if tok != nil { token := tok.(*Token) - log.Debugf("Tokenising %v - %v", token.Value, token.OperationType.Type) + log.Debugf("Tokenising %v", token.toString()) tokens = append(tokens, token) } if err != nil { @@ -233,8 +255,10 @@ func (p *pathTokeniser) Tokenise(path string) ([]*Token, error) { postProcessedTokens = append(postProcessedTokens, token) if index != len(tokens)-1 && token.CheckForPostTraverse && - tokens[index+1].OperationType == TraversePath { - postProcessedTokens = append(postProcessedTokens, &Token{TokenType: Operation, OperationType: Pipe, Value: "PIPE"}) + tokens[index+1].TokenType == OperationToken && + tokens[index+1].Operation.OperationType == TraversePath { + op := &Operation{OperationType: Pipe, Value: "PIPE"} + postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op}) } } diff --git a/pkg/yqlib/treeops/path_tree.go b/pkg/yqlib/treeops/path_tree.go index e07cfa7c..bee15ff8 100644 --- a/pkg/yqlib/treeops/path_tree.go +++ b/pkg/yqlib/treeops/path_tree.go @@ -6,14 +6,14 @@ var myPathTokeniser = NewPathTokeniser() var myPathPostfixer = NewPathPostFixer() type PathTreeNode struct { - PathElement *PathElement + Operation *Operation Lhs *PathTreeNode Rhs *PathTreeNode } type PathTreeCreator interface { ParsePath(path string) (*PathTreeNode, error) - CreatePathTree(postFixPath []*PathElement) (*PathTreeNode, error) + CreatePathTree(postFixPath []*Operation) (*PathTreeNode, error) } type pathTreeCreator struct { @@ -28,26 +28,26 @@ func (p *pathTreeCreator) ParsePath(path string) (*PathTreeNode, error) { if err != nil { return nil, err } - var pathElements []*PathElement - pathElements, err = myPathPostfixer.ConvertToPostfix(tokens) + var Operations []*Operation + Operations, err = myPathPostfixer.ConvertToPostfix(tokens) if err != nil { return nil, err } - return p.CreatePathTree(pathElements) + return p.CreatePathTree(Operations) } -func (p *pathTreeCreator) CreatePathTree(postFixPath []*PathElement) (*PathTreeNode, error) { +func (p *pathTreeCreator) CreatePathTree(postFixPath []*Operation) (*PathTreeNode, error) { var stack = make([]*PathTreeNode, 0) if len(postFixPath) == 0 { return nil, nil } - for _, pathElement := range postFixPath { - var newNode = PathTreeNode{PathElement: pathElement} - log.Debugf("pathTree %v ", pathElement.toString()) - if pathElement.OperationType.NumArgs > 0 { - numArgs := pathElement.OperationType.NumArgs + for _, Operation := range postFixPath { + var newNode = PathTreeNode{Operation: Operation} + log.Debugf("pathTree %v ", Operation.toString()) + if Operation.OperationType.NumArgs > 0 { + numArgs := Operation.OperationType.NumArgs if numArgs == 1 { remaining, rhs := stack[:len(stack)-1], stack[len(stack)-1] newNode.Rhs = rhs diff --git a/pkg/yqlib/treeops/sample.yaml b/pkg/yqlib/treeops/sample.yaml index 7997a4d4..dcee3caf 100644 --- a/pkg/yqlib/treeops/sample.yaml +++ b/pkg/yqlib/treeops/sample.yaml @@ -1 +1 @@ -{a: {b: apple, c: cactus}} \ No newline at end of file +{a: {cat: apple, mad: things}} \ No newline at end of file diff --git a/pkg/yqlib/treeops/temp.json b/pkg/yqlib/treeops/temp.json index 7d768a0b..3ce4e779 100644 --- a/pkg/yqlib/treeops/temp.json +++ b/pkg/yqlib/treeops/temp.json @@ -1,4 +1,6 @@ { - "a": [1,2], - "b": [3,4] -} \ No newline at end of file + "a": { + "cat": "apple", + "mad": "things" + } +} diff --git a/pkg/yqlib/treeops/value_node_builder.go b/pkg/yqlib/treeops/value_node_builder.go deleted file mode 100644 index 7c3e7d21..00000000 --- a/pkg/yqlib/treeops/value_node_builder.go +++ /dev/null @@ -1,20 +0,0 @@ -package treeops - -import "gopkg.in/yaml.v3" - -func BuildCandidateNodeFrom(token *Token) *CandidateNode { - var node yaml.Node = yaml.Node{Kind: yaml.ScalarNode} - node.Value = token.StringValue - - switch token.Value.(type) { - case float32, float64: - node.Tag = "!!float" - case int, int64, int32: - node.Tag = "!!int" - case bool: - node.Tag = "!!bool" - case string: - node.Tag = "!!str" - } - return &CandidateNode{Node: &node} -}