From f479a7e8e3d28fe149dbf565994d531269096cf9 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 9 Oct 2020 10:59:03 +1100 Subject: [PATCH] wip --- pkg/yqlib/data_tree_navigator.go | 73 ---------- pkg/yqlib/data_tree_navigator_test.go | 1 - pkg/yqlib/path_tree_test.go | 1 - pkg/yqlib/treeops/data_tree_navigator.go | 75 +++++++++++ pkg/yqlib/treeops/data_tree_navigator_test.go | 125 ++++++++++++++++++ pkg/yqlib/treeops/lib.go | 59 +++++++++ pkg/yqlib/treeops/matchKeyString.go | 34 +++++ pkg/yqlib/{ => treeops}/path_postfix.go | 2 +- pkg/yqlib/{ => treeops}/path_postfix_test.go | 2 +- pkg/yqlib/{ => treeops}/path_tokeniser.go | 2 +- .../{ => treeops}/path_tokeniser_test.go | 2 +- pkg/yqlib/{ => treeops}/path_tree.go | 21 ++- pkg/yqlib/treeops/path_tree_test.go | 1 + pkg/yqlib/treeops/traverse.go | 94 +++++++++++++ 14 files changed, 411 insertions(+), 81 deletions(-) delete mode 100644 pkg/yqlib/data_tree_navigator.go delete mode 100644 pkg/yqlib/data_tree_navigator_test.go delete mode 100644 pkg/yqlib/path_tree_test.go create mode 100644 pkg/yqlib/treeops/data_tree_navigator.go create mode 100644 pkg/yqlib/treeops/data_tree_navigator_test.go create mode 100644 pkg/yqlib/treeops/lib.go create mode 100644 pkg/yqlib/treeops/matchKeyString.go rename pkg/yqlib/{ => treeops}/path_postfix.go (99%) rename pkg/yqlib/{ => treeops}/path_postfix_test.go (99%) rename pkg/yqlib/{ => treeops}/path_tokeniser.go (99%) rename pkg/yqlib/{ => treeops}/path_tokeniser_test.go (99%) rename pkg/yqlib/{ => treeops}/path_tree.go (61%) create mode 100644 pkg/yqlib/treeops/path_tree_test.go create mode 100644 pkg/yqlib/treeops/traverse.go diff --git a/pkg/yqlib/data_tree_navigator.go b/pkg/yqlib/data_tree_navigator.go deleted file mode 100644 index b67ea409..00000000 --- a/pkg/yqlib/data_tree_navigator.go +++ /dev/null @@ -1,73 +0,0 @@ -package yqlib - -type dataTreeNavigator struct { -} - -type DataTreeNavigator interface { - GetMatchingNodes(matchingNodes []*NodeContext, pathNode *PathTreeNode) ([]*NodeContext, error) -} - -func NewTreeNavigator() DataTreeNavigator { - return &dataTreeNavigator{} -} - -func (d *dataTreeNavigator) traverseSingle(matchingNode *NodeContext, pathNode *PathElement) ([]*NodeContext, error) { - var value = matchingNode.Node - // match all for splat - // match all and recurse for deep - // etc and so forth - -} - -func (d *dataTreeNavigator) traverse(matchingNodes []*NodeContext, pathNode *PathElement) ([]*NodeContext, error) { - var newMatchingNodes = make([]*NodeContext, 0) - var newNodes []*NodeContext - var err error - for _, node := range matchingNodes { - - newNodes, err = d.traverseSingle(node, pathNode) - if err != nil { - return nil, err - } - newMatchingNodes = append(newMatchingNodes, newNodes...) - } - - return newMatchingNodes, nil -} - -func (d *dataTreeNavigator) GetMatchingNodes(matchingNodes []*NodeContext, pathNode *PathTreeNode) ([]*NodeContext, error) { - if pathNode.PathElement.PathElementType == PathKey || pathNode.PathElement.PathElementType == ArrayIndex { - return d.traverse(matchingNodes, pathNode.PathElement) - } else { - var lhs, rhs []*NodeContext - var err error - switch pathNode.PathElement.OperationType { - case Traverse: - lhs, err = d.GetMatchingNodes(matchingNodes, pathNode.Lhs) - if err != nil { - return nil, err - } - return d.GetMatchingNodes(lhs, pathNode.Rhs) - case Or, And: - lhs, err = d.GetMatchingNodes(matchingNodes, pathNode.Lhs) - if err != nil { - return nil, err - } - rhs, err = d.GetMatchingNodes(matchingNodes, pathNode.Rhs) - if err != nil { - return nil, err - } - return d.setFunction(pathNode.PathElement, lhs, rhs), nil - case Equals: - lhs, err = d.GetMatchingNodes(matchingNodes, pathNode.Lhs) - if err != nil { - return nil, err - } - return d.findMatchingValues(lhs, pathNode.Rhs) - case EqualsSelf: - return d.findMatchingValues(matchingNodes, pathNode.Rhs) - } - - } - -} diff --git a/pkg/yqlib/data_tree_navigator_test.go b/pkg/yqlib/data_tree_navigator_test.go deleted file mode 100644 index 88c44e97..00000000 --- a/pkg/yqlib/data_tree_navigator_test.go +++ /dev/null @@ -1 +0,0 @@ -package yqlib diff --git a/pkg/yqlib/path_tree_test.go b/pkg/yqlib/path_tree_test.go deleted file mode 100644 index 88c44e97..00000000 --- a/pkg/yqlib/path_tree_test.go +++ /dev/null @@ -1 +0,0 @@ -package yqlib diff --git a/pkg/yqlib/treeops/data_tree_navigator.go b/pkg/yqlib/treeops/data_tree_navigator.go new file mode 100644 index 00000000..1b14490e --- /dev/null +++ b/pkg/yqlib/treeops/data_tree_navigator.go @@ -0,0 +1,75 @@ +package treeops + +type dataTreeNavigator struct { + traverser Traverser +} + +type NavigationPrefs struct { + FollowAlias bool +} + +type DataTreeNavigator interface { + GetMatchingNodes(matchingNodes []*CandidateNode, pathNode *PathTreeNode) ([]*CandidateNode, error) +} + +func NewDataTreeNavigator(navigationPrefs NavigationPrefs) DataTreeNavigator { + traverse := NewTraverser(navigationPrefs) + return &dataTreeNavigator{traverse} +} + +func (d *dataTreeNavigator) traverse(matchingNodes []*CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) { + log.Debugf("-- Traversing") + var newMatchingNodes = make([]*CandidateNode, 0) + var newNodes []*CandidateNode + var err error + for _, node := range matchingNodes { + + newNodes, err = d.traverser.Traverse(node, pathNode) + if err != nil { + return nil, err + } + newMatchingNodes = append(newMatchingNodes, newNodes...) + } + + return newMatchingNodes, nil +} + +func (d *dataTreeNavigator) GetMatchingNodes(matchingNodes []*CandidateNode, pathNode *PathTreeNode) ([]*CandidateNode, error) { + log.Debugf("Processing Path: %v", pathNode.PathElement.toString()) + if pathNode.PathElement.PathElementType == PathKey || pathNode.PathElement.PathElementType == ArrayIndex { + return d.traverse(matchingNodes, pathNode.PathElement) + } else { + var lhs []*CandidateNode //, rhs + var err error + switch pathNode.PathElement.OperationType { + case Traverse: + lhs, err = d.GetMatchingNodes(matchingNodes, pathNode.Lhs) + if err != nil { + return nil, err + } + return d.GetMatchingNodes(lhs, pathNode.Rhs) + // case Or, And: + // lhs, err = d.GetMatchingNodes(matchingNodes, pathNode.Lhs) + // if err != nil { + // return nil, err + // } + // rhs, err = d.GetMatchingNodes(matchingNodes, pathNode.Rhs) + // if err != nil { + // return nil, err + // } + // return d.setFunction(pathNode.PathElement, lhs, rhs), nil + // case Equals: + // lhs, err = d.GetMatchingNodes(matchingNodes, pathNode.Lhs) + // if err != nil { + // return nil, err + // } + // return d.findMatchingValues(lhs, pathNode.Rhs) + // case EqualsSelf: + // return d.findMatchingValues(matchingNodes, pathNode.Rhs) + default: + return nil, nil + } + + } + +} diff --git a/pkg/yqlib/treeops/data_tree_navigator_test.go b/pkg/yqlib/treeops/data_tree_navigator_test.go new file mode 100644 index 00000000..0a377379 --- /dev/null +++ b/pkg/yqlib/treeops/data_tree_navigator_test.go @@ -0,0 +1,125 @@ +package treeops + +import ( + "strings" + "testing" + + "github.com/mikefarah/yq/v3/test" + yaml "gopkg.in/yaml.v3" +) + +var treeNavigator = NewDataTreeNavigator(NavigationPrefs{}) +var treeCreator = NewPathTreeCreator() + +func readDoc(t *testing.T, content string) []*CandidateNode { + decoder := yaml.NewDecoder(strings.NewReader(content)) + var dataBucket yaml.Node + err := decoder.Decode(&dataBucket) + if err != nil { + t.Error(err) + } + return []*CandidateNode{&CandidateNode{Node: &dataBucket, Document: 0}} +} + +func resultsToString(results []*CandidateNode) string { + var pretty string = "" + for _, n := range results { + pretty = pretty + "\n" + NodeToString(n) + } + return pretty +} + +func TestDataTreeNavigatorSimple(t *testing.T) { + + nodes := readDoc(t, `a: + b: apple`) + + path, errPath := treeCreator.ParsePath("a") + if errPath != nil { + t.Error(errPath) + } + results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + + if errNav != nil { + t.Error(errNav) + } + + expected := ` +-- Node -- + Document 0, path: [a] + Tag: !!map, Kind: MappingNode, Anchor: + b: apple +` + + test.AssertResult(t, expected, resultsToString(results)) +} + +func TestDataTreeNavigatorSimpleDeep(t *testing.T) { + + nodes := readDoc(t, `a: + b: apple`) + + path, errPath := treeCreator.ParsePath("a.b") + if errPath != nil { + t.Error(errPath) + } + results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + + if errNav != nil { + t.Error(errNav) + } + + expected := ` +-- Node -- + Document 0, path: [a b] + Tag: !!str, Kind: ScalarNode, Anchor: + apple +` + + test.AssertResult(t, expected, resultsToString(results)) +} + +func TestDataTreeNavigatorSimpleMismatch(t *testing.T) { + + nodes := readDoc(t, `a: + c: apple`) + + path, errPath := treeCreator.ParsePath("a.b") + if errPath != nil { + t.Error(errPath) + } + results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + + if errNav != nil { + t.Error(errNav) + } + + expected := `` + + test.AssertResult(t, expected, resultsToString(results)) +} + +func TestDataTreeNavigatorWild(t *testing.T) { + + nodes := readDoc(t, `a: + cat: apple`) + + path, errPath := treeCreator.ParsePath("a.*a*") + if errPath != nil { + t.Error(errPath) + } + results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + + if errNav != nil { + t.Error(errNav) + } + + expected := ` +-- Node -- + Document 0, path: [a cat] + Tag: !!str, Kind: ScalarNode, Anchor: + apple +` + + test.AssertResult(t, expected, resultsToString(results)) +} diff --git a/pkg/yqlib/treeops/lib.go b/pkg/yqlib/treeops/lib.go new file mode 100644 index 00000000..b175dc09 --- /dev/null +++ b/pkg/yqlib/treeops/lib.go @@ -0,0 +1,59 @@ +package treeops + +import ( + "bytes" + "fmt" + + "gopkg.in/op/go-logging.v1" + "gopkg.in/yaml.v3" +) + +type CandidateNode struct { + Node *yaml.Node // the actual node + Path []interface{} /// the path we took to get to this node + Document uint // the document index of this node + + // middle nodes are nodes that match along the original path, but not a + // target match of the path. This is only relevant when ShouldOnlyDeeplyVisitLeaves is false. + IsMiddleNode bool +} + +var log = logging.MustGetLogger("yq-treeops") + +func NodeToString(node *CandidateNode) string { + if !log.IsEnabledFor(logging.DEBUG) { + return "" + } + value := node.Node + if value == nil { + return "-- node is nil --" + } + buf := new(bytes.Buffer) + encoder := yaml.NewEncoder(buf) + errorEncoding := encoder.Encode(value) + if errorEncoding != nil { + log.Error("Error debugging node, %v", errorEncoding.Error()) + } + encoder.Close() + return fmt.Sprintf(`-- Node -- + Document %v, path: %v + Tag: %v, Kind: %v, Anchor: %v + %v`, node.Document, node.Path, value.Tag, KindString(value.Kind), value.Anchor, buf.String()) +} + +func KindString(kind yaml.Kind) string { + switch kind { + case yaml.ScalarNode: + return "ScalarNode" + case yaml.SequenceNode: + return "SequenceNode" + case yaml.MappingNode: + return "MappingNode" + case yaml.DocumentNode: + return "DocumentNode" + case yaml.AliasNode: + return "AliasNode" + default: + return "unknown!" + } +} diff --git a/pkg/yqlib/treeops/matchKeyString.go b/pkg/yqlib/treeops/matchKeyString.go new file mode 100644 index 00000000..34714fd5 --- /dev/null +++ b/pkg/yqlib/treeops/matchKeyString.go @@ -0,0 +1,34 @@ +package treeops + +func Match(name string, pattern string) (matched bool) { + if pattern == "" { + return name == pattern + } + log.Debug("pattern: %v", pattern) + if pattern == "*" { + log.Debug("wild!") + return true + } + return deepMatch([]rune(name), []rune(pattern)) +} + +func deepMatch(str, pattern []rune) bool { + for len(pattern) > 0 { + switch pattern[0] { + default: + if len(str) == 0 || str[0] != pattern[0] { + return false + } + case '?': + if len(str) == 0 { + return false + } + case '*': + return deepMatch(str, pattern[1:]) || + (len(str) > 0 && deepMatch(str[1:], pattern)) + } + str = str[1:] + pattern = pattern[1:] + } + return len(str) == 0 && len(pattern) == 0 +} diff --git a/pkg/yqlib/path_postfix.go b/pkg/yqlib/treeops/path_postfix.go similarity index 99% rename from pkg/yqlib/path_postfix.go rename to pkg/yqlib/treeops/path_postfix.go index 30de1290..3dd4af69 100644 --- a/pkg/yqlib/path_postfix.go +++ b/pkg/yqlib/treeops/path_postfix.go @@ -1,4 +1,4 @@ -package yqlib +package treeops import ( "errors" diff --git a/pkg/yqlib/path_postfix_test.go b/pkg/yqlib/treeops/path_postfix_test.go similarity index 99% rename from pkg/yqlib/path_postfix_test.go rename to pkg/yqlib/treeops/path_postfix_test.go index 60ba9060..2e9d6361 100644 --- a/pkg/yqlib/path_postfix_test.go +++ b/pkg/yqlib/treeops/path_postfix_test.go @@ -1,4 +1,4 @@ -package yqlib +package treeops import ( "testing" diff --git a/pkg/yqlib/path_tokeniser.go b/pkg/yqlib/treeops/path_tokeniser.go similarity index 99% rename from pkg/yqlib/path_tokeniser.go rename to pkg/yqlib/treeops/path_tokeniser.go index cbd6e5e2..acabe798 100644 --- a/pkg/yqlib/path_tokeniser.go +++ b/pkg/yqlib/treeops/path_tokeniser.go @@ -1,4 +1,4 @@ -package yqlib +package treeops import ( "strconv" diff --git a/pkg/yqlib/path_tokeniser_test.go b/pkg/yqlib/treeops/path_tokeniser_test.go similarity index 99% rename from pkg/yqlib/path_tokeniser_test.go rename to pkg/yqlib/treeops/path_tokeniser_test.go index 31b6379e..4e7405ef 100644 --- a/pkg/yqlib/path_tokeniser_test.go +++ b/pkg/yqlib/treeops/path_tokeniser_test.go @@ -1,4 +1,4 @@ -package yqlib +package treeops import ( "testing" diff --git a/pkg/yqlib/path_tree.go b/pkg/yqlib/treeops/path_tree.go similarity index 61% rename from pkg/yqlib/path_tree.go rename to pkg/yqlib/treeops/path_tree.go index 070ee89d..a42b06f5 100644 --- a/pkg/yqlib/path_tree.go +++ b/pkg/yqlib/treeops/path_tree.go @@ -1,7 +1,10 @@ -package yqlib +package treeops import "fmt" +var myPathTokeniser = NewPathTokeniser() +var myPathPostfixer = NewPathPostFixer() + type PathTreeNode struct { PathElement *PathElement Lhs *PathTreeNode @@ -9,7 +12,8 @@ type PathTreeNode struct { } type PathTreeCreator interface { - CreatePathTree([]*PathElement) (*PathTreeNode, error) + ParsePath(path string) (*PathTreeNode, error) + CreatePathTree(postFixPath []*PathElement) (*PathTreeNode, error) } type pathTreeCreator struct { @@ -19,6 +23,19 @@ func NewPathTreeCreator() PathTreeCreator { return &pathTreeCreator{} } +func (p *pathTreeCreator) ParsePath(path string) (*PathTreeNode, error) { + tokens, err := myPathTokeniser.Tokenise(path) + if err != nil { + return nil, err + } + var pathElements []*PathElement + pathElements, err = myPathPostfixer.ConvertToPostfix(tokens) + if err != nil { + return nil, err + } + return p.CreatePathTree(pathElements) +} + func (p *pathTreeCreator) CreatePathTree(postFixPath []*PathElement) (*PathTreeNode, error) { var stack = make([]*PathTreeNode, 0) diff --git a/pkg/yqlib/treeops/path_tree_test.go b/pkg/yqlib/treeops/path_tree_test.go new file mode 100644 index 00000000..005dee7e --- /dev/null +++ b/pkg/yqlib/treeops/path_tree_test.go @@ -0,0 +1 @@ +package treeops diff --git a/pkg/yqlib/treeops/traverse.go b/pkg/yqlib/treeops/traverse.go new file mode 100644 index 00000000..df9928c5 --- /dev/null +++ b/pkg/yqlib/treeops/traverse.go @@ -0,0 +1,94 @@ +package treeops + +import ( + "fmt" + + "gopkg.in/yaml.v3" +) + +type traverser struct { + prefs NavigationPrefs +} + +type Traverser interface { + Traverse(matchingNode *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) +} + +func NewTraverser(navigationPrefs NavigationPrefs) Traverser { + return &traverser{navigationPrefs} +} + +func (t *traverser) keyMatches(key *yaml.Node, pathNode *PathElement) bool { + return Match(key.Value, fmt.Sprintf("%v", pathNode.Value)) +} + +func (t *traverser) traverseMap(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) { + // value.Content is a concatenated array of key, value, + // so keys are in the even indexes, values in odd. + // merge aliases are defined first, but we only want to traverse them + // if we don't find a match directly on this node first. + //TODO ALIASES, auto creation? + + var newMatches = make([]*CandidateNode, 0) + + node := candidate.Node + + var contents = node.Content + for index := 0; index < len(contents); index = index + 2 { + key := contents[index] + value := contents[index+1] + + log.Debug("checking %v (%v)", key.Value, key.Tag) + if t.keyMatches(key, pathNode) { + log.Debug("MATCHED") + newMatches = append(newMatches, &CandidateNode{ + Node: value, + Path: append(candidate.Path, key.Value), + Document: candidate.Document, + }) + } + } + return newMatches, nil + +} + +func (t *traverser) Traverse(matchingNode *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) { + log.Debug(NodeToString(matchingNode)) + value := matchingNode.Node + switch value.Kind { + case yaml.MappingNode: + log.Debug("its a map with %v entries", len(value.Content)/2) + return t.traverseMap(matchingNode, pathNode) + + // case yaml.SequenceNode: + // log.Debug("its a sequence of %v things!", len(value.Content)) + + // switch head := head.(type) { + // case int64: + // return n.recurseArray(value, head, head, tail, pathStack) + // default: + + // if head == "+" { + // return n.appendArray(value, head, tail, pathStack) + // } else if len(value.Content) == 0 && head == "**" { + // return n.navigationStrategy.Visit(nodeContext) + // } + // return n.splatArray(value, head, tail, pathStack) + // } + // case yaml.AliasNode: + // log.Debug("its an alias!") + // DebugNode(value.Alias) + // if n.navigationStrategy.FollowAlias(nodeContext) { + // log.Debug("following the alias") + // return n.recurse(value.Alias, head, tail, pathStack) + // } + // return nil + case yaml.DocumentNode: + log.Debug("digging into doc node") + return t.Traverse(&CandidateNode{ + Node: matchingNode.Node.Content[0], + Document: matchingNode.Document}, pathNode) + default: + return nil, nil + } +}