wip

pkg/yqlib/data_tree_navigator.go

@@ -1,73 +0,0 @@
-package yqlib
-
-type dataTreeNavigator struct {
-}
-
-type DataTreeNavigator interface {
-	GetMatchingNodes(matchingNodes []*NodeContext, pathNode *PathTreeNode) ([]*NodeContext, error)
-}
-
-func NewTreeNavigator() DataTreeNavigator {
-	return &dataTreeNavigator{}
-}
-
-func (d *dataTreeNavigator) traverseSingle(matchingNode *NodeContext, pathNode *PathElement) ([]*NodeContext, error) {
-	var value = matchingNode.Node
-	// match all for splat
-	// match all and recurse for deep
-	// etc and so forth
-
-}
-
-func (d *dataTreeNavigator) traverse(matchingNodes []*NodeContext, pathNode *PathElement) ([]*NodeContext, error) {
-	var newMatchingNodes = make([]*NodeContext, 0)
-	var newNodes []*NodeContext
-	var err error
-	for _, node := range matchingNodes {
-
-		newNodes, err = d.traverseSingle(node, pathNode)
-		if err != nil {
-			return nil, err
-		}
-		newMatchingNodes = append(newMatchingNodes, newNodes...)
-	}
-
-	return newMatchingNodes, nil
-}
-
-func (d *dataTreeNavigator) GetMatchingNodes(matchingNodes []*NodeContext, pathNode *PathTreeNode) ([]*NodeContext, error) {
-	if pathNode.PathElement.PathElementType == PathKey || pathNode.PathElement.PathElementType == ArrayIndex {
-		return d.traverse(matchingNodes, pathNode.PathElement)
-	} else {
-		var lhs, rhs []*NodeContext
-		var err error
-		switch pathNode.PathElement.OperationType {
-		case Traverse:
-			lhs, err = d.GetMatchingNodes(matchingNodes, pathNode.Lhs)
-			if err != nil {
-				return nil, err
-			}
-			return d.GetMatchingNodes(lhs, pathNode.Rhs)
-		case Or, And:
-			lhs, err = d.GetMatchingNodes(matchingNodes, pathNode.Lhs)
-			if err != nil {
-				return nil, err
-			}
-			rhs, err = d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
-			if err != nil {
-				return nil, err
-			}
-			return d.setFunction(pathNode.PathElement, lhs, rhs), nil
-		case Equals:
-			lhs, err = d.GetMatchingNodes(matchingNodes, pathNode.Lhs)
-			if err != nil {
-				return nil, err
-			}
-			return d.findMatchingValues(lhs, pathNode.Rhs)
-		case EqualsSelf:
-			return d.findMatchingValues(matchingNodes, pathNode.Rhs)
-		}
-
-	}
-
-}

pkg/yqlib/data_tree_navigator_test.go

@@ -1 +0,0 @@
-package yqlib

pkg/yqlib/path_tree_test.go

@@ -1 +0,0 @@
-package yqlib

pkg/yqlib/treeops/data_tree_navigator.go

@@ -0,0 +1,75 @@
+package treeops
+
+type dataTreeNavigator struct {
+	traverser Traverser
+}
+
+type NavigationPrefs struct {
+	FollowAlias bool
+}
+
+type DataTreeNavigator interface {
+	GetMatchingNodes(matchingNodes []*CandidateNode, pathNode *PathTreeNode) ([]*CandidateNode, error)
+}
+
+func NewDataTreeNavigator(navigationPrefs NavigationPrefs) DataTreeNavigator {
+	traverse := NewTraverser(navigationPrefs)
+	return &dataTreeNavigator{traverse}
+}
+
+func (d *dataTreeNavigator) traverse(matchingNodes []*CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) {
+	log.Debugf("-- Traversing")
+	var newMatchingNodes = make([]*CandidateNode, 0)
+	var newNodes []*CandidateNode
+	var err error
+	for _, node := range matchingNodes {
+
+		newNodes, err = d.traverser.Traverse(node, pathNode)
+		if err != nil {
+			return nil, err
+		}
+		newMatchingNodes = append(newMatchingNodes, newNodes...)
+	}
+
+	return newMatchingNodes, nil
+}
+
+func (d *dataTreeNavigator) GetMatchingNodes(matchingNodes []*CandidateNode, pathNode *PathTreeNode) ([]*CandidateNode, error) {
+	log.Debugf("Processing Path: %v", pathNode.PathElement.toString())
+	if pathNode.PathElement.PathElementType == PathKey || pathNode.PathElement.PathElementType == ArrayIndex {
+		return d.traverse(matchingNodes, pathNode.PathElement)
+	} else {
+		var lhs []*CandidateNode //, rhs
+		var err error
+		switch pathNode.PathElement.OperationType {
+		case Traverse:
+			lhs, err = d.GetMatchingNodes(matchingNodes, pathNode.Lhs)
+			if err != nil {
+				return nil, err
+			}
+			return d.GetMatchingNodes(lhs, pathNode.Rhs)
+		// case Or, And:
+		// 	lhs, err = d.GetMatchingNodes(matchingNodes, pathNode.Lhs)
+		// 	if err != nil {
+		// 		return nil, err
+		// 	}
+		// 	rhs, err = d.GetMatchingNodes(matchingNodes, pathNode.Rhs)
+		// 	if err != nil {
+		// 		return nil, err
+		// 	}
+		// 	return d.setFunction(pathNode.PathElement, lhs, rhs), nil
+		// case Equals:
+		// 	lhs, err = d.GetMatchingNodes(matchingNodes, pathNode.Lhs)
+		// 	if err != nil {
+		// 		return nil, err
+		// 	}
+		// 	return d.findMatchingValues(lhs, pathNode.Rhs)
+		// case EqualsSelf:
+		// 	return d.findMatchingValues(matchingNodes, pathNode.Rhs)
+		default:
+			return nil, nil
+		}
+
+	}
+
+}

pkg/yqlib/treeops/data_tree_navigator_test.go

@@ -0,0 +1,125 @@
+package treeops
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/mikefarah/yq/v3/test"
+	yaml "gopkg.in/yaml.v3"
+)
+
+var treeNavigator = NewDataTreeNavigator(NavigationPrefs{})
+var treeCreator = NewPathTreeCreator()
+
+func readDoc(t *testing.T, content string) []*CandidateNode {
+	decoder := yaml.NewDecoder(strings.NewReader(content))
+	var dataBucket yaml.Node
+	err := decoder.Decode(&dataBucket)
+	if err != nil {
+		t.Error(err)
+	}
+	return []*CandidateNode{&CandidateNode{Node: &dataBucket, Document: 0}}
+}
+
+func resultsToString(results []*CandidateNode) string {
+	var pretty string = ""
+	for _, n := range results {
+		pretty = pretty + "\n" + NodeToString(n)
+	}
+	return pretty
+}
+
+func TestDataTreeNavigatorSimple(t *testing.T) {
+
+	nodes := readDoc(t, `a: 
+  b: apple`)
+
+	path, errPath := treeCreator.ParsePath("a")
+	if errPath != nil {
+		t.Error(errPath)
+	}
+	results, errNav := treeNavigator.GetMatchingNodes(nodes, path)
+
+	if errNav != nil {
+		t.Error(errNav)
+	}
+
+	expected := `
+-- Node --
+  Document 0, path: [a]
+  Tag: !!map, Kind: MappingNode, Anchor: 
+  b: apple
+`
+
+	test.AssertResult(t, expected, resultsToString(results))
+}
+
+func TestDataTreeNavigatorSimpleDeep(t *testing.T) {
+
+	nodes := readDoc(t, `a: 
+  b: apple`)
+
+	path, errPath := treeCreator.ParsePath("a.b")
+	if errPath != nil {
+		t.Error(errPath)
+	}
+	results, errNav := treeNavigator.GetMatchingNodes(nodes, path)
+
+	if errNav != nil {
+		t.Error(errNav)
+	}
+
+	expected := `
+-- Node --
+  Document 0, path: [a b]
+  Tag: !!str, Kind: ScalarNode, Anchor: 
+  apple
+`
+
+	test.AssertResult(t, expected, resultsToString(results))
+}
+
+func TestDataTreeNavigatorSimpleMismatch(t *testing.T) {
+
+	nodes := readDoc(t, `a: 
+  c: apple`)
+
+	path, errPath := treeCreator.ParsePath("a.b")
+	if errPath != nil {
+		t.Error(errPath)
+	}
+	results, errNav := treeNavigator.GetMatchingNodes(nodes, path)
+
+	if errNav != nil {
+		t.Error(errNav)
+	}
+
+	expected := ``
+
+	test.AssertResult(t, expected, resultsToString(results))
+}
+
+func TestDataTreeNavigatorWild(t *testing.T) {
+
+	nodes := readDoc(t, `a: 
+  cat: apple`)
+
+	path, errPath := treeCreator.ParsePath("a.*a*")
+	if errPath != nil {
+		t.Error(errPath)
+	}
+	results, errNav := treeNavigator.GetMatchingNodes(nodes, path)
+
+	if errNav != nil {
+		t.Error(errNav)
+	}
+
+	expected := `
+-- Node --
+  Document 0, path: [a cat]
+  Tag: !!str, Kind: ScalarNode, Anchor: 
+  apple
+`
+
+	test.AssertResult(t, expected, resultsToString(results))
+}

pkg/yqlib/treeops/lib.go

@@ -0,0 +1,59 @@
+package treeops
+
+import (
+	"bytes"
+	"fmt"
+
+	"gopkg.in/op/go-logging.v1"
+	"gopkg.in/yaml.v3"
+)
+
+type CandidateNode struct {
+	Node     *yaml.Node    // the actual node
+	Path     []interface{} /// the path we took to get to this node
+	Document uint          // the document index of this node
+
+	// middle nodes are nodes that match along the original path, but not a
+	// target match of the path. This is only relevant when ShouldOnlyDeeplyVisitLeaves is false.
+	IsMiddleNode bool
+}
+
+var log = logging.MustGetLogger("yq-treeops")
+
+func NodeToString(node *CandidateNode) string {
+	if !log.IsEnabledFor(logging.DEBUG) {
+		return ""
+	}
+	value := node.Node
+	if value == nil {
+		return "-- node is nil --"
+	}
+	buf := new(bytes.Buffer)
+	encoder := yaml.NewEncoder(buf)
+	errorEncoding := encoder.Encode(value)
+	if errorEncoding != nil {
+		log.Error("Error debugging node, %v", errorEncoding.Error())
+	}
+	encoder.Close()
+	return fmt.Sprintf(`-- Node --
+  Document %v, path: %v
+  Tag: %v, Kind: %v, Anchor: %v
+  %v`, node.Document, node.Path, value.Tag, KindString(value.Kind), value.Anchor, buf.String())
+}
+
+func KindString(kind yaml.Kind) string {
+	switch kind {
+	case yaml.ScalarNode:
+		return "ScalarNode"
+	case yaml.SequenceNode:
+		return "SequenceNode"
+	case yaml.MappingNode:
+		return "MappingNode"
+	case yaml.DocumentNode:
+		return "DocumentNode"
+	case yaml.AliasNode:
+		return "AliasNode"
+	default:
+		return "unknown!"
+	}
+}

pkg/yqlib/treeops/matchKeyString.go

@@ -0,0 +1,34 @@
+package treeops
+
+func Match(name string, pattern string) (matched bool) {
+	if pattern == "" {
+		return name == pattern
+	}
+	log.Debug("pattern: %v", pattern)
+	if pattern == "*" {
+		log.Debug("wild!")
+		return true
+	}
+	return deepMatch([]rune(name), []rune(pattern))
+}
+
+func deepMatch(str, pattern []rune) bool {
+	for len(pattern) > 0 {
+		switch pattern[0] {
+		default:
+			if len(str) == 0 || str[0] != pattern[0] {
+				return false
+			}
+		case '?':
+			if len(str) == 0 {
+				return false
+			}
+		case '*':
+			return deepMatch(str, pattern[1:]) ||
+				(len(str) > 0 && deepMatch(str[1:], pattern))
+		}
+		str = str[1:]
+		pattern = pattern[1:]
+	}
+	return len(str) == 0 && len(pattern) == 0
+}

pkg/yqlib/treeops/path_postfix.go

@@ -1,4 +1,4 @@
-package yqlib
+package treeops
 
 import (
 	"errors"

pkg/yqlib/treeops/path_postfix_test.go

@@ -1,4 +1,4 @@
-package yqlib
+package treeops
 
 import (
 	"testing"

pkg/yqlib/treeops/path_tokeniser.go

@@ -1,4 +1,4 @@
-package yqlib
+package treeops
 
 import (
 	"strconv"

pkg/yqlib/treeops/path_tokeniser_test.go

@@ -1,4 +1,4 @@
-package yqlib
+package treeops
 
 import (
 	"testing"

pkg/yqlib/treeops/path_tree.go

@@ -1,7 +1,10 @@
-package yqlib
+package treeops
 
 import "fmt"
 
+var myPathTokeniser = NewPathTokeniser()
+var myPathPostfixer = NewPathPostFixer()
+
 type PathTreeNode struct {
 	PathElement *PathElement
 	Lhs         *PathTreeNode
@@ -9,7 +12,8 @@ type PathTreeNode struct {
 }
 
 type PathTreeCreator interface {
-	CreatePathTree([]*PathElement) (*PathTreeNode, error)
+	ParsePath(path string) (*PathTreeNode, error)
+	CreatePathTree(postFixPath []*PathElement) (*PathTreeNode, error)
 }
 
 type pathTreeCreator struct {
@@ -19,6 +23,19 @@ func NewPathTreeCreator() PathTreeCreator {
 	return &pathTreeCreator{}
 }
 
+func (p *pathTreeCreator) ParsePath(path string) (*PathTreeNode, error) {
+	tokens, err := myPathTokeniser.Tokenise(path)
+	if err != nil {
+		return nil, err
+	}
+	var pathElements []*PathElement
+	pathElements, err = myPathPostfixer.ConvertToPostfix(tokens)
+	if err != nil {
+		return nil, err
+	}
+	return p.CreatePathTree(pathElements)
+}
+
 func (p *pathTreeCreator) CreatePathTree(postFixPath []*PathElement) (*PathTreeNode, error) {
 	var stack = make([]*PathTreeNode, 0)
 

pkg/yqlib/treeops/path_tree_test.go

@@ -0,0 +1 @@
+package treeops

pkg/yqlib/treeops/traverse.go

@@ -0,0 +1,94 @@
+package treeops
+
+import (
+	"fmt"
+
+	"gopkg.in/yaml.v3"
+)
+
+type traverser struct {
+	prefs NavigationPrefs
+}
+
+type Traverser interface {
+	Traverse(matchingNode *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error)
+}
+
+func NewTraverser(navigationPrefs NavigationPrefs) Traverser {
+	return &traverser{navigationPrefs}
+}
+
+func (t *traverser) keyMatches(key *yaml.Node, pathNode *PathElement) bool {
+	return Match(key.Value, fmt.Sprintf("%v", pathNode.Value))
+}
+
+func (t *traverser) traverseMap(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) {
+	// value.Content is a concatenated array of key, value,
+	// so keys are in the even indexes, values in odd.
+	// merge aliases are defined first, but we only want to traverse them
+	// if we don't find a match directly on this node first.
+	//TODO ALIASES, auto creation?
+
+	var newMatches = make([]*CandidateNode, 0)
+
+	node := candidate.Node
+
+	var contents = node.Content
+	for index := 0; index < len(contents); index = index + 2 {
+		key := contents[index]
+		value := contents[index+1]
+
+		log.Debug("checking %v (%v)", key.Value, key.Tag)
+		if t.keyMatches(key, pathNode) {
+			log.Debug("MATCHED")
+			newMatches = append(newMatches, &CandidateNode{
+				Node:     value,
+				Path:     append(candidate.Path, key.Value),
+				Document: candidate.Document,
+			})
+		}
+	}
+	return newMatches, nil
+
+}
+
+func (t *traverser) Traverse(matchingNode *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) {
+	log.Debug(NodeToString(matchingNode))
+	value := matchingNode.Node
+	switch value.Kind {
+	case yaml.MappingNode:
+		log.Debug("its a map with %v entries", len(value.Content)/2)
+		return t.traverseMap(matchingNode, pathNode)
+
+	// case yaml.SequenceNode:
+	// 	log.Debug("its a sequence of %v things!", len(value.Content))
+
+	// 	switch head := head.(type) {
+	// 	case int64:
+	// 		return n.recurseArray(value, head, head, tail, pathStack)
+	// 	default:
+
+	// 		if head == "+" {
+	// 			return n.appendArray(value, head, tail, pathStack)
+	// 		} else if len(value.Content) == 0 && head == "**" {
+	// 			return n.navigationStrategy.Visit(nodeContext)
+	// 		}
+	// 		return n.splatArray(value, head, tail, pathStack)
+	// 	}
+	// case yaml.AliasNode:
+	// 	log.Debug("its an alias!")
+	// 	DebugNode(value.Alias)
+	// 	if n.navigationStrategy.FollowAlias(nodeContext) {
+	// 		log.Debug("following the alias")
+	// 		return n.recurse(value.Alias, head, tail, pathStack)
+	// 	}
+	// 	return nil
+	case yaml.DocumentNode:
+		log.Debug("digging into doc node")
+		return t.Traverse(&CandidateNode{
+			Node:     matchingNode.Node.Content[0],
+			Document: matchingNode.Document}, pathNode)
+	default:
+		return nil, nil
+	}
+}