wip

pkg/yqlib/treeops/data_tree_navigator.go

@@ -8,7 +8,7 @@ import (
 )
 
 type dataTreeNavigator struct {
-	leafTraverser LeafTraverser
+	navigationPrefs NavigationPrefs
 }
 
 type NavigationPrefs struct {
@@ -20,27 +20,7 @@ type DataTreeNavigator interface {
 }
 
 func NewDataTreeNavigator(navigationPrefs NavigationPrefs) DataTreeNavigator {
-	leafTraverser := NewLeafTraverser(navigationPrefs)
+	return &dataTreeNavigator{navigationPrefs}
-	return &dataTreeNavigator{leafTraverser}
-}
-
-func (d *dataTreeNavigator) traverse(matchMap *orderedmap.OrderedMap, pathNode *PathElement) (*orderedmap.OrderedMap, error) {
-	log.Debugf("-- Traversing")
-	var matchingNodeMap = orderedmap.NewOrderedMap()
-	var newNodes []*CandidateNode
-	var err error
-
-	for el := matchMap.Front(); el != nil; el = el.Next() {
-		newNodes, err = d.leafTraverser.Traverse(el.Value.(*CandidateNode), pathNode)
-		if err != nil {
-			return nil, err
-		}
-		for _, n := range newNodes {
-			matchingNodeMap.Set(n.GetKey(), n)
-		}
-	}
-
-	return matchingNodeMap, nil
 }
 
 func (d *dataTreeNavigator) GetMatchingNodes(matchingNodes []*CandidateNode, pathNode *PathTreeNode) ([]*CandidateNode, error) {
@@ -75,19 +55,10 @@ func (d *dataTreeNavigator) getMatchingNodes(matchingNodes *orderedmap.OrderedMa
 		}
 	}
 	log.Debug(">>")
-
-	if pathNode.PathElement.PathElementType == SelfReference {
-		return matchingNodes, nil
-	} else if pathNode.PathElement.PathElementType == PathKey {
-		return d.traverse(matchingNodes, pathNode.PathElement)
-	} else if pathNode.PathElement.PathElementType == Value {
-		return nodeToMap(pathNode.PathElement.CandidateNode), nil
-	} else {
 	handler := pathNode.PathElement.OperationType.Handler
 	if handler != nil {
 		return handler(d, matchingNodes, pathNode)
 	}
 	return nil, fmt.Errorf("Unknown operator %v", pathNode.PathElement.OperationType)
-	}
 
 }

pkg/yqlib/treeops/lib.go

@@ -11,20 +11,6 @@ import (
 
 var log = logging.MustGetLogger("yq-treeops")
 
-type PathElementType uint32
-
-const (
-	PathKey PathElementType = 1 << iota
-	DocumentKey
-	Operation
-	SelfReference
-	OpenBracket
-	CloseBracket
-	OpenCollect
-	CloseCollect
-	Value // e.g. string, int
-)
-
 type OperationType struct {
 	Type       string
 	NumArgs    uint // number of arguments to the op
@@ -32,8 +18,6 @@ type OperationType struct {
 	Handler    OperatorHandler
 }
 
-var None = &OperationType{Type: "NONE", NumArgs: 0, Precedence: 0}
-
 var Or = &OperationType{Type: "OR", NumArgs: 2, Precedence: 20, Handler: OrOperator}
 var And = &OperationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: AndOperator}
 
@@ -49,6 +33,12 @@ var Pipe = &OperationType{Type: "PIPE", NumArgs: 2, Precedence: 45, Handler: Pip
 
 var Length = &OperationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: LengthOperator}
 var Collect = &OperationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: CollectOperator}
+var TraversePath = &OperationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
+
+var DocumentFilter = &OperationType{Type: "DOCUMENT_FILTER", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
+var SelfReference = &OperationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
+var ValueOp = &OperationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
+
 var RecursiveDescent = &OperationType{Type: "RECURSIVE_DESCENT", NumArgs: 0, Precedence: 50, Handler: RecursiveDescentOperator}
 
 // not sure yet
@@ -63,7 +53,6 @@ var DeleteChild = &OperationType{Type: "DELETE", NumArgs: 2, Precedence: 40, Han
 // filters matches if they have the existing path
 
 type PathElement struct {
-	PathElementType PathElementType
 	OperationType *OperationType
 	Value         interface{}
 	StringValue   string
@@ -72,22 +61,17 @@ type PathElement struct {
 
 // debugging purposes only
 func (p *PathElement) toString() string {
-	var result string = ``
+	if p.OperationType == TraversePath {
-	switch p.PathElementType {
+		return fmt.Sprintf("%v", p.Value)
-	case PathKey:
+	} else if p.OperationType == DocumentFilter {
-		result = result + fmt.Sprintf("%v", p.Value)
+		return fmt.Sprintf("d%v", p.Value)
-	case DocumentKey:
+	} else if p.OperationType == SelfReference {
-		result = result + fmt.Sprintf("D%v", p.Value)
+		return "SELF"
-	case SelfReference:
+	} else if p.OperationType == ValueOp {
-		result = result + fmt.Sprintf("SELF")
+		return fmt.Sprintf("%v (%T)", p.Value, p.Value)
-	case Operation:
+	} else {
-		result = result + fmt.Sprintf("%v", p.OperationType.Type)
+		return fmt.Sprintf("%v", p.OperationType.Type)
-	case Value:
-		result = result + fmt.Sprintf("%v (%T)", p.Value, p.Value)
-	default:
-		result = result + "I HAVENT GOT A STRATEGY"
 	}
-	return result
 }
 
 type YqTreeLib interface {

pkg/yqlib/treeops/operator_multilpy.go

@@ -61,12 +61,12 @@ func multiply(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*Ca
 
 func createTraversalTree(path []interface{}) *PathTreeNode {
 	if len(path) == 0 {
-		return &PathTreeNode{PathElement: &PathElement{PathElementType: SelfReference}}
+		return &PathTreeNode{PathElement: &PathElement{OperationType: SelfReference}}
 	} else if len(path) == 1 {
-		return &PathTreeNode{PathElement: &PathElement{PathElementType: PathKey, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}}
+		return &PathTreeNode{PathElement: &PathElement{OperationType: TraversePath, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}}
 	}
 	return &PathTreeNode{
-		PathElement: &PathElement{PathElementType: Operation, OperationType: Pipe},
+		PathElement: &PathElement{OperationType: Pipe},
 		Lhs:         createTraversalTree(path[0:1]),
 		Rhs:         createTraversalTree(path[1:])}
 
@@ -77,11 +77,11 @@ func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *Candid
 
 	lhsPath := rhs.Path[pathIndexToStartFrom:]
 
-	assignmentOp := &PathElement{PathElementType: Operation, OperationType: AssignAttributes}
+	assignmentOp := &PathElement{OperationType: AssignAttributes}
 	if rhs.Node.Kind == yaml.ScalarNode {
 		assignmentOp.OperationType = Assign
 	}
-	rhsOp := &PathElement{PathElementType: Value, CandidateNode: rhs}
+	rhsOp := &PathElement{OperationType: ValueOp, CandidateNode: rhs}
 
 	assignmentOpNode := &PathTreeNode{PathElement: assignmentOp, Lhs: createTraversalTree(lhsPath), Rhs: &PathTreeNode{PathElement: rhsOp}}
 

pkg/yqlib/treeops/operator_recursive_descent.go

@@ -16,11 +16,14 @@ func RecursiveDescentOperator(d *dataTreeNavigator, matchMap *orderedmap.Ordered
 }
 
 func recursiveDecent(d *dataTreeNavigator, results *orderedmap.OrderedMap, matchMap *orderedmap.OrderedMap) error {
+	splatPathElement := &PathElement{OperationType: TraversePath, Value: "[]"}
+	splatTreeNode := &PathTreeNode{PathElement: splatPathElement}
+
 	for el := matchMap.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 		results.Set(candidate.GetKey(), candidate)
 
-		children, err := d.traverse(nodeToMap(candidate), &PathElement{PathElementType: PathKey, Value: "[]"})
+		children, err := TraversePathOperator(d, nodeToMap(candidate), splatTreeNode)
 
 		if err != nil {
 			return err

pkg/yqlib/treeops/operator_traverse_path.go

@@ -3,26 +3,86 @@ package treeops
 import (
 	"fmt"
 
+	"github.com/elliotchance/orderedmap"
 	"gopkg.in/yaml.v3"
 )
 
-type traverser struct {
+func TraversePathOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
-	prefs NavigationPrefs
+	log.Debugf("-- Traversing")
+	var matchingNodeMap = orderedmap.NewOrderedMap()
+	var newNodes []*CandidateNode
+	var err error
+
+	for el := matchMap.Front(); el != nil; el = el.Next() {
+		newNodes, err = traverse(d, el.Value.(*CandidateNode), pathNode.PathElement)
+		if err != nil {
+			return nil, err
+		}
+		for _, n := range newNodes {
+			matchingNodeMap.Set(n.GetKey(), n)
+		}
+	}
+
+	return matchingNodeMap, nil
 }
 
-type LeafTraverser interface {
+func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) {
-	Traverse(matchingNode *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error)
+	log.Debug("Traversing %v", NodeToString(matchingNode))
+	value := matchingNode.Node
+
+	if value.Kind == 0 {
+		log.Debugf("Guessing kind")
+		// we must ahve added this automatically, lets guess what it should be now
+		switch pathNode.Value.(type) {
+		case int, int64:
+			log.Debugf("probably an array")
+			value.Kind = yaml.SequenceNode
+		default:
+			log.Debugf("probabel a map")
+			value.Kind = yaml.MappingNode
+		}
+	}
+
+	switch value.Kind {
+	case yaml.MappingNode:
+		log.Debug("its a map with %v entries", len(value.Content)/2)
+		return traverseMap(matchingNode, pathNode)
+
+	case yaml.SequenceNode:
+		log.Debug("its a sequence of %v things!", len(value.Content))
+		return traverseArray(matchingNode, pathNode)
+	// 	default:
+
+	// 		if head == "+" {
+	// 			return n.appendArray(value, head, tail, pathStack)
+	// 		} else if len(value.Content) == 0 && head == "**" {
+	// 			return n.navigationStrategy.Visit(nodeContext)
+	// 		}
+	// 		return n.splatArray(value, head, tail, pathStack)
+	// 	}
+	// case yaml.AliasNode:
+	// 	log.Debug("its an alias!")
+	// 	DebugNode(value.Alias)
+	// 	if n.navigationStrategy.FollowAlias(nodeContext) {
+	// 		log.Debug("following the alias")
+	// 		return n.recurse(value.Alias, head, tail, pathStack)
+	// 	}
+	// 	return nil
+	case yaml.DocumentNode:
+		log.Debug("digging into doc node")
+		return traverse(d, &CandidateNode{
+			Node:     matchingNode.Node.Content[0],
+			Document: matchingNode.Document}, pathNode)
+	default:
+		return nil, nil
+	}
 }
 
-func NewLeafTraverser(navigationPrefs NavigationPrefs) LeafTraverser {
+func keyMatches(key *yaml.Node, pathNode *PathElement) bool {
-	return &traverser{navigationPrefs}
-}
-
-func (t *traverser) keyMatches(key *yaml.Node, pathNode *PathElement) bool {
 	return pathNode.Value == "[]" || Match(key.Value, pathNode.StringValue)
 }
 
-func (t *traverser) traverseMap(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) {
+func traverseMap(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) {
 	// value.Content is a concatenated array of key, value,
 	// so keys are in the even indexes, values in odd.
 	// merge aliases are defined first, but we only want to traverse them
@@ -39,7 +99,7 @@ func (t *traverser) traverseMap(candidate *CandidateNode, pathNode *PathElement)
 		value := contents[index+1]
 
 		log.Debug("checking %v (%v)", key.Value, key.Tag)
-		if t.keyMatches(key, pathNode) {
+		if keyMatches(key, pathNode) {
 			log.Debug("MATCHED")
 			newMatches = append(newMatches, &CandidateNode{
 				Node:     value,
@@ -63,7 +123,7 @@ func (t *traverser) traverseMap(candidate *CandidateNode, pathNode *PathElement)
 	return newMatches, nil
 }
 
-func (t *traverser) traverseArray(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) {
+func traverseArray(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) {
 	log.Debug("pathNode Value %v", pathNode.Value)
 	if pathNode.Value == "[]" {
 
@@ -104,55 +164,3 @@ func (t *traverser) traverseArray(candidate *CandidateNode, pathNode *PathElemen
 	}}, nil
 
 }
-
-func (t *traverser) Traverse(matchingNode *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) {
-	log.Debug("Traversing %v", NodeToString(matchingNode))
-	value := matchingNode.Node
-
-	if value.Kind == 0 {
-		log.Debugf("Guessing kind")
-		// we must ahve added this automatically, lets guess what it should be now
-		switch pathNode.Value.(type) {
-		case int, int64:
-			log.Debugf("probably an array")
-			value.Kind = yaml.SequenceNode
-		default:
-			log.Debugf("probabel a map")
-			value.Kind = yaml.MappingNode
-		}
-	}
-
-	switch value.Kind {
-	case yaml.MappingNode:
-		log.Debug("its a map with %v entries", len(value.Content)/2)
-		return t.traverseMap(matchingNode, pathNode)
-
-	case yaml.SequenceNode:
-		log.Debug("its a sequence of %v things!", len(value.Content))
-		return t.traverseArray(matchingNode, pathNode)
-	// 	default:
-
-	// 		if head == "+" {
-	// 			return n.appendArray(value, head, tail, pathStack)
-	// 		} else if len(value.Content) == 0 && head == "**" {
-	// 			return n.navigationStrategy.Visit(nodeContext)
-	// 		}
-	// 		return n.splatArray(value, head, tail, pathStack)
-	// 	}
-	// case yaml.AliasNode:
-	// 	log.Debug("its an alias!")
-	// 	DebugNode(value.Alias)
-	// 	if n.navigationStrategy.FollowAlias(nodeContext) {
-	// 		log.Debug("following the alias")
-	// 		return n.recurse(value.Alias, head, tail, pathStack)
-	// 	}
-	// 	return nil
-	case yaml.DocumentNode:
-		log.Debug("digging into doc node")
-		return t.Traverse(&CandidateNode{
-			Node:     matchingNode.Node.Content[0],
-			Document: matchingNode.Document}, pathNode)
-	default:
-		return nil, nil
-	}
-}

pkg/yqlib/treeops/operator_traverse_path_test.go

@@ -0,0 +1,21 @@
+package treeops
+
+import (
+	"testing"
+)
+
+var traversePathOperatorScenarios = []expressionScenario{
+	{
+		document:   `{a: {b: apple}}`,
+		expression: `.a`,
+		expected: []string{
+			"D0, P[a], (!!map)::{b: apple}\n",
+		},
+	},
+}
+
+func TestTraversePathOperatorScenarios(t *testing.T) {
+	for _, tt := range traversePathOperatorScenarios {
+		testScenario(t, &tt)
+	}
+}

pkg/yqlib/treeops/path_parse_test.go

@@ -81,6 +81,7 @@ var pathTests = []struct {
 }
 
 var tokeniser = NewPathTokeniser()
+var postFixer = NewPathPostFixer()
 
 func TestPathParsing(t *testing.T) {
 	for _, tt := range pathTests {

pkg/yqlib/treeops/path_postfix.go

@@ -18,32 +18,31 @@ func NewPathPostFixer() PathPostFixer {
 }
 
 func popOpToResult(opStack []*Token, result []*PathElement) ([]*Token, []*PathElement) {
-	var operatorToPushToPostFix *Token
+	var newOp *Token
-	opStack, operatorToPushToPostFix = opStack[0:len(opStack)-1], opStack[len(opStack)-1]
+	opStack, newOp = opStack[0:len(opStack)-1], opStack[len(opStack)-1]
-	var pathElement = PathElement{PathElementType: Operation, OperationType: operatorToPushToPostFix.OperationType}
+	var pathElement = PathElement{OperationType: newOp.OperationType, Value: newOp.Value, StringValue: newOp.StringValue}
+
+	if newOp.OperationType == ValueOp {
+		var candidateNode = BuildCandidateNodeFrom(newOp)
+		pathElement.CandidateNode = candidateNode
+	}
+
 	return opStack, append(result, &pathElement)
 }
 
 func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*PathElement, error) {
 	var result []*PathElement
 	// surround the whole thing with quotes
-	var opStack = []*Token{&Token{PathElementType: OpenBracket, OperationType: None, Value: "("}}
+	var opStack = []*Token{&Token{TokenType: OpenBracket, Value: "("}}
-	var tokens = append(infixTokens, &Token{PathElementType: CloseBracket, OperationType: None, Value: ")"})
+	var tokens = append(infixTokens, &Token{TokenType: CloseBracket, Value: ")"})
 
 	for _, token := range tokens {
 		log.Debugf("postfix processing token %v", token.Value)
-		switch token.PathElementType {
+		switch token.TokenType {
-		case Value:
-			var candidateNode = BuildCandidateNodeFrom(token)
-			var pathElement = PathElement{PathElementType: token.PathElementType, Value: token.Value, StringValue: token.StringValue, CandidateNode: candidateNode}
-			result = append(result, &pathElement)
-		case PathKey, SelfReference, DocumentKey:
-			var pathElement = PathElement{PathElementType: token.PathElementType, Value: token.Value, StringValue: token.StringValue}
-			result = append(result, &pathElement)
 		case OpenBracket, OpenCollect:
 			opStack = append(opStack, token)
 		case CloseCollect:
-			for len(opStack) > 0 && opStack[len(opStack)-1].PathElementType != OpenCollect {
+			for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != OpenCollect {
 				opStack, result = popOpToResult(opStack, result)
 			}
 			if len(opStack) == 0 {
@@ -52,10 +51,10 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*PathElement,
 			// now we should have [] as the last element on the opStack, get rid of it
 			opStack = opStack[0 : len(opStack)-1]
 			//and append a collect to the opStack
-			opStack = append(opStack, &Token{PathElementType: Operation, OperationType: Pipe})
+			opStack = append(opStack, &Token{TokenType: Operation, OperationType: Pipe})
-			opStack = append(opStack, &Token{PathElementType: Operation, OperationType: Collect})
+			opStack = append(opStack, &Token{TokenType: Operation, OperationType: Collect})
 		case CloseBracket:
-			for len(opStack) > 0 && opStack[len(opStack)-1].PathElementType != OpenBracket {
+			for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != OpenBracket {
 				opStack, result = popOpToResult(opStack, result)
 			}
 			if len(opStack) == 0 {

pkg/yqlib/treeops/path_postfix_test.go

@@ -1,355 +0,0 @@
-package treeops
-
-import (
-	"testing"
-
-	"github.com/mikefarah/yq/v3/test"
-)
-
-// var tokeniser = NewPathTokeniser()
-var postFixer = NewPathPostFixer()
-
-func testExpression(expression string) (string, error) {
-	tokens, err := tokeniser.Tokenise(expression)
-	if err != nil {
-		return "", err
-	}
-	results, errorP := postFixer.ConvertToPostfix(tokens)
-	if errorP != nil {
-		return "", errorP
-	}
-	formatted := ""
-	for _, path := range results {
-		formatted = formatted + path.toString() + ", "
-	}
-	return formatted, nil
-}
-
-
-func TestPostFixTraverseBar(t *testing.T) {
-	var infix = ".animals | [.]"
-	var expectedOutput = `PathKey - animals
---------
-SELF
---------
-Operation - COLLECT
---------
-Operation - PIPE
---------
-`
-
-	actual, err := testExpression(infix)
-	if err != nil {
-		t.Error(err)
-	}
-
-	test.AssertResultComplex(t, expectedOutput, actual)
-}
-
-func TestPostFixPipeEquals(t *testing.T) {
-	var infix = `.animals |   (. == "cat") `
-	var expectedOutput = `PathKey - animals
---------
-SELF
---------
-Value - cat (string)
---------
-Operation - EQUALS
---------
-Operation - PIPE
---------
-`
-
-	actual, err := testExpression(infix)
-	if err != nil {
-		t.Error(err)
-	}
-
-	test.AssertResultComplex(t, expectedOutput, actual)
-}
-
-func TestPostFixCollect(t *testing.T) {
-	var infix = "[.a]"
-	var expectedOutput = `PathKey - a
---------
-Operation - COLLECT
---------
-`
-
-	actual, err := testExpression(infix)
-	if err != nil {
-		t.Error(err)
-	}
-
-	test.AssertResultComplex(t, expectedOutput, actual)
-}
-
-func TestPostFixSplatSearch(t *testing.T) {
-	var infix = `.a | (.[].b == "apple")`
-	var expectedOutput = `PathKey - a
---------
-PathKey - []
---------
-PathKey - b
---------
-Operation - PIPE
---------
-Value - apple (string)
---------
-Operation - EQUALS
---------
-Operation - PIPE
---------
-`
-
-	actual, err := testExpression(infix)
-	if err != nil {
-		t.Error(err)
-	}
-
-	test.AssertResultComplex(t, expectedOutput, actual)
-}
-
-func TestPostFixCollectWithExpression(t *testing.T) {
-	var infix = `[ (.a == "fred") | (.d, .f)]`
-	var expectedOutput = `PathKey - a
---------
-Value - fred (string)
---------
-Operation - EQUALS
---------
-PathKey - d
---------
-PathKey - f
---------
-Operation - OR
---------
-Operation - PIPE
---------
-Operation - COLLECT
---------
-`
-
-	actual, err := testExpression(infix)
-	if err != nil {
-		t.Error(err)
-	}
-
-	test.AssertResultComplex(t, expectedOutput, actual)
-}
-
-func TestPostFixLength(t *testing.T) {
-	var infix = ".a | length"
-	var expectedOutput = `PathKey - a
---------
-Operation - LENGTH
---------
-Operation - PIPE
---------
-`
-
-	actual, err := testExpression(infix)
-	if err != nil {
-		t.Error(err)
-	}
-
-	test.AssertResultComplex(t, expectedOutput, actual)
-}
-
-func TestPostFixSimpleExample(t *testing.T) {
-	var infix = ".a"
-	var expectedOutput = `PathKey - a
---------
-`
-
-	actual, err := testExpression(infix)
-	if err != nil {
-		t.Error(err)
-	}
-
-	test.AssertResultComplex(t, expectedOutput, actual)
-}
-
-func TestPostFixSimplePathExample(t *testing.T) {
-	var infix = ".apples.bananas*.cat"
-	var expectedOutput = `PathKey - apples
---------
-PathKey - bananas*
---------
-Operation - PIPE
---------
-PathKey - cat
---------
-Operation - PIPE
---------
-`
-
-	actual, err := testExpression(infix)
-	if err != nil {
-		t.Error(err)
-	}
-
-	test.AssertResultComplex(t, expectedOutput, actual)
-}
-
-func TestPostFixSimpleAssign(t *testing.T) {
-	var infix = ".a.b |= \"frog\""
-	var expectedOutput = `PathKey - a
---------
-PathKey - b
---------
-Operation - PIPE
---------
-Value - frog (string)
---------
-Operation - ASSIGN
---------
-`
-
-	actual, err := testExpression(infix)
-	if err != nil {
-		t.Error(err)
-	}
-
-	test.AssertResultComplex(t, expectedOutput, actual)
-}
-
-func TestPostFixSimplePathNumbersExample(t *testing.T) {
-	var infix = ".apples[0].cat"
-	var expectedOutput = `PathKey - apples
---------
-PathKey - 0
---------
-Operation - PIPE
---------
-PathKey - cat
---------
-Operation - PIPE
---------
-`
-
-	actual, err := testExpression(infix)
-	if err != nil {
-		t.Error(err)
-	}
-
-	test.AssertResultComplex(t, expectedOutput, actual)
-}
-
-func TestPostFixSimplePathSplatArrayExample(t *testing.T) {
-	var infix = ".apples[].cat"
-	var expectedOutput = `PathKey - apples
---------
-PathKey - []
---------
-Operation - PIPE
---------
-PathKey - cat
---------
-Operation - PIPE
---------
-`
-
-	actual, err := testExpression(infix)
-	if err != nil {
-		t.Error(err)
-	}
-
-	test.AssertResultComplex(t, expectedOutput, actual)
-}
-
-func TestPostFixOrExample(t *testing.T) {
-	var infix = ".a, .b"
-	var expectedOutput = `PathKey - a
---------
-PathKey - b
---------
-Operation - OR
---------
-`
-
-	actual, err := testExpression(infix)
-	if err != nil {
-		t.Error(err)
-	}
-
-	test.AssertResultComplex(t, expectedOutput, actual)
-}
-
-func TestPostFixEqualsNumberExample(t *testing.T) {
-	var infix = ".animal == 3"
-	var expectedOutput = `PathKey - animal
---------
-Value - 3 (int64)
---------
-Operation - EQUALS
---------
-`
-
-	actual, err := testExpression(infix)
-	if err != nil {
-		t.Error(err)
-	}
-
-	test.AssertResultComplex(t, expectedOutput, actual)
-}
-
-func TestPostFixOrWithEqualsExample(t *testing.T) {
-	var infix = ".a==\"thing\", .b==.thongs"
-	var expectedOutput = `PathKey - a
---------
-Value - thing (string)
---------
-Operation - EQUALS
---------
-PathKey - b
---------
-PathKey - thongs
---------
-Operation - EQUALS
---------
-Operation - OR
---------
-`
-
-	actual, err := testExpression(infix)
-	if err != nil {
-		t.Error(err)
-	}
-
-	test.AssertResultComplex(t, expectedOutput, actual)
-}
-
-func TestPostFixOrWithEqualsPathExample(t *testing.T) {
-	var infix = ".apples.monkeys==\"thing\", .bogs.bobos==true"
-	var expectedOutput = `PathKey - apples
---------
-PathKey - monkeys
---------
-Operation - PIPE
---------
-Value - thing (string)
---------
-Operation - EQUALS
---------
-PathKey - bogs
---------
-PathKey - bobos
---------
-Operation - PIPE
---------
-Value - true (bool)
---------
-Operation - EQUALS
---------
-Operation - OR
---------
-`
-
-	actual, err := testExpression(infix)
-	if err != nil {
-		t.Error(err)
-	}
-
-	test.AssertResultComplex(t, expectedOutput, actual)
-}

pkg/yqlib/treeops/path_tokeniser.go

@@ -11,12 +11,21 @@ func skip(*lex.Scanner, *machines.Match) (interface{}, error) {
 	return nil, nil
 }
 
+type TokenType uint32
+
+const (
+	Operation = 1 << iota
+	OpenBracket
+	CloseBracket
+	OpenCollect
+	CloseCollect
+)
+
 type Token struct {
-	PathElementType PathElementType
+	TokenType     TokenType
 	OperationType *OperationType
 	Value         interface{}
 	StringValue   string
-	PrefixSelf      bool
 
 	CheckForPostTraverse bool // e.g. [1]cat should really be [1].cat
 }
@@ -28,7 +37,13 @@ func pathToken(wrapped bool) lex.Action {
 		if wrapped {
 			value = unwrap(value)
 		}
-		return &Token{PathElementType: PathKey, OperationType: None, Value: value, StringValue: value, CheckForPostTraverse: true}, nil
+		return &Token{TokenType: Operation, OperationType: TraversePath, Value: value, StringValue: value, CheckForPostTraverse: true}, nil
+	}
+}
+
+func literalPathToken(value string) lex.Action {
+	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
+		return &Token{TokenType: Operation, OperationType: TraversePath, Value: value, StringValue: value, CheckForPostTraverse: true}, nil
 	}
 }
 
@@ -40,20 +55,20 @@ func documentToken() lex.Action {
 		if errParsingInt != nil {
 			return nil, errParsingInt
 		}
-		return &Token{PathElementType: DocumentKey, OperationType: None, Value: number, StringValue: numberString, CheckForPostTraverse: true}, nil
+		return &Token{TokenType: Operation, OperationType: DocumentFilter, Value: number, StringValue: numberString, CheckForPostTraverse: true}, nil
 	}
 }
 
 func opToken(op *OperationType) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		value := string(m.Bytes)
-		return &Token{PathElementType: Operation, OperationType: op, Value: op.Type, StringValue: value}, nil
+		return &Token{TokenType: Operation, OperationType: op, Value: op.Type, StringValue: value}, nil
 	}
 }
 
-func literalToken(pType PathElementType, literal string, checkForPost bool) lex.Action {
+func literalToken(pType TokenType, literal string, checkForPost bool) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &Token{PathElementType: pType, OperationType: None, Value: literal, StringValue: literal, CheckForPostTraverse: checkForPost}, nil
+		return &Token{TokenType: Operation, OperationType: ValueOp, Value: literal, StringValue: literal, CheckForPostTraverse: checkForPost}, nil
 	}
 }
 
@@ -73,7 +88,7 @@ func arrayIndextoken(precedingDot bool) lex.Action {
 		if errParsingInt != nil {
 			return nil, errParsingInt
 		}
-		return &Token{PathElementType: PathKey, OperationType: None, Value: number, StringValue: numberString, CheckForPostTraverse: true}, nil
+		return &Token{TokenType: Operation, OperationType: TraversePath, Value: number, StringValue: numberString, CheckForPostTraverse: true}, nil
 	}
 }
 
@@ -84,7 +99,7 @@ func numberValue() lex.Action {
 		if errParsingInt != nil {
 			return nil, errParsingInt
 		}
-		return &Token{PathElementType: Value, OperationType: None, Value: number, StringValue: numberString}, nil
+		return &Token{TokenType: Operation, OperationType: ValueOp, Value: number, StringValue: numberString}, nil
 	}
 }
 
@@ -95,13 +110,13 @@ func floatValue() lex.Action {
 		if errParsingInt != nil {
 			return nil, errParsingInt
 		}
-		return &Token{PathElementType: Value, OperationType: None, Value: number, StringValue: numberString}, nil
+		return &Token{TokenType: Operation, OperationType: ValueOp, Value: number, StringValue: numberString}, nil
 	}
 }
 
 func booleanValue(val bool) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &Token{PathElementType: Value, OperationType: None, Value: val, StringValue: string(m.Bytes)}, nil
+		return &Token{TokenType: Operation, OperationType: ValueOp, Value: val, StringValue: string(m.Bytes)}, nil
 	}
 }
 
@@ -111,13 +126,13 @@ func stringValue(wrapped bool) lex.Action {
 		if wrapped {
 			value = unwrap(value)
 		}
-		return &Token{PathElementType: Value, OperationType: None, Value: value, StringValue: value}, nil
+		return &Token{TokenType: Operation, OperationType: ValueOp, Value: value, StringValue: value}, nil
 	}
 }
 
 func selfToken() lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &Token{PathElementType: SelfReference, OperationType: None, Value: "SELF", StringValue: "SELF"}, nil
+		return &Token{TokenType: Operation, OperationType: SelfReference}, nil
 	}
 }
 
@@ -127,7 +142,7 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`\(`), literalToken(OpenBracket, "(", false))
 	lexer.Add([]byte(`\)`), literalToken(CloseBracket, ")", true))
 
-	lexer.Add([]byte(`\.?\[\]`), literalToken(PathKey, "[]", true))
+	lexer.Add([]byte(`\.?\[\]`), literalPathToken("[]"))
 	lexer.Add([]byte(`\.\.`), opToken(RecursiveDescent))
 
 	lexer.Add([]byte(`,`), opToken(Union))
@@ -218,8 +233,8 @@ func (p *pathTokeniser) Tokenise(path string) ([]*Token, error) {
 		postProcessedTokens = append(postProcessedTokens, token)
 
 		if index != len(tokens)-1 && token.CheckForPostTraverse &&
-			tokens[index+1].PathElementType == PathKey {
+			tokens[index+1].OperationType == TraversePath {
-			postProcessedTokens = append(postProcessedTokens, &Token{PathElementType: Operation, OperationType: Pipe, Value: "PIPE"})
+			postProcessedTokens = append(postProcessedTokens, &Token{TokenType: Operation, OperationType: Pipe, Value: "PIPE"})
 		}
 	}
 

pkg/yqlib/treeops/path_tree.go

@@ -46,7 +46,7 @@ func (p *pathTreeCreator) CreatePathTree(postFixPath []*PathElement) (*PathTreeN
 	for _, pathElement := range postFixPath {
 		var newNode = PathTreeNode{PathElement: pathElement}
 		log.Debugf("pathTree %v ", pathElement.toString())
-		if pathElement.PathElementType == Operation && pathElement.OperationType.NumArgs > 0 {
+		if pathElement.OperationType.NumArgs > 0 {
 			numArgs := pathElement.OperationType.NumArgs
 			if numArgs == 1 {
 				remaining, rhs := stack[:len(stack)-1], stack[len(stack)-1]

release_instructions.txt

@@ -12,7 +12,6 @@
 
 - snapcraft
   - will auto create a candidate, test it works then promote
-  - see https://build.snapcraft.io/user/mikefarah/yq
   
   sudo snap remove yq
   sudo snap install --edge yq