simplified, refactored

pkg/yqlib/treeops/data_tree_navigator.go

@@ -48,17 +48,17 @@ func (d *dataTreeNavigator) getMatchingNodes(matchingNodes *orderedmap.OrderedMa
 		log.Debugf("getMatchingNodes - nothing to do")
 		return matchingNodes, nil
 	}
-	log.Debugf("Processing Path: %v", pathNode.PathElement.toString())
+	log.Debugf("Processing Op: %v", pathNode.Operation.toString())
 	if log.IsEnabledFor(logging.DEBUG) {
 		for el := matchingNodes.Front(); el != nil; el = el.Next() {
 			log.Debug(NodeToString(el.Value.(*CandidateNode)))
 		}
 	}
 	log.Debug(">>")
-	handler := pathNode.PathElement.OperationType.Handler
+	handler := pathNode.Operation.OperationType.Handler
 	if handler != nil {
 		return handler(d, matchingNodes, pathNode)
 	}
-	return nil, fmt.Errorf("Unknown operator %v", pathNode.PathElement.OperationType)
+	return nil, fmt.Errorf("Unknown operator %v", pathNode.Operation.OperationType)
 
 }

pkg/yqlib/treeops/lib.go

@@ -36,8 +36,8 @@ var Collect = &OperationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handle
 var TraversePath = &OperationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
 
 var DocumentFilter = &OperationType{Type: "DOCUMENT_FILTER", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
-var SelfReference = &OperationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
+var SelfReference = &OperationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: SelfOperator}
-var ValueOp = &OperationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
+var ValueOp = &OperationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: ValueOperator}
 
 var RecursiveDescent = &OperationType{Type: "RECURSIVE_DESCENT", NumArgs: 0, Precedence: 50, Handler: RecursiveDescentOperator}
 
@@ -52,15 +52,38 @@ var DeleteChild = &OperationType{Type: "DELETE", NumArgs: 2, Precedence: 40, Han
 // var Exists = &OperationType{Type: "Length", NumArgs: 2, Precedence: 35}
 // filters matches if they have the existing path
 
-type PathElement struct {
+type Operation struct {
 	OperationType *OperationType
 	Value         interface{}
 	StringValue   string
 	CandidateNode *CandidateNode // used for Value Path elements
 }
 
+func CreateValueOperation(value interface{}, stringValue string) *Operation {
+	var node yaml.Node = yaml.Node{Kind: yaml.ScalarNode}
+	node.Value = stringValue
+
+	switch value.(type) {
+	case float32, float64:
+		node.Tag = "!!float"
+	case int, int64, int32:
+		node.Tag = "!!int"
+	case bool:
+		node.Tag = "!!bool"
+	case string:
+		node.Tag = "!!str"
+	}
+
+	return &Operation{
+		OperationType: ValueOp,
+		Value:         value,
+		StringValue:   stringValue,
+		CandidateNode: &CandidateNode{Node: &node},
+	}
+}
+
 // debugging purposes only
-func (p *PathElement) toString() string {
+func (p *Operation) toString() string {
 	if p.OperationType == TraversePath {
 		return fmt.Sprintf("%v", p.Value)
 	} else if p.OperationType == DocumentFilter {

pkg/yqlib/treeops/operator_equals.go

@@ -34,7 +34,7 @@ func hasMatch(d *dataTreeNavigator, candidate *CandidateNode, lhs *PathTreeNode,
 	}
 
 	// TODO = handle other RHS types
-	return containsMatchingValue(childMatches, rhs.PathElement.StringValue), nil
+	return containsMatchingValue(childMatches, rhs.Operation.StringValue), nil
 }
 
 func containsMatchingValue(matchMap *orderedmap.OrderedMap, valuePattern string) bool {

pkg/yqlib/treeops/operator_multilpy.go

@@ -61,12 +61,12 @@ func multiply(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*Ca
 
 func createTraversalTree(path []interface{}) *PathTreeNode {
 	if len(path) == 0 {
-		return &PathTreeNode{PathElement: &PathElement{OperationType: SelfReference}}
+		return &PathTreeNode{Operation: &Operation{OperationType: SelfReference}}
 	} else if len(path) == 1 {
-		return &PathTreeNode{PathElement: &PathElement{OperationType: TraversePath, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}}
+		return &PathTreeNode{Operation: &Operation{OperationType: TraversePath, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}}
 	}
 	return &PathTreeNode{
-		PathElement: &PathElement{OperationType: Pipe},
+		Operation: &Operation{OperationType: Pipe},
 		Lhs:         createTraversalTree(path[0:1]),
 		Rhs:         createTraversalTree(path[1:])}
 
@@ -77,13 +77,13 @@ func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *Candid
 
 	lhsPath := rhs.Path[pathIndexToStartFrom:]
 
-	assignmentOp := &PathElement{OperationType: AssignAttributes}
+	assignmentOp := &Operation{OperationType: AssignAttributes}
 	if rhs.Node.Kind == yaml.ScalarNode {
 		assignmentOp.OperationType = Assign
 	}
-	rhsOp := &PathElement{OperationType: ValueOp, CandidateNode: rhs}
+	rhsOp := &Operation{OperationType: ValueOp, CandidateNode: rhs}
 
-	assignmentOpNode := &PathTreeNode{PathElement: assignmentOp, Lhs: createTraversalTree(lhsPath), Rhs: &PathTreeNode{PathElement: rhsOp}}
+	assignmentOpNode := &PathTreeNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath), Rhs: &PathTreeNode{Operation: rhsOp}}
 
 	_, err := d.getMatchingNodes(nodeToMap(lhs), assignmentOpNode)
 

pkg/yqlib/treeops/operator_recursive_descent.go

@@ -16,8 +16,8 @@ func RecursiveDescentOperator(d *dataTreeNavigator, matchMap *orderedmap.Ordered
 }
 
 func recursiveDecent(d *dataTreeNavigator, results *orderedmap.OrderedMap, matchMap *orderedmap.OrderedMap) error {
-	splatPathElement := &PathElement{OperationType: TraversePath, Value: "[]"}
+	splatOperation := &Operation{OperationType: TraversePath, Value: "[]"}
-	splatTreeNode := &PathTreeNode{PathElement: splatPathElement}
+	splatTreeNode := &PathTreeNode{Operation: splatOperation}
 
 	for el := matchMap.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)

pkg/yqlib/treeops/operator_self.go

@@ -0,0 +1,7 @@
+package treeops
+
+import "github.com/elliotchance/orderedmap"
+
+func SelfOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
+	return matchMap, nil
+}

pkg/yqlib/treeops/operator_traverse_path.go

@@ -14,7 +14,7 @@ func TraversePathOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap,
 	var err error
 
 	for el := matchMap.Front(); el != nil; el = el.Next() {
-		newNodes, err = traverse(d, el.Value.(*CandidateNode), pathNode.PathElement)
+		newNodes, err = traverse(d, el.Value.(*CandidateNode), pathNode.Operation)
 		if err != nil {
 			return nil, err
 		}
@@ -26,7 +26,7 @@ func TraversePathOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap,
 	return matchingNodeMap, nil
 }
 
-func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) {
+func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, pathNode *Operation) ([]*CandidateNode, error) {
 	log.Debug("Traversing %v", NodeToString(matchingNode))
 	value := matchingNode.Node
 
@@ -78,11 +78,11 @@ func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, pathNode *PathE
 	}
 }
 
-func keyMatches(key *yaml.Node, pathNode *PathElement) bool {
+func keyMatches(key *yaml.Node, pathNode *Operation) bool {
 	return pathNode.Value == "[]" || Match(key.Value, pathNode.StringValue)
 }
 
-func traverseMap(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) {
+func traverseMap(candidate *CandidateNode, pathNode *Operation) ([]*CandidateNode, error) {
 	// value.Content is a concatenated array of key, value,
 	// so keys are in the even indexes, values in odd.
 	// merge aliases are defined first, but we only want to traverse them
@@ -123,7 +123,7 @@ func traverseMap(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateN
 	return newMatches, nil
 }
 
-func traverseArray(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) {
+func traverseArray(candidate *CandidateNode, pathNode *Operation) ([]*CandidateNode, error) {
 	log.Debug("pathNode Value %v", pathNode.Value)
 	if pathNode.Value == "[]" {
 

pkg/yqlib/treeops/operator_traverse_path_test.go

@@ -12,6 +12,60 @@ var traversePathOperatorScenarios = []expressionScenario{
 			"D0, P[a], (!!map)::{b: apple}\n",
 		},
 	},
+	{
+		document:   `[{b: apple}, {c: banana}]`,
+		expression: `.[]`,
+		expected: []string{
+			"D0, P[0], (!!map)::{b: apple}\n",
+			"D0, P[1], (!!map)::{c: banana}\n",
+		},
+	},
+	{
+		document:   `{}`,
+		expression: `.a.b`,
+		expected: []string{
+			"D0, P[a b], ()::null\n",
+		},
+	},
+	{
+		document:   `{}`,
+		expression: `.[1].a`,
+		expected: []string{
+			"D0, P[1 a], ()::null\n",
+		},
+	},
+	{
+		document:   `{}`,
+		expression: `.a.[1]`,
+		expected: []string{
+			"D0, P[a 1], ()::null\n",
+		},
+	},
+	{
+		document:   `{a: {cat: apple, mad: things}}`,
+		expression: `.a."*a*"`,
+		expected: []string{
+			"D0, P[a cat], (!!str)::apple\n",
+			"D0, P[a mad], (!!str)::things\n",
+		},
+	},
+	{
+		document:   `{a: {cat: {b: 3}, mad: {b: 4}, fad: {c: t}}}`,
+		expression: `.a."*a*".b`,
+		expected: []string{
+			"D0, P[a cat b], (!!int)::3\n",
+			"D0, P[a mad b], (!!int)::4\n",
+			"D0, P[a fad b], ()::null\n",
+		},
+	},
+	{
+		document:   `{a: {cat: apple, mad: things}}`,
+		expression: `.a | (.cat, .mad)`,
+		expected: []string{
+			"D0, P[a cat], (!!str)::apple\n",
+			"D0, P[a mad], (!!str)::things\n",
+		},
+	},
 }
 
 func TestTraversePathOperatorScenarios(t *testing.T) {

pkg/yqlib/treeops/operator_value.go

@@ -0,0 +1,7 @@
+package treeops
+
+import "github.com/elliotchance/orderedmap"
+
+func ValueOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
+	return nodeToMap(pathNode.Operation.CandidateNode), nil
+}

pkg/yqlib/treeops/operator_value_test.go

@@ -11,7 +11,8 @@ var valueOperatorScenarios = []expressionScenario{
 		expected: []string{
 			"D0, P[], (!!int)::1\n",
 		},
-	}, {
+	},
+	{
 		document:   ``,
 		expression: `-1`,
 		expected: []string{

pkg/yqlib/treeops/path_parse_test.go

@@ -41,22 +41,22 @@ var pathTests = []struct {
 	// {`.[].a`, append(make([]interface{}, 0), "[]", "PIPE", "a")},
 	{
 		`d0.a`,
-		append(make([]interface{}, 0), int64(0), "PIPE", "a"),
+		append(make([]interface{}, 0), "d0", "PIPE", "a"),
-		append(make([]interface{}, 0), "D0", "a", "PIPE"),
+		append(make([]interface{}, 0), "d0", "a", "PIPE"),
 	},
 	{
 		`.a | (.[].b == "apple")`,
-		append(make([]interface{}, 0), "a", "PIPE", "(", "[]", "PIPE", "b", "EQUALS", "apple", ")"),
+		append(make([]interface{}, 0), "a", "PIPE", "(", "[]", "PIPE", "b", "EQUALS", "apple (string)", ")"),
 		append(make([]interface{}, 0), "a", "[]", "b", "PIPE", "apple (string)", "EQUALS", "PIPE"),
 	},
 	{
 		`.[] | select(. == "*at")`,
-		append(make([]interface{}, 0), "[]", "PIPE", "SELECT", "(", "SELF", "EQUALS", "*at", ")"),
+		append(make([]interface{}, 0), "[]", "PIPE", "SELECT", "(", "SELF", "EQUALS", "*at (string)", ")"),
 		append(make([]interface{}, 0), "[]", "SELF", "*at (string)", "EQUALS", "SELECT", "PIPE"),
 	},
 	{
 		`[true]`,
-		append(make([]interface{}, 0), "[", true, "]"),
+		append(make([]interface{}, 0), "[", "true (bool)", "]"),
 		append(make([]interface{}, 0), "true (bool)", "COLLECT", "PIPE"),
 	},
 
@@ -91,7 +91,7 @@ func TestPathParsing(t *testing.T) {
 		}
 		var tokenValues []interface{}
 		for _, token := range tokens {
-			tokenValues = append(tokenValues, token.Value)
+			tokenValues = append(tokenValues, token.toString())
 		}
 		test.AssertResultComplexWithContext(t, tt.expectedTokens, tokenValues, fmt.Sprintf("tokenise: %v", tt.path))
 

pkg/yqlib/treeops/path_postfix.go

@@ -7,7 +7,7 @@ import (
 )
 
 type PathPostFixer interface {
-	ConvertToPostfix([]*Token) ([]*PathElement, error)
+	ConvertToPostfix([]*Token) ([]*Operation, error)
 }
 
 type pathPostFixer struct {
@@ -17,27 +17,20 @@ func NewPathPostFixer() PathPostFixer {
 	return &pathPostFixer{}
 }
 
-func popOpToResult(opStack []*Token, result []*PathElement) ([]*Token, []*PathElement) {
+func popOpToResult(opStack []*Token, result []*Operation) ([]*Token, []*Operation) {
 	var newOp *Token
 	opStack, newOp = opStack[0:len(opStack)-1], opStack[len(opStack)-1]
-	var pathElement = PathElement{OperationType: newOp.OperationType, Value: newOp.Value, StringValue: newOp.StringValue}
+	return opStack, append(result, newOp.Operation)
-
-	if newOp.OperationType == ValueOp {
-		var candidateNode = BuildCandidateNodeFrom(newOp)
-		pathElement.CandidateNode = candidateNode
-	}
-
-	return opStack, append(result, &pathElement)
 }
 
-func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*PathElement, error) {
+func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*Operation, error) {
-	var result []*PathElement
+	var result []*Operation
 	// surround the whole thing with quotes
-	var opStack = []*Token{&Token{TokenType: OpenBracket, Value: "("}}
+	var opStack = []*Token{&Token{TokenType: OpenBracket}}
-	var tokens = append(infixTokens, &Token{TokenType: CloseBracket, Value: ")"})
+	var tokens = append(infixTokens, &Token{TokenType: CloseBracket})
 
 	for _, token := range tokens {
-		log.Debugf("postfix processing token %v", token.Value)
+		log.Debugf("postfix processing token %v, %v", token.toString(), token.Operation)
 		switch token.TokenType {
 		case OpenBracket, OpenCollect:
 			opStack = append(opStack, token)
@@ -51,8 +44,8 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*PathElement,
 			// now we should have [] as the last element on the opStack, get rid of it
 			opStack = opStack[0 : len(opStack)-1]
 			//and append a collect to the opStack
-			opStack = append(opStack, &Token{TokenType: Operation, OperationType: Pipe})
+			opStack = append(opStack, &Token{TokenType: OperationToken, Operation: &Operation{OperationType: Pipe}})
-			opStack = append(opStack, &Token{TokenType: Operation, OperationType: Collect})
+			opStack = append(opStack, &Token{TokenType: OperationToken, Operation: &Operation{OperationType: Collect}})
 		case CloseBracket:
 			for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != OpenBracket {
 				opStack, result = popOpToResult(opStack, result)
@@ -64,9 +57,11 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*PathElement,
 			opStack = opStack[0 : len(opStack)-1]
 
 		default:
-			var currentPrecedence = token.OperationType.Precedence
+			var currentPrecedence = token.Operation.OperationType.Precedence
 			// pop off higher precedent operators onto the result
-			for len(opStack) > 0 && opStack[len(opStack)-1].OperationType.Precedence >= currentPrecedence {
+			for len(opStack) > 0 &&
+				opStack[len(opStack)-1].TokenType == OperationToken &&
+				opStack[len(opStack)-1].Operation.OperationType.Precedence >= currentPrecedence {
 				opStack, result = popOpToResult(opStack, result)
 			}
 			// add this operator to the opStack

pkg/yqlib/treeops/path_tokeniser.go

@@ -1,6 +1,7 @@
 package treeops
 
 import (
+	"fmt"
 	"strconv"
 
 	lex "github.com/timtadh/lexmachine"
@@ -14,7 +15,7 @@ func skip(*lex.Scanner, *machines.Match) (interface{}, error) {
 type TokenType uint32
 
 const (
-	Operation = 1 << iota
+	OperationToken = 1 << iota
 	OpenBracket
 	CloseBracket
 	OpenCollect
@@ -22,14 +23,28 @@ const (
 )
 
 type Token struct {
-	TokenType     TokenType
+	TokenType TokenType
-	OperationType *OperationType
+	Operation *Operation
-	Value         interface{}
-	StringValue   string
 
 	CheckForPostTraverse bool // e.g. [1]cat should really be [1].cat
 }
 
+func (t *Token) toString() string {
+	if t.TokenType == OperationToken {
+		return t.Operation.toString()
+	} else if t.TokenType == OpenBracket {
+		return "("
+	} else if t.TokenType == CloseBracket {
+		return ")"
+	} else if t.TokenType == OpenCollect {
+		return "["
+	} else if t.TokenType == CloseCollect {
+		return "]"
+	} else {
+		return fmt.Sprintf("NFI")
+	}
+}
+
 func pathToken(wrapped bool) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		value := string(m.Bytes)
@@ -37,13 +52,15 @@ func pathToken(wrapped bool) lex.Action {
 		if wrapped {
 			value = unwrap(value)
 		}
-		return &Token{TokenType: Operation, OperationType: TraversePath, Value: value, StringValue: value, CheckForPostTraverse: true}, nil
+		op := &Operation{OperationType: TraversePath, Value: value, StringValue: value}
+		return &Token{TokenType: OperationToken, Operation: op, CheckForPostTraverse: true}, nil
 	}
 }
 
 func literalPathToken(value string) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &Token{TokenType: Operation, OperationType: TraversePath, Value: value, StringValue: value, CheckForPostTraverse: true}, nil
+		op := &Operation{OperationType: TraversePath, Value: value, StringValue: value}
+		return &Token{TokenType: OperationToken, Operation: op, CheckForPostTraverse: true}, nil
 	}
 }
 
@@ -55,20 +72,22 @@ func documentToken() lex.Action {
 		if errParsingInt != nil {
 			return nil, errParsingInt
 		}
-		return &Token{TokenType: Operation, OperationType: DocumentFilter, Value: number, StringValue: numberString, CheckForPostTraverse: true}, nil
+		op := &Operation{OperationType: DocumentFilter, Value: number, StringValue: numberString}
+		return &Token{TokenType: OperationToken, Operation: op, CheckForPostTraverse: true}, nil
 	}
 }
 
 func opToken(op *OperationType) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		value := string(m.Bytes)
-		return &Token{TokenType: Operation, OperationType: op, Value: op.Type, StringValue: value}, nil
+		op := &Operation{OperationType: op, Value: op.Type, StringValue: value}
+		return &Token{TokenType: OperationToken, Operation: op}, nil
 	}
 }
 
-func literalToken(pType TokenType, literal string, checkForPost bool) lex.Action {
+func literalToken(pType TokenType, checkForPost bool) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &Token{TokenType: Operation, OperationType: ValueOp, Value: literal, StringValue: literal, CheckForPostTraverse: checkForPost}, nil
+		return &Token{TokenType: pType, CheckForPostTraverse: checkForPost}, nil
 	}
 }
 
@@ -88,7 +107,8 @@ func arrayIndextoken(precedingDot bool) lex.Action {
 		if errParsingInt != nil {
 			return nil, errParsingInt
 		}
-		return &Token{TokenType: Operation, OperationType: TraversePath, Value: number, StringValue: numberString, CheckForPostTraverse: true}, nil
+		op := &Operation{OperationType: TraversePath, Value: number, StringValue: numberString}
+		return &Token{TokenType: OperationToken, Operation: op, CheckForPostTraverse: true}, nil
 	}
 }
 
@@ -99,7 +119,8 @@ func numberValue() lex.Action {
 		if errParsingInt != nil {
 			return nil, errParsingInt
 		}
-		return &Token{TokenType: Operation, OperationType: ValueOp, Value: number, StringValue: numberString}, nil
+
+		return &Token{TokenType: OperationToken, Operation: CreateValueOperation(number, numberString)}, nil
 	}
 }
 
@@ -110,13 +131,13 @@ func floatValue() lex.Action {
 		if errParsingInt != nil {
 			return nil, errParsingInt
 		}
-		return &Token{TokenType: Operation, OperationType: ValueOp, Value: number, StringValue: numberString}, nil
+		return &Token{TokenType: OperationToken, Operation: CreateValueOperation(number, numberString)}, nil
 	}
 }
 
 func booleanValue(val bool) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &Token{TokenType: Operation, OperationType: ValueOp, Value: val, StringValue: string(m.Bytes)}, nil
+		return &Token{TokenType: OperationToken, Operation: CreateValueOperation(val, string(m.Bytes))}, nil
 	}
 }
 
@@ -126,21 +147,22 @@ func stringValue(wrapped bool) lex.Action {
 		if wrapped {
 			value = unwrap(value)
 		}
-		return &Token{TokenType: Operation, OperationType: ValueOp, Value: value, StringValue: value}, nil
+		return &Token{TokenType: OperationToken, Operation: CreateValueOperation(value, value)}, nil
 	}
 }
 
 func selfToken() lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
-		return &Token{TokenType: Operation, OperationType: SelfReference}, nil
+		op := &Operation{OperationType: SelfReference}
+		return &Token{TokenType: OperationToken, Operation: op}, nil
 	}
 }
 
 // Creates the lexer object and compiles the NFA.
 func initLexer() (*lex.Lexer, error) {
 	lexer := lex.NewLexer()
-	lexer.Add([]byte(`\(`), literalToken(OpenBracket, "(", false))
+	lexer.Add([]byte(`\(`), literalToken(OpenBracket, false))
-	lexer.Add([]byte(`\)`), literalToken(CloseBracket, ")", true))
+	lexer.Add([]byte(`\)`), literalToken(CloseBracket, true))
 
 	lexer.Add([]byte(`\.?\[\]`), literalPathToken("[]"))
 	lexer.Add([]byte(`\.\.`), opToken(RecursiveDescent))
@@ -180,8 +202,8 @@ func initLexer() (*lex.Lexer, error) {
 
 	lexer.Add([]byte(`"[^ "]+"`), stringValue(true))
 
-	lexer.Add([]byte(`\[`), literalToken(OpenCollect, "[", false))
+	lexer.Add([]byte(`\[`), literalToken(OpenCollect, false))
-	lexer.Add([]byte(`\]`), literalToken(CloseCollect, "]", true))
+	lexer.Add([]byte(`\]`), literalToken(CloseCollect, true))
 	lexer.Add([]byte(`\*`), opToken(Multiply))
 
 	// lexer.Add([]byte(`[^ \,\|\.\[\(\)=]+`), stringValue(false))
@@ -219,7 +241,7 @@ func (p *pathTokeniser) Tokenise(path string) ([]*Token, error) {
 
 		if tok != nil {
 			token := tok.(*Token)
-			log.Debugf("Tokenising %v - %v", token.Value, token.OperationType.Type)
+			log.Debugf("Tokenising %v", token.toString())
 			tokens = append(tokens, token)
 		}
 		if err != nil {
@@ -233,8 +255,10 @@ func (p *pathTokeniser) Tokenise(path string) ([]*Token, error) {
 		postProcessedTokens = append(postProcessedTokens, token)
 
 		if index != len(tokens)-1 && token.CheckForPostTraverse &&
-			tokens[index+1].OperationType == TraversePath {
+			tokens[index+1].TokenType == OperationToken &&
-			postProcessedTokens = append(postProcessedTokens, &Token{TokenType: Operation, OperationType: Pipe, Value: "PIPE"})
+			tokens[index+1].Operation.OperationType == TraversePath {
+			op := &Operation{OperationType: Pipe, Value: "PIPE"}
+			postProcessedTokens = append(postProcessedTokens, &Token{TokenType: OperationToken, Operation: op})
 		}
 	}
 

pkg/yqlib/treeops/path_tree.go

@@ -6,14 +6,14 @@ var myPathTokeniser = NewPathTokeniser()
 var myPathPostfixer = NewPathPostFixer()
 
 type PathTreeNode struct {
-	PathElement *PathElement
+	Operation *Operation
 	Lhs         *PathTreeNode
 	Rhs         *PathTreeNode
 }
 
 type PathTreeCreator interface {
 	ParsePath(path string) (*PathTreeNode, error)
-	CreatePathTree(postFixPath []*PathElement) (*PathTreeNode, error)
+	CreatePathTree(postFixPath []*Operation) (*PathTreeNode, error)
 }
 
 type pathTreeCreator struct {
@@ -28,26 +28,26 @@ func (p *pathTreeCreator) ParsePath(path string) (*PathTreeNode, error) {
 	if err != nil {
 		return nil, err
 	}
-	var pathElements []*PathElement
+	var Operations []*Operation
-	pathElements, err = myPathPostfixer.ConvertToPostfix(tokens)
+	Operations, err = myPathPostfixer.ConvertToPostfix(tokens)
 	if err != nil {
 		return nil, err
 	}
-	return p.CreatePathTree(pathElements)
+	return p.CreatePathTree(Operations)
 }
 
-func (p *pathTreeCreator) CreatePathTree(postFixPath []*PathElement) (*PathTreeNode, error) {
+func (p *pathTreeCreator) CreatePathTree(postFixPath []*Operation) (*PathTreeNode, error) {
 	var stack = make([]*PathTreeNode, 0)
 
 	if len(postFixPath) == 0 {
 		return nil, nil
 	}
 
-	for _, pathElement := range postFixPath {
+	for _, Operation := range postFixPath {
-		var newNode = PathTreeNode{PathElement: pathElement}
+		var newNode = PathTreeNode{Operation: Operation}
-		log.Debugf("pathTree %v ", pathElement.toString())
+		log.Debugf("pathTree %v ", Operation.toString())
-		if pathElement.OperationType.NumArgs > 0 {
+		if Operation.OperationType.NumArgs > 0 {
-			numArgs := pathElement.OperationType.NumArgs
+			numArgs := Operation.OperationType.NumArgs
 			if numArgs == 1 {
 				remaining, rhs := stack[:len(stack)-1], stack[len(stack)-1]
 				newNode.Rhs = rhs

pkg/yqlib/treeops/sample.yaml

@@ -1 +1 @@
-{a: {b: apple, c: cactus}}
+{a: {cat: apple, mad: things}}

pkg/yqlib/treeops/temp.json

@@ -1,4 +1,6 @@
 {
-  "a": [1,2],
+  "a": {
-  "b": [3,4]
+    "cat": "apple",
-}
+    "mad": "things"
+  }
+}

pkg/yqlib/treeops/value_node_builder.go

@@ -1,20 +0,0 @@
-package treeops
-
-import "gopkg.in/yaml.v3"
-
-func BuildCandidateNodeFrom(token *Token) *CandidateNode {
-	var node yaml.Node = yaml.Node{Kind: yaml.ScalarNode}
-	node.Value = token.StringValue
-
-	switch token.Value.(type) {
-	case float32, float64:
-		node.Tag = "!!float"
-	case int, int64, int32:
-		node.Tag = "!!int"
-	case bool:
-		node.Tag = "!!bool"
-	case string:
-		node.Tag = "!!str"
-	}
-	return &CandidateNode{Node: &node}
-}