suyiiyii
/
yq
mirror of https://github.com/mikefarah/yq.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

wip

This commit is contained in:
Mike Farah 2020-10-20 13:53:26 +11:00
parent 4bc98776a6
commit 73cf6224f2
12 changed files with 180 additions and 534 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
pkg/yqlib/treeops/data_tree_navigator.go
View File

@ -8,7 +8,7 @@ import (
)
type dataTreeNavigator struct {
leafTraverser LeafTraverser
navigationPrefs NavigationPrefs
}
type NavigationPrefs struct {
@ -20,27 +20,7 @@ type DataTreeNavigator interface {
}
func NewDataTreeNavigator(navigationPrefs NavigationPrefs) DataTreeNavigator {
leafTraverser := NewLeafTraverser(navigationPrefs)
return &dataTreeNavigator{leafTraverser}
}
func (d *dataTreeNavigator) traverse(matchMap *orderedmap.OrderedMap, pathNode *PathElement) (*orderedmap.OrderedMap, error) {
log.Debugf("-- Traversing")
var matchingNodeMap = orderedmap.NewOrderedMap()
var newNodes []*CandidateNode
var err error
for el := matchMap.Front(); el != nil; el = el.Next() {
newNodes, err = d.leafTraverser.Traverse(el.Value.(*CandidateNode), pathNode)
if err != nil {
return nil, err
}
for _, n := range newNodes {
matchingNodeMap.Set(n.GetKey(), n)
}
}
return matchingNodeMap, nil
return &dataTreeNavigator{navigationPrefs}
}
func (d *dataTreeNavigator) GetMatchingNodes(matchingNodes []*CandidateNode, pathNode *PathTreeNode) ([]*CandidateNode, error) {
@ -75,19 +55,10 @@ func (d *dataTreeNavigator) getMatchingNodes(matchingNodes *orderedmap.OrderedMa
}
}
log.Debug(">>")
if pathNode.PathElement.PathElementType == SelfReference {
return matchingNodes, nil
} else if pathNode.PathElement.PathElementType == PathKey {
return d.traverse(matchingNodes, pathNode.PathElement)
} else if pathNode.PathElement.PathElementType == Value {
return nodeToMap(pathNode.PathElement.CandidateNode), nil
} else {
handler := pathNode.PathElement.OperationType.Handler
if handler != nil {
return handler(d, matchingNodes, pathNode)
}
return nil, fmt.Errorf("Unknown operator %v", pathNode.PathElement.OperationType)
handler := pathNode.PathElement.OperationType.Handler
if handler != nil {
return handler(d, matchingNodes, pathNode)
}
return nil, fmt.Errorf("Unknown operator %v", pathNode.PathElement.OperationType)
}

56
pkg/yqlib/treeops/lib.go
View File

@ -11,20 +11,6 @@ import (
var log = logging.MustGetLogger("yq-treeops")
type PathElementType uint32
const (
PathKey PathElementType = 1 << iota
DocumentKey
Operation
SelfReference
OpenBracket
CloseBracket
OpenCollect
CloseCollect
Value // e.g. string, int
)
type OperationType struct {
Type string
NumArgs uint // number of arguments to the op
@ -32,8 +18,6 @@ type OperationType struct {
Handler OperatorHandler
}
var None = &OperationType{Type: "NONE", NumArgs: 0, Precedence: 0}
var Or = &OperationType{Type: "OR", NumArgs: 2, Precedence: 20, Handler: OrOperator}
var And = &OperationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: AndOperator}
@ -49,6 +33,12 @@ var Pipe = &OperationType{Type: "PIPE", NumArgs: 2, Precedence: 45, Handler: Pip
var Length = &OperationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: LengthOperator}
var Collect = &OperationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: CollectOperator}
var TraversePath = &OperationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
var DocumentFilter = &OperationType{Type: "DOCUMENT_FILTER", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
var SelfReference = &OperationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
var ValueOp = &OperationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator}
var RecursiveDescent = &OperationType{Type: "RECURSIVE_DESCENT", NumArgs: 0, Precedence: 50, Handler: RecursiveDescentOperator}
// not sure yet
@ -63,31 +53,25 @@ var DeleteChild = &OperationType{Type: "DELETE", NumArgs: 2, Precedence: 40, Han
// filters matches if they have the existing path
type PathElement struct {
PathElementType PathElementType
OperationType *OperationType
Value interface{}
StringValue string
CandidateNode *CandidateNode // used for Value Path elements
OperationType *OperationType
Value interface{}
StringValue string
CandidateNode *CandidateNode // used for Value Path elements
}
// debugging purposes only
func (p *PathElement) toString() string {
var result string = ``
switch p.PathElementType {
case PathKey:
result = result + fmt.Sprintf("%v", p.Value)
case DocumentKey:
result = result + fmt.Sprintf("D%v", p.Value)
case SelfReference:
result = result + fmt.Sprintf("SELF")
case Operation:
result = result + fmt.Sprintf("%v", p.OperationType.Type)
case Value:
result = result + fmt.Sprintf("%v (%T)", p.Value, p.Value)
default:
result = result + "I HAVENT GOT A STRATEGY"
if p.OperationType == TraversePath {
return fmt.Sprintf("%v", p.Value)
} else if p.OperationType == DocumentFilter {
return fmt.Sprintf("d%v", p.Value)
} else if p.OperationType == SelfReference {
return "SELF"
} else if p.OperationType == ValueOp {
return fmt.Sprintf("%v (%T)", p.Value, p.Value)
} else {
return fmt.Sprintf("%v", p.OperationType.Type)
}
return result
}
type YqTreeLib interface {

10
pkg/yqlib/treeops/operator_multilpy.go
View File

@ -61,12 +61,12 @@ func multiply(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*Ca
func createTraversalTree(path []interface{}) *PathTreeNode {
if len(path) == 0 {
return &PathTreeNode{PathElement: &PathElement{PathElementType: SelfReference}}
return &PathTreeNode{PathElement: &PathElement{OperationType: SelfReference}}
} else if len(path) == 1 {
return &PathTreeNode{PathElement: &PathElement{PathElementType: PathKey, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}}
return &PathTreeNode{PathElement: &PathElement{OperationType: TraversePath, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}}
}
return &PathTreeNode{
PathElement: &PathElement{PathElementType: Operation, OperationType: Pipe},
PathElement: &PathElement{OperationType: Pipe},
Lhs: createTraversalTree(path[0:1]),
Rhs: createTraversalTree(path[1:])}
@ -77,11 +77,11 @@ func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *Candid
lhsPath := rhs.Path[pathIndexToStartFrom:]
assignmentOp := &PathElement{PathElementType: Operation, OperationType: AssignAttributes}
assignmentOp := &PathElement{OperationType: AssignAttributes}
if rhs.Node.Kind == yaml.ScalarNode {
assignmentOp.OperationType = Assign
}
rhsOp := &PathElement{PathElementType: Value, CandidateNode: rhs}
rhsOp := &PathElement{OperationType: ValueOp, CandidateNode: rhs}
assignmentOpNode := &PathTreeNode{PathElement: assignmentOp, Lhs: createTraversalTree(lhsPath), Rhs: &PathTreeNode{PathElement: rhsOp}}

5
pkg/yqlib/treeops/operator_recursive_descent.go
View File

@ -16,11 +16,14 @@ func RecursiveDescentOperator(d *dataTreeNavigator, matchMap *orderedmap.Ordered
}
func recursiveDecent(d *dataTreeNavigator, results *orderedmap.OrderedMap, matchMap *orderedmap.OrderedMap) error {
splatPathElement := &PathElement{OperationType: TraversePath, Value: "[]"}
splatTreeNode := &PathTreeNode{PathElement: splatPathElement}
for el := matchMap.Front(); el != nil; el = el.Next() {
candidate := el.Value.(*CandidateNode)
results.Set(candidate.GetKey(), candidate)
children, err := d.traverse(nodeToMap(candidate), &PathElement{PathElementType: PathKey, Value: "[]"})
children, err := TraversePathOperator(d, nodeToMap(candidate), splatTreeNode)
if err != nil {
return err

136
pkg/yqlib/treeops/leaf_traverser.go → pkg/yqlib/treeops/operator_traverse_path.go
View File

@ -3,26 +3,86 @@ package treeops
import (
"fmt"
"github.com/elliotchance/orderedmap"
"gopkg.in/yaml.v3"
)
type traverser struct {
prefs NavigationPrefs
func TraversePathOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
log.Debugf("-- Traversing")
var matchingNodeMap = orderedmap.NewOrderedMap()
var newNodes []*CandidateNode
var err error
for el := matchMap.Front(); el != nil; el = el.Next() {
newNodes, err = traverse(d, el.Value.(*CandidateNode), pathNode.PathElement)
if err != nil {
return nil, err
}
for _, n := range newNodes {
matchingNodeMap.Set(n.GetKey(), n)
}
}
return matchingNodeMap, nil
}
type LeafTraverser interface {
Traverse(matchingNode *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error)
func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) {
log.Debug("Traversing %v", NodeToString(matchingNode))
value := matchingNode.Node
if value.Kind == 0 {
log.Debugf("Guessing kind")
// we must ahve added this automatically, lets guess what it should be now
switch pathNode.Value.(type) {
case int, int64:
log.Debugf("probably an array")
value.Kind = yaml.SequenceNode
default:
log.Debugf("probabel a map")
value.Kind = yaml.MappingNode
}
}
switch value.Kind {
case yaml.MappingNode:
log.Debug("its a map with %v entries", len(value.Content)/2)
return traverseMap(matchingNode, pathNode)
case yaml.SequenceNode:
log.Debug("its a sequence of %v things!", len(value.Content))
return traverseArray(matchingNode, pathNode)
// default:
// if head == "+" {
// return n.appendArray(value, head, tail, pathStack)
// } else if len(value.Content) == 0 && head == "**" {
// return n.navigationStrategy.Visit(nodeContext)
// }
// return n.splatArray(value, head, tail, pathStack)
// }
// case yaml.AliasNode:
// log.Debug("its an alias!")
// DebugNode(value.Alias)
// if n.navigationStrategy.FollowAlias(nodeContext) {
// log.Debug("following the alias")
// return n.recurse(value.Alias, head, tail, pathStack)
// }
// return nil
case yaml.DocumentNode:
log.Debug("digging into doc node")
return traverse(d, &CandidateNode{
Node: matchingNode.Node.Content[0],
Document: matchingNode.Document}, pathNode)
default:
return nil, nil
}
}
func NewLeafTraverser(navigationPrefs NavigationPrefs) LeafTraverser {
return &traverser{navigationPrefs}
}
func (t *traverser) keyMatches(key *yaml.Node, pathNode *PathElement) bool {
func keyMatches(key *yaml.Node, pathNode *PathElement) bool {
return pathNode.Value == "[]" || Match(key.Value, pathNode.StringValue)
}
func (t *traverser) traverseMap(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) {
func traverseMap(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) {
// value.Content is a concatenated array of key, value,
// so keys are in the even indexes, values in odd.
// merge aliases are defined first, but we only want to traverse them
@ -39,7 +99,7 @@ func (t *traverser) traverseMap(candidate *CandidateNode, pathNode *PathElement)
value := contents[index+1]
log.Debug("checking %v (%v)", key.Value, key.Tag)
if t.keyMatches(key, pathNode) {
if keyMatches(key, pathNode) {
log.Debug("MATCHED")
newMatches = append(newMatches, &CandidateNode{
Node: value,
@ -63,7 +123,7 @@ func (t *traverser) traverseMap(candidate *CandidateNode, pathNode *PathElement)
return newMatches, nil
}
func (t *traverser) traverseArray(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) {
func traverseArray(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) {
log.Debug("pathNode Value %v", pathNode.Value)
if pathNode.Value == "[]" {
@ -104,55 +164,3 @@ func (t *traverser) traverseArray(candidate *CandidateNode, pathNode *PathElemen
}}, nil
}
func (t *traverser) Traverse(matchingNode *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) {
log.Debug("Traversing %v", NodeToString(matchingNode))
value := matchingNode.Node
if value.Kind == 0 {
log.Debugf("Guessing kind")
// we must ahve added this automatically, lets guess what it should be now
switch pathNode.Value.(type) {
case int, int64:
log.Debugf("probably an array")
value.Kind = yaml.SequenceNode
default:
log.Debugf("probabel a map")
value.Kind = yaml.MappingNode
}
}
switch value.Kind {
case yaml.MappingNode:
log.Debug("its a map with %v entries", len(value.Content)/2)
return t.traverseMap(matchingNode, pathNode)
case yaml.SequenceNode:
log.Debug("its a sequence of %v things!", len(value.Content))
return t.traverseArray(matchingNode, pathNode)
// default:
// if head == "+" {
// return n.appendArray(value, head, tail, pathStack)
// } else if len(value.Content) == 0 && head == "**" {
// return n.navigationStrategy.Visit(nodeContext)
// }
// return n.splatArray(value, head, tail, pathStack)
// }
// case yaml.AliasNode:
// log.Debug("its an alias!")
// DebugNode(value.Alias)
// if n.navigationStrategy.FollowAlias(nodeContext) {
// log.Debug("following the alias")
// return n.recurse(value.Alias, head, tail, pathStack)
// }
// return nil
case yaml.DocumentNode:
log.Debug("digging into doc node")
return t.Traverse(&CandidateNode{
Node: matchingNode.Node.Content[0],
Document: matchingNode.Document}, pathNode)
default:
return nil, nil
}
}

21
pkg/yqlib/treeops/operator_traverse_path_test.go Normal file
View File

@ -0,0 +1,21 @@
package treeops
import (
"testing"
)
var traversePathOperatorScenarios = []expressionScenario{
{
document: `{a: {b: apple}}`,
expression: `.a`,
expected: []string{
"D0, P[a], (!!map)::{b: apple}\n",
},
},
}
func TestTraversePathOperatorScenarios(t *testing.T) {
for _, tt := range traversePathOperatorScenarios {
testScenario(t, &tt)
}
}

1
pkg/yqlib/treeops/path_parse_test.go
View File

@ -81,6 +81,7 @@ var pathTests = []struct {
}
var tokeniser = NewPathTokeniser()
var postFixer = NewPathPostFixer()
func TestPathParsing(t *testing.T) {
for _, tt := range pathTests {

33
pkg/yqlib/treeops/path_postfix.go
View File

@ -18,32 +18,31 @@ func NewPathPostFixer() PathPostFixer {
}
func popOpToResult(opStack []*Token, result []*PathElement) ([]*Token, []*PathElement) {
var operatorToPushToPostFix *Token
opStack, operatorToPushToPostFix = opStack[0:len(opStack)-1], opStack[len(opStack)-1]
var pathElement = PathElement{PathElementType: Operation, OperationType: operatorToPushToPostFix.OperationType}
var newOp *Token
opStack, newOp = opStack[0:len(opStack)-1], opStack[len(opStack)-1]
var pathElement = PathElement{OperationType: newOp.OperationType, Value: newOp.Value, StringValue: newOp.StringValue}
if newOp.OperationType == ValueOp {
var candidateNode = BuildCandidateNodeFrom(newOp)
pathElement.CandidateNode = candidateNode
}
return opStack, append(result, &pathElement)
}
func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*PathElement, error) {
var result []*PathElement
// surround the whole thing with quotes
var opStack = []*Token{&Token{PathElementType: OpenBracket, OperationType: None, Value: "("}}
var tokens = append(infixTokens, &Token{PathElementType: CloseBracket, OperationType: None, Value: ")"})
var opStack = []*Token{&Token{TokenType: OpenBracket, Value: "("}}
var tokens = append(infixTokens, &Token{TokenType: CloseBracket, Value: ")"})
for _, token := range tokens {
log.Debugf("postfix processing token %v", token.Value)
switch token.PathElementType {
case Value:
var candidateNode = BuildCandidateNodeFrom(token)
var pathElement = PathElement{PathElementType: token.PathElementType, Value: token.Value, StringValue: token.StringValue, CandidateNode: candidateNode}
result = append(result, &pathElement)
case PathKey, SelfReference, DocumentKey:
var pathElement = PathElement{PathElementType: token.PathElementType, Value: token.Value, StringValue: token.StringValue}
result = append(result, &pathElement)
switch token.TokenType {
case OpenBracket, OpenCollect:
opStack = append(opStack, token)
case CloseCollect:
for len(opStack) > 0 && opStack[len(opStack)-1].PathElementType != OpenCollect {
for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != OpenCollect {
opStack, result = popOpToResult(opStack, result)
}
if len(opStack) == 0 {
@ -52,10 +51,10 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*PathElement,
// now we should have [] as the last element on the opStack, get rid of it
opStack = opStack[0 : len(opStack)-1]
//and append a collect to the opStack
opStack = append(opStack, &Token{PathElementType: Operation, OperationType: Pipe})
opStack = append(opStack, &Token{PathElementType: Operation, OperationType: Collect})
opStack = append(opStack, &Token{TokenType: Operation, OperationType: Pipe})
opStack = append(opStack, &Token{TokenType: Operation, OperationType: Collect})
case CloseBracket:
for len(opStack) > 0 && opStack[len(opStack)-1].PathElementType != OpenBracket {
for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != OpenBracket {
opStack, result = popOpToResult(opStack, result)
}
if len(opStack) == 0 {

355
pkg/yqlib/treeops/path_postfix_test.go
View File

@ -1,355 +0,0 @@
package treeops
import (
"testing"
"github.com/mikefarah/yq/v3/test"
)
// var tokeniser = NewPathTokeniser()
var postFixer = NewPathPostFixer()
func testExpression(expression string) (string, error) {
tokens, err := tokeniser.Tokenise(expression)
if err != nil {
return "", err
}
results, errorP := postFixer.ConvertToPostfix(tokens)
if errorP != nil {
return "", errorP
}
formatted := ""
for _, path := range results {
formatted = formatted + path.toString() + ", "
}
return formatted, nil
}
func TestPostFixTraverseBar(t *testing.T) {
var infix = ".animals | [.]"
var expectedOutput = `PathKey - animals
--------
SELF
--------
Operation - COLLECT
--------
Operation - PIPE
--------
`
actual, err := testExpression(infix)
if err != nil {
t.Error(err)
}
test.AssertResultComplex(t, expectedOutput, actual)
}
func TestPostFixPipeEquals(t *testing.T) {
var infix = `.animals | (. == "cat") `
var expectedOutput = `PathKey - animals
--------
SELF
--------
Value - cat (string)
--------
Operation - EQUALS
--------
Operation - PIPE
--------
`
actual, err := testExpression(infix)
if err != nil {
t.Error(err)
}
test.AssertResultComplex(t, expectedOutput, actual)
}
func TestPostFixCollect(t *testing.T) {
var infix = "[.a]"
var expectedOutput = `PathKey - a
--------
Operation - COLLECT
--------
`
actual, err := testExpression(infix)
if err != nil {
t.Error(err)
}
test.AssertResultComplex(t, expectedOutput, actual)
}
func TestPostFixSplatSearch(t *testing.T) {
var infix = `.a | (.[].b == "apple")`
var expectedOutput = `PathKey - a
--------
PathKey - []
--------
PathKey - b
--------
Operation - PIPE
--------
Value - apple (string)
--------
Operation - EQUALS
--------
Operation - PIPE
--------
`
actual, err := testExpression(infix)
if err != nil {
t.Error(err)
}
test.AssertResultComplex(t, expectedOutput, actual)
}
func TestPostFixCollectWithExpression(t *testing.T) {
var infix = `[ (.a == "fred") | (.d, .f)]`
var expectedOutput = `PathKey - a
--------
Value - fred (string)
--------
Operation - EQUALS
--------
PathKey - d
--------
PathKey - f
--------
Operation - OR
--------
Operation - PIPE
--------
Operation - COLLECT
--------
`
actual, err := testExpression(infix)
if err != nil {
t.Error(err)
}
test.AssertResultComplex(t, expectedOutput, actual)
}
func TestPostFixLength(t *testing.T) {
var infix = ".a | length"
var expectedOutput = `PathKey - a
--------
Operation - LENGTH
--------
Operation - PIPE
--------
`
actual, err := testExpression(infix)
if err != nil {
t.Error(err)
}
test.AssertResultComplex(t, expectedOutput, actual)
}
func TestPostFixSimpleExample(t *testing.T) {
var infix = ".a"
var expectedOutput = `PathKey - a
--------
`
actual, err := testExpression(infix)
if err != nil {
t.Error(err)
}
test.AssertResultComplex(t, expectedOutput, actual)
}
func TestPostFixSimplePathExample(t *testing.T) {
var infix = ".apples.bananas*.cat"
var expectedOutput = `PathKey - apples
--------
PathKey - bananas*
--------
Operation - PIPE
--------
PathKey - cat
--------
Operation - PIPE
--------
`
actual, err := testExpression(infix)
if err != nil {
t.Error(err)
}
test.AssertResultComplex(t, expectedOutput, actual)
}
func TestPostFixSimpleAssign(t *testing.T) {
var infix = ".a.b |= \"frog\""
var expectedOutput = `PathKey - a
--------
PathKey - b
--------
Operation - PIPE
--------
Value - frog (string)
--------
Operation - ASSIGN
--------
`
actual, err := testExpression(infix)
if err != nil {
t.Error(err)
}
test.AssertResultComplex(t, expectedOutput, actual)
}
func TestPostFixSimplePathNumbersExample(t *testing.T) {
var infix = ".apples[0].cat"
var expectedOutput = `PathKey - apples
--------
PathKey - 0
--------
Operation - PIPE
--------
PathKey - cat
--------
Operation - PIPE
--------
`
actual, err := testExpression(infix)
if err != nil {
t.Error(err)
}
test.AssertResultComplex(t, expectedOutput, actual)
}
func TestPostFixSimplePathSplatArrayExample(t *testing.T) {
var infix = ".apples[].cat"
var expectedOutput = `PathKey - apples
--------
PathKey - []
--------
Operation - PIPE
--------
PathKey - cat
--------
Operation - PIPE
--------
`
actual, err := testExpression(infix)
if err != nil {
t.Error(err)
}
test.AssertResultComplex(t, expectedOutput, actual)
}
func TestPostFixOrExample(t *testing.T) {
var infix = ".a, .b"
var expectedOutput = `PathKey - a
--------
PathKey - b
--------
Operation - OR
--------
`
actual, err := testExpression(infix)
if err != nil {
t.Error(err)
}
test.AssertResultComplex(t, expectedOutput, actual)
}
func TestPostFixEqualsNumberExample(t *testing.T) {
var infix = ".animal == 3"
var expectedOutput = `PathKey - animal
--------
Value - 3 (int64)
--------
Operation - EQUALS
--------
`
actual, err := testExpression(infix)
if err != nil {
t.Error(err)
}
test.AssertResultComplex(t, expectedOutput, actual)
}
func TestPostFixOrWithEqualsExample(t *testing.T) {
var infix = ".a==\"thing\", .b==.thongs"
var expectedOutput = `PathKey - a
--------
Value - thing (string)
--------
Operation - EQUALS
--------
PathKey - b
--------
PathKey - thongs
--------
Operation - EQUALS
--------
Operation - OR
--------
`
actual, err := testExpression(infix)
if err != nil {
t.Error(err)
}
test.AssertResultComplex(t, expectedOutput, actual)
}
func TestPostFixOrWithEqualsPathExample(t *testing.T) {
var infix = ".apples.monkeys==\"thing\", .bogs.bobos==true"
var expectedOutput = `PathKey - apples
--------
PathKey - monkeys
--------
Operation - PIPE
--------
Value - thing (string)
--------
Operation - EQUALS
--------
PathKey - bogs
--------
PathKey - bobos
--------
Operation - PIPE
--------
Value - true (bool)
--------
Operation - EQUALS
--------
Operation - OR
--------
`
actual, err := testExpression(infix)
if err != nil {
t.Error(err)
}
test.AssertResultComplex(t, expectedOutput, actual)
}

53
pkg/yqlib/treeops/path_tokeniser.go
View File

@ -11,12 +11,21 @@ func skip(*lex.Scanner, *machines.Match) (interface{}, error) {
return nil, nil
}
type TokenType uint32
const (
Operation = 1 << iota
OpenBracket
CloseBracket
OpenCollect
CloseCollect
)
type Token struct {
PathElementType PathElementType
OperationType *OperationType
Value interface{}
StringValue string
PrefixSelf bool
TokenType TokenType
OperationType *OperationType
Value interface{}
StringValue string
CheckForPostTraverse bool // e.g. [1]cat should really be [1].cat
}
@ -28,7 +37,13 @@ func pathToken(wrapped bool) lex.Action {
if wrapped {
value = unwrap(value)
}
return &Token{PathElementType: PathKey, OperationType: None, Value: value, StringValue: value, CheckForPostTraverse: true}, nil
return &Token{TokenType: Operation, OperationType: TraversePath, Value: value, StringValue: value, CheckForPostTraverse: true}, nil
}
}
func literalPathToken(value string) lex.Action {
return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
return &Token{TokenType: Operation, OperationType: TraversePath, Value: value, StringValue: value, CheckForPostTraverse: true}, nil
}
}
@ -40,20 +55,20 @@ func documentToken() lex.Action {
if errParsingInt != nil {
return nil, errParsingInt
}
return &Token{PathElementType: DocumentKey, OperationType: None, Value: number, StringValue: numberString, CheckForPostTraverse: true}, nil
return &Token{TokenType: Operation, OperationType: DocumentFilter, Value: number, StringValue: numberString, CheckForPostTraverse: true}, nil
}
}
func opToken(op *OperationType) lex.Action {
return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
value := string(m.Bytes)
return &Token{PathElementType: Operation, OperationType: op, Value: op.Type, StringValue: value}, nil
return &Token{TokenType: Operation, OperationType: op, Value: op.Type, StringValue: value}, nil
}
}
func literalToken(pType PathElementType, literal string, checkForPost bool) lex.Action {
func literalToken(pType TokenType, literal string, checkForPost bool) lex.Action {
return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
return &Token{PathElementType: pType, OperationType: None, Value: literal, StringValue: literal, CheckForPostTraverse: checkForPost}, nil
return &Token{TokenType: Operation, OperationType: ValueOp, Value: literal, StringValue: literal, CheckForPostTraverse: checkForPost}, nil
}
}
@ -73,7 +88,7 @@ func arrayIndextoken(precedingDot bool) lex.Action {
if errParsingInt != nil {
return nil, errParsingInt
}
return &Token{PathElementType: PathKey, OperationType: None, Value: number, StringValue: numberString, CheckForPostTraverse: true}, nil
return &Token{TokenType: Operation, OperationType: TraversePath, Value: number, StringValue: numberString, CheckForPostTraverse: true}, nil
}
}
@ -84,7 +99,7 @@ func numberValue() lex.Action {
if errParsingInt != nil {
return nil, errParsingInt
}
return &Token{PathElementType: Value, OperationType: None, Value: number, StringValue: numberString}, nil
return &Token{TokenType: Operation, OperationType: ValueOp, Value: number, StringValue: numberString}, nil
}
}
@ -95,13 +110,13 @@ func floatValue() lex.Action {
if errParsingInt != nil {
return nil, errParsingInt
}
return &Token{PathElementType: Value, OperationType: None, Value: number, StringValue: numberString}, nil
return &Token{TokenType: Operation, OperationType: ValueOp, Value: number, StringValue: numberString}, nil
}
}
func booleanValue(val bool) lex.Action {
return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
return &Token{PathElementType: Value, OperationType: None, Value: val, StringValue: string(m.Bytes)}, nil
return &Token{TokenType: Operation, OperationType: ValueOp, Value: val, StringValue: string(m.Bytes)}, nil
}
}
@ -111,13 +126,13 @@ func stringValue(wrapped bool) lex.Action {
if wrapped {
value = unwrap(value)
}
return &Token{PathElementType: Value, OperationType: None, Value: value, StringValue: value}, nil
return &Token{TokenType: Operation, OperationType: ValueOp, Value: value, StringValue: value}, nil
}
}
func selfToken() lex.Action {
return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
return &Token{PathElementType: SelfReference, OperationType: None, Value: "SELF", StringValue: "SELF"}, nil
return &Token{TokenType: Operation, OperationType: SelfReference}, nil
}
}
@ -127,7 +142,7 @@ func initLexer() (*lex.Lexer, error) {
lexer.Add([]byte(`\(`), literalToken(OpenBracket, "(", false))
lexer.Add([]byte(`\)`), literalToken(CloseBracket, ")", true))
lexer.Add([]byte(`\.?\[\]`), literalToken(PathKey, "[]", true))
lexer.Add([]byte(`\.?\[\]`), literalPathToken("[]"))
lexer.Add([]byte(`\.\.`), opToken(RecursiveDescent))
lexer.Add([]byte(`,`), opToken(Union))
@ -218,8 +233,8 @@ func (p *pathTokeniser) Tokenise(path string) ([]*Token, error) {
postProcessedTokens = append(postProcessedTokens, token)
if index != len(tokens)-1 && token.CheckForPostTraverse &&
tokens[index+1].PathElementType == PathKey {
postProcessedTokens = append(postProcessedTokens, &Token{PathElementType: Operation, OperationType: Pipe, Value: "PIPE"})
tokens[index+1].OperationType == TraversePath {
postProcessedTokens = append(postProcessedTokens, &Token{TokenType: Operation, OperationType: Pipe, Value: "PIPE"})
}
}

2
pkg/yqlib/treeops/path_tree.go
View File

@ -46,7 +46,7 @@ func (p *pathTreeCreator) CreatePathTree(postFixPath []*PathElement) (*PathTreeN
for _, pathElement := range postFixPath {
var newNode = PathTreeNode{PathElement: pathElement}
log.Debugf("pathTree %v ", pathElement.toString())
if pathElement.PathElementType == Operation && pathElement.OperationType.NumArgs > 0 {
if pathElement.OperationType.NumArgs > 0 {
numArgs := pathElement.OperationType.NumArgs
if numArgs == 1 {
remaining, rhs := stack[:len(stack)-1], stack[len(stack)-1]

1
release_instructions.txt
View File

@ -12,7 +12,6 @@
- snapcraft
- will auto create a candidate, test it works then promote
- see https://build.snapcraft.io/user/mikefarah/yq
sudo snap remove yq
sudo snap install --edge yq