From 73cf6224f2e951ee1992a04e444d40501e646143 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Tue, 20 Oct 2020 13:53:26 +1100 Subject: [PATCH] wip --- pkg/yqlib/treeops/data_tree_navigator.go | 41 +- pkg/yqlib/treeops/lib.go | 56 +-- pkg/yqlib/treeops/operator_multilpy.go | 10 +- .../treeops/operator_recursive_descent.go | 5 +- ...traverser.go => operator_traverse_path.go} | 136 +++---- .../treeops/operator_traverse_path_test.go | 21 ++ pkg/yqlib/treeops/path_parse_test.go | 1 + pkg/yqlib/treeops/path_postfix.go | 33 +- pkg/yqlib/treeops/path_postfix_test.go | 355 ------------------ pkg/yqlib/treeops/path_tokeniser.go | 53 ++- pkg/yqlib/treeops/path_tree.go | 2 +- release_instructions.txt | 1 - 12 files changed, 180 insertions(+), 534 deletions(-) rename pkg/yqlib/treeops/{leaf_traverser.go => operator_traverse_path.go} (76%) create mode 100644 pkg/yqlib/treeops/operator_traverse_path_test.go delete mode 100644 pkg/yqlib/treeops/path_postfix_test.go diff --git a/pkg/yqlib/treeops/data_tree_navigator.go b/pkg/yqlib/treeops/data_tree_navigator.go index 8076647a..7306300c 100644 --- a/pkg/yqlib/treeops/data_tree_navigator.go +++ b/pkg/yqlib/treeops/data_tree_navigator.go @@ -8,7 +8,7 @@ import ( ) type dataTreeNavigator struct { - leafTraverser LeafTraverser + navigationPrefs NavigationPrefs } type NavigationPrefs struct { @@ -20,27 +20,7 @@ type DataTreeNavigator interface { } func NewDataTreeNavigator(navigationPrefs NavigationPrefs) DataTreeNavigator { - leafTraverser := NewLeafTraverser(navigationPrefs) - return &dataTreeNavigator{leafTraverser} -} - -func (d *dataTreeNavigator) traverse(matchMap *orderedmap.OrderedMap, pathNode *PathElement) (*orderedmap.OrderedMap, error) { - log.Debugf("-- Traversing") - var matchingNodeMap = orderedmap.NewOrderedMap() - var newNodes []*CandidateNode - var err error - - for el := matchMap.Front(); el != nil; el = el.Next() { - newNodes, err = d.leafTraverser.Traverse(el.Value.(*CandidateNode), pathNode) - if err != nil { - return nil, err - } - for _, n := range newNodes { - matchingNodeMap.Set(n.GetKey(), n) - } - } - - return matchingNodeMap, nil + return &dataTreeNavigator{navigationPrefs} } func (d *dataTreeNavigator) GetMatchingNodes(matchingNodes []*CandidateNode, pathNode *PathTreeNode) ([]*CandidateNode, error) { @@ -75,19 +55,10 @@ func (d *dataTreeNavigator) getMatchingNodes(matchingNodes *orderedmap.OrderedMa } } log.Debug(">>") - - if pathNode.PathElement.PathElementType == SelfReference { - return matchingNodes, nil - } else if pathNode.PathElement.PathElementType == PathKey { - return d.traverse(matchingNodes, pathNode.PathElement) - } else if pathNode.PathElement.PathElementType == Value { - return nodeToMap(pathNode.PathElement.CandidateNode), nil - } else { - handler := pathNode.PathElement.OperationType.Handler - if handler != nil { - return handler(d, matchingNodes, pathNode) - } - return nil, fmt.Errorf("Unknown operator %v", pathNode.PathElement.OperationType) + handler := pathNode.PathElement.OperationType.Handler + if handler != nil { + return handler(d, matchingNodes, pathNode) } + return nil, fmt.Errorf("Unknown operator %v", pathNode.PathElement.OperationType) } diff --git a/pkg/yqlib/treeops/lib.go b/pkg/yqlib/treeops/lib.go index f30c87dc..28c2f459 100644 --- a/pkg/yqlib/treeops/lib.go +++ b/pkg/yqlib/treeops/lib.go @@ -11,20 +11,6 @@ import ( var log = logging.MustGetLogger("yq-treeops") -type PathElementType uint32 - -const ( - PathKey PathElementType = 1 << iota - DocumentKey - Operation - SelfReference - OpenBracket - CloseBracket - OpenCollect - CloseCollect - Value // e.g. string, int -) - type OperationType struct { Type string NumArgs uint // number of arguments to the op @@ -32,8 +18,6 @@ type OperationType struct { Handler OperatorHandler } -var None = &OperationType{Type: "NONE", NumArgs: 0, Precedence: 0} - var Or = &OperationType{Type: "OR", NumArgs: 2, Precedence: 20, Handler: OrOperator} var And = &OperationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: AndOperator} @@ -49,6 +33,12 @@ var Pipe = &OperationType{Type: "PIPE", NumArgs: 2, Precedence: 45, Handler: Pip var Length = &OperationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: LengthOperator} var Collect = &OperationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: CollectOperator} +var TraversePath = &OperationType{Type: "TRAVERSE_PATH", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator} + +var DocumentFilter = &OperationType{Type: "DOCUMENT_FILTER", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator} +var SelfReference = &OperationType{Type: "SELF", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator} +var ValueOp = &OperationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: TraversePathOperator} + var RecursiveDescent = &OperationType{Type: "RECURSIVE_DESCENT", NumArgs: 0, Precedence: 50, Handler: RecursiveDescentOperator} // not sure yet @@ -63,31 +53,25 @@ var DeleteChild = &OperationType{Type: "DELETE", NumArgs: 2, Precedence: 40, Han // filters matches if they have the existing path type PathElement struct { - PathElementType PathElementType - OperationType *OperationType - Value interface{} - StringValue string - CandidateNode *CandidateNode // used for Value Path elements + OperationType *OperationType + Value interface{} + StringValue string + CandidateNode *CandidateNode // used for Value Path elements } // debugging purposes only func (p *PathElement) toString() string { - var result string = `` - switch p.PathElementType { - case PathKey: - result = result + fmt.Sprintf("%v", p.Value) - case DocumentKey: - result = result + fmt.Sprintf("D%v", p.Value) - case SelfReference: - result = result + fmt.Sprintf("SELF") - case Operation: - result = result + fmt.Sprintf("%v", p.OperationType.Type) - case Value: - result = result + fmt.Sprintf("%v (%T)", p.Value, p.Value) - default: - result = result + "I HAVENT GOT A STRATEGY" + if p.OperationType == TraversePath { + return fmt.Sprintf("%v", p.Value) + } else if p.OperationType == DocumentFilter { + return fmt.Sprintf("d%v", p.Value) + } else if p.OperationType == SelfReference { + return "SELF" + } else if p.OperationType == ValueOp { + return fmt.Sprintf("%v (%T)", p.Value, p.Value) + } else { + return fmt.Sprintf("%v", p.OperationType.Type) } - return result } type YqTreeLib interface { diff --git a/pkg/yqlib/treeops/operator_multilpy.go b/pkg/yqlib/treeops/operator_multilpy.go index cb5c3ff0..57b73299 100644 --- a/pkg/yqlib/treeops/operator_multilpy.go +++ b/pkg/yqlib/treeops/operator_multilpy.go @@ -61,12 +61,12 @@ func multiply(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*Ca func createTraversalTree(path []interface{}) *PathTreeNode { if len(path) == 0 { - return &PathTreeNode{PathElement: &PathElement{PathElementType: SelfReference}} + return &PathTreeNode{PathElement: &PathElement{OperationType: SelfReference}} } else if len(path) == 1 { - return &PathTreeNode{PathElement: &PathElement{PathElementType: PathKey, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}} + return &PathTreeNode{PathElement: &PathElement{OperationType: TraversePath, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}} } return &PathTreeNode{ - PathElement: &PathElement{PathElementType: Operation, OperationType: Pipe}, + PathElement: &PathElement{OperationType: Pipe}, Lhs: createTraversalTree(path[0:1]), Rhs: createTraversalTree(path[1:])} @@ -77,11 +77,11 @@ func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *Candid lhsPath := rhs.Path[pathIndexToStartFrom:] - assignmentOp := &PathElement{PathElementType: Operation, OperationType: AssignAttributes} + assignmentOp := &PathElement{OperationType: AssignAttributes} if rhs.Node.Kind == yaml.ScalarNode { assignmentOp.OperationType = Assign } - rhsOp := &PathElement{PathElementType: Value, CandidateNode: rhs} + rhsOp := &PathElement{OperationType: ValueOp, CandidateNode: rhs} assignmentOpNode := &PathTreeNode{PathElement: assignmentOp, Lhs: createTraversalTree(lhsPath), Rhs: &PathTreeNode{PathElement: rhsOp}} diff --git a/pkg/yqlib/treeops/operator_recursive_descent.go b/pkg/yqlib/treeops/operator_recursive_descent.go index f87c5716..583eb9f9 100644 --- a/pkg/yqlib/treeops/operator_recursive_descent.go +++ b/pkg/yqlib/treeops/operator_recursive_descent.go @@ -16,11 +16,14 @@ func RecursiveDescentOperator(d *dataTreeNavigator, matchMap *orderedmap.Ordered } func recursiveDecent(d *dataTreeNavigator, results *orderedmap.OrderedMap, matchMap *orderedmap.OrderedMap) error { + splatPathElement := &PathElement{OperationType: TraversePath, Value: "[]"} + splatTreeNode := &PathTreeNode{PathElement: splatPathElement} + for el := matchMap.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) results.Set(candidate.GetKey(), candidate) - children, err := d.traverse(nodeToMap(candidate), &PathElement{PathElementType: PathKey, Value: "[]"}) + children, err := TraversePathOperator(d, nodeToMap(candidate), splatTreeNode) if err != nil { return err diff --git a/pkg/yqlib/treeops/leaf_traverser.go b/pkg/yqlib/treeops/operator_traverse_path.go similarity index 76% rename from pkg/yqlib/treeops/leaf_traverser.go rename to pkg/yqlib/treeops/operator_traverse_path.go index 21539a6a..14803d72 100644 --- a/pkg/yqlib/treeops/leaf_traverser.go +++ b/pkg/yqlib/treeops/operator_traverse_path.go @@ -3,26 +3,86 @@ package treeops import ( "fmt" + "github.com/elliotchance/orderedmap" "gopkg.in/yaml.v3" ) -type traverser struct { - prefs NavigationPrefs +func TraversePathOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { + log.Debugf("-- Traversing") + var matchingNodeMap = orderedmap.NewOrderedMap() + var newNodes []*CandidateNode + var err error + + for el := matchMap.Front(); el != nil; el = el.Next() { + newNodes, err = traverse(d, el.Value.(*CandidateNode), pathNode.PathElement) + if err != nil { + return nil, err + } + for _, n := range newNodes { + matchingNodeMap.Set(n.GetKey(), n) + } + } + + return matchingNodeMap, nil } -type LeafTraverser interface { - Traverse(matchingNode *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) +func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) { + log.Debug("Traversing %v", NodeToString(matchingNode)) + value := matchingNode.Node + + if value.Kind == 0 { + log.Debugf("Guessing kind") + // we must ahve added this automatically, lets guess what it should be now + switch pathNode.Value.(type) { + case int, int64: + log.Debugf("probably an array") + value.Kind = yaml.SequenceNode + default: + log.Debugf("probabel a map") + value.Kind = yaml.MappingNode + } + } + + switch value.Kind { + case yaml.MappingNode: + log.Debug("its a map with %v entries", len(value.Content)/2) + return traverseMap(matchingNode, pathNode) + + case yaml.SequenceNode: + log.Debug("its a sequence of %v things!", len(value.Content)) + return traverseArray(matchingNode, pathNode) + // default: + + // if head == "+" { + // return n.appendArray(value, head, tail, pathStack) + // } else if len(value.Content) == 0 && head == "**" { + // return n.navigationStrategy.Visit(nodeContext) + // } + // return n.splatArray(value, head, tail, pathStack) + // } + // case yaml.AliasNode: + // log.Debug("its an alias!") + // DebugNode(value.Alias) + // if n.navigationStrategy.FollowAlias(nodeContext) { + // log.Debug("following the alias") + // return n.recurse(value.Alias, head, tail, pathStack) + // } + // return nil + case yaml.DocumentNode: + log.Debug("digging into doc node") + return traverse(d, &CandidateNode{ + Node: matchingNode.Node.Content[0], + Document: matchingNode.Document}, pathNode) + default: + return nil, nil + } } -func NewLeafTraverser(navigationPrefs NavigationPrefs) LeafTraverser { - return &traverser{navigationPrefs} -} - -func (t *traverser) keyMatches(key *yaml.Node, pathNode *PathElement) bool { +func keyMatches(key *yaml.Node, pathNode *PathElement) bool { return pathNode.Value == "[]" || Match(key.Value, pathNode.StringValue) } -func (t *traverser) traverseMap(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) { +func traverseMap(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) { // value.Content is a concatenated array of key, value, // so keys are in the even indexes, values in odd. // merge aliases are defined first, but we only want to traverse them @@ -39,7 +99,7 @@ func (t *traverser) traverseMap(candidate *CandidateNode, pathNode *PathElement) value := contents[index+1] log.Debug("checking %v (%v)", key.Value, key.Tag) - if t.keyMatches(key, pathNode) { + if keyMatches(key, pathNode) { log.Debug("MATCHED") newMatches = append(newMatches, &CandidateNode{ Node: value, @@ -63,7 +123,7 @@ func (t *traverser) traverseMap(candidate *CandidateNode, pathNode *PathElement) return newMatches, nil } -func (t *traverser) traverseArray(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) { +func traverseArray(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) { log.Debug("pathNode Value %v", pathNode.Value) if pathNode.Value == "[]" { @@ -104,55 +164,3 @@ func (t *traverser) traverseArray(candidate *CandidateNode, pathNode *PathElemen }}, nil } - -func (t *traverser) Traverse(matchingNode *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) { - log.Debug("Traversing %v", NodeToString(matchingNode)) - value := matchingNode.Node - - if value.Kind == 0 { - log.Debugf("Guessing kind") - // we must ahve added this automatically, lets guess what it should be now - switch pathNode.Value.(type) { - case int, int64: - log.Debugf("probably an array") - value.Kind = yaml.SequenceNode - default: - log.Debugf("probabel a map") - value.Kind = yaml.MappingNode - } - } - - switch value.Kind { - case yaml.MappingNode: - log.Debug("its a map with %v entries", len(value.Content)/2) - return t.traverseMap(matchingNode, pathNode) - - case yaml.SequenceNode: - log.Debug("its a sequence of %v things!", len(value.Content)) - return t.traverseArray(matchingNode, pathNode) - // default: - - // if head == "+" { - // return n.appendArray(value, head, tail, pathStack) - // } else if len(value.Content) == 0 && head == "**" { - // return n.navigationStrategy.Visit(nodeContext) - // } - // return n.splatArray(value, head, tail, pathStack) - // } - // case yaml.AliasNode: - // log.Debug("its an alias!") - // DebugNode(value.Alias) - // if n.navigationStrategy.FollowAlias(nodeContext) { - // log.Debug("following the alias") - // return n.recurse(value.Alias, head, tail, pathStack) - // } - // return nil - case yaml.DocumentNode: - log.Debug("digging into doc node") - return t.Traverse(&CandidateNode{ - Node: matchingNode.Node.Content[0], - Document: matchingNode.Document}, pathNode) - default: - return nil, nil - } -} diff --git a/pkg/yqlib/treeops/operator_traverse_path_test.go b/pkg/yqlib/treeops/operator_traverse_path_test.go new file mode 100644 index 00000000..14217970 --- /dev/null +++ b/pkg/yqlib/treeops/operator_traverse_path_test.go @@ -0,0 +1,21 @@ +package treeops + +import ( + "testing" +) + +var traversePathOperatorScenarios = []expressionScenario{ + { + document: `{a: {b: apple}}`, + expression: `.a`, + expected: []string{ + "D0, P[a], (!!map)::{b: apple}\n", + }, + }, +} + +func TestTraversePathOperatorScenarios(t *testing.T) { + for _, tt := range traversePathOperatorScenarios { + testScenario(t, &tt) + } +} diff --git a/pkg/yqlib/treeops/path_parse_test.go b/pkg/yqlib/treeops/path_parse_test.go index ec8f015c..7e2fe345 100644 --- a/pkg/yqlib/treeops/path_parse_test.go +++ b/pkg/yqlib/treeops/path_parse_test.go @@ -81,6 +81,7 @@ var pathTests = []struct { } var tokeniser = NewPathTokeniser() +var postFixer = NewPathPostFixer() func TestPathParsing(t *testing.T) { for _, tt := range pathTests { diff --git a/pkg/yqlib/treeops/path_postfix.go b/pkg/yqlib/treeops/path_postfix.go index 646ecadf..934f22bb 100644 --- a/pkg/yqlib/treeops/path_postfix.go +++ b/pkg/yqlib/treeops/path_postfix.go @@ -18,32 +18,31 @@ func NewPathPostFixer() PathPostFixer { } func popOpToResult(opStack []*Token, result []*PathElement) ([]*Token, []*PathElement) { - var operatorToPushToPostFix *Token - opStack, operatorToPushToPostFix = opStack[0:len(opStack)-1], opStack[len(opStack)-1] - var pathElement = PathElement{PathElementType: Operation, OperationType: operatorToPushToPostFix.OperationType} + var newOp *Token + opStack, newOp = opStack[0:len(opStack)-1], opStack[len(opStack)-1] + var pathElement = PathElement{OperationType: newOp.OperationType, Value: newOp.Value, StringValue: newOp.StringValue} + + if newOp.OperationType == ValueOp { + var candidateNode = BuildCandidateNodeFrom(newOp) + pathElement.CandidateNode = candidateNode + } + return opStack, append(result, &pathElement) } func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*PathElement, error) { var result []*PathElement // surround the whole thing with quotes - var opStack = []*Token{&Token{PathElementType: OpenBracket, OperationType: None, Value: "("}} - var tokens = append(infixTokens, &Token{PathElementType: CloseBracket, OperationType: None, Value: ")"}) + var opStack = []*Token{&Token{TokenType: OpenBracket, Value: "("}} + var tokens = append(infixTokens, &Token{TokenType: CloseBracket, Value: ")"}) for _, token := range tokens { log.Debugf("postfix processing token %v", token.Value) - switch token.PathElementType { - case Value: - var candidateNode = BuildCandidateNodeFrom(token) - var pathElement = PathElement{PathElementType: token.PathElementType, Value: token.Value, StringValue: token.StringValue, CandidateNode: candidateNode} - result = append(result, &pathElement) - case PathKey, SelfReference, DocumentKey: - var pathElement = PathElement{PathElementType: token.PathElementType, Value: token.Value, StringValue: token.StringValue} - result = append(result, &pathElement) + switch token.TokenType { case OpenBracket, OpenCollect: opStack = append(opStack, token) case CloseCollect: - for len(opStack) > 0 && opStack[len(opStack)-1].PathElementType != OpenCollect { + for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != OpenCollect { opStack, result = popOpToResult(opStack, result) } if len(opStack) == 0 { @@ -52,10 +51,10 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*PathElement, // now we should have [] as the last element on the opStack, get rid of it opStack = opStack[0 : len(opStack)-1] //and append a collect to the opStack - opStack = append(opStack, &Token{PathElementType: Operation, OperationType: Pipe}) - opStack = append(opStack, &Token{PathElementType: Operation, OperationType: Collect}) + opStack = append(opStack, &Token{TokenType: Operation, OperationType: Pipe}) + opStack = append(opStack, &Token{TokenType: Operation, OperationType: Collect}) case CloseBracket: - for len(opStack) > 0 && opStack[len(opStack)-1].PathElementType != OpenBracket { + for len(opStack) > 0 && opStack[len(opStack)-1].TokenType != OpenBracket { opStack, result = popOpToResult(opStack, result) } if len(opStack) == 0 { diff --git a/pkg/yqlib/treeops/path_postfix_test.go b/pkg/yqlib/treeops/path_postfix_test.go deleted file mode 100644 index ef5382f2..00000000 --- a/pkg/yqlib/treeops/path_postfix_test.go +++ /dev/null @@ -1,355 +0,0 @@ -package treeops - -import ( - "testing" - - "github.com/mikefarah/yq/v3/test" -) - -// var tokeniser = NewPathTokeniser() -var postFixer = NewPathPostFixer() - -func testExpression(expression string) (string, error) { - tokens, err := tokeniser.Tokenise(expression) - if err != nil { - return "", err - } - results, errorP := postFixer.ConvertToPostfix(tokens) - if errorP != nil { - return "", errorP - } - formatted := "" - for _, path := range results { - formatted = formatted + path.toString() + ", " - } - return formatted, nil -} - - -func TestPostFixTraverseBar(t *testing.T) { - var infix = ".animals | [.]" - var expectedOutput = `PathKey - animals --------- -SELF --------- -Operation - COLLECT --------- -Operation - PIPE --------- -` - - actual, err := testExpression(infix) - if err != nil { - t.Error(err) - } - - test.AssertResultComplex(t, expectedOutput, actual) -} - -func TestPostFixPipeEquals(t *testing.T) { - var infix = `.animals | (. == "cat") ` - var expectedOutput = `PathKey - animals --------- -SELF --------- -Value - cat (string) --------- -Operation - EQUALS --------- -Operation - PIPE --------- -` - - actual, err := testExpression(infix) - if err != nil { - t.Error(err) - } - - test.AssertResultComplex(t, expectedOutput, actual) -} - -func TestPostFixCollect(t *testing.T) { - var infix = "[.a]" - var expectedOutput = `PathKey - a --------- -Operation - COLLECT --------- -` - - actual, err := testExpression(infix) - if err != nil { - t.Error(err) - } - - test.AssertResultComplex(t, expectedOutput, actual) -} - -func TestPostFixSplatSearch(t *testing.T) { - var infix = `.a | (.[].b == "apple")` - var expectedOutput = `PathKey - a --------- -PathKey - [] --------- -PathKey - b --------- -Operation - PIPE --------- -Value - apple (string) --------- -Operation - EQUALS --------- -Operation - PIPE --------- -` - - actual, err := testExpression(infix) - if err != nil { - t.Error(err) - } - - test.AssertResultComplex(t, expectedOutput, actual) -} - -func TestPostFixCollectWithExpression(t *testing.T) { - var infix = `[ (.a == "fred") | (.d, .f)]` - var expectedOutput = `PathKey - a --------- -Value - fred (string) --------- -Operation - EQUALS --------- -PathKey - d --------- -PathKey - f --------- -Operation - OR --------- -Operation - PIPE --------- -Operation - COLLECT --------- -` - - actual, err := testExpression(infix) - if err != nil { - t.Error(err) - } - - test.AssertResultComplex(t, expectedOutput, actual) -} - -func TestPostFixLength(t *testing.T) { - var infix = ".a | length" - var expectedOutput = `PathKey - a --------- -Operation - LENGTH --------- -Operation - PIPE --------- -` - - actual, err := testExpression(infix) - if err != nil { - t.Error(err) - } - - test.AssertResultComplex(t, expectedOutput, actual) -} - -func TestPostFixSimpleExample(t *testing.T) { - var infix = ".a" - var expectedOutput = `PathKey - a --------- -` - - actual, err := testExpression(infix) - if err != nil { - t.Error(err) - } - - test.AssertResultComplex(t, expectedOutput, actual) -} - -func TestPostFixSimplePathExample(t *testing.T) { - var infix = ".apples.bananas*.cat" - var expectedOutput = `PathKey - apples --------- -PathKey - bananas* --------- -Operation - PIPE --------- -PathKey - cat --------- -Operation - PIPE --------- -` - - actual, err := testExpression(infix) - if err != nil { - t.Error(err) - } - - test.AssertResultComplex(t, expectedOutput, actual) -} - -func TestPostFixSimpleAssign(t *testing.T) { - var infix = ".a.b |= \"frog\"" - var expectedOutput = `PathKey - a --------- -PathKey - b --------- -Operation - PIPE --------- -Value - frog (string) --------- -Operation - ASSIGN --------- -` - - actual, err := testExpression(infix) - if err != nil { - t.Error(err) - } - - test.AssertResultComplex(t, expectedOutput, actual) -} - -func TestPostFixSimplePathNumbersExample(t *testing.T) { - var infix = ".apples[0].cat" - var expectedOutput = `PathKey - apples --------- -PathKey - 0 --------- -Operation - PIPE --------- -PathKey - cat --------- -Operation - PIPE --------- -` - - actual, err := testExpression(infix) - if err != nil { - t.Error(err) - } - - test.AssertResultComplex(t, expectedOutput, actual) -} - -func TestPostFixSimplePathSplatArrayExample(t *testing.T) { - var infix = ".apples[].cat" - var expectedOutput = `PathKey - apples --------- -PathKey - [] --------- -Operation - PIPE --------- -PathKey - cat --------- -Operation - PIPE --------- -` - - actual, err := testExpression(infix) - if err != nil { - t.Error(err) - } - - test.AssertResultComplex(t, expectedOutput, actual) -} - -func TestPostFixOrExample(t *testing.T) { - var infix = ".a, .b" - var expectedOutput = `PathKey - a --------- -PathKey - b --------- -Operation - OR --------- -` - - actual, err := testExpression(infix) - if err != nil { - t.Error(err) - } - - test.AssertResultComplex(t, expectedOutput, actual) -} - -func TestPostFixEqualsNumberExample(t *testing.T) { - var infix = ".animal == 3" - var expectedOutput = `PathKey - animal --------- -Value - 3 (int64) --------- -Operation - EQUALS --------- -` - - actual, err := testExpression(infix) - if err != nil { - t.Error(err) - } - - test.AssertResultComplex(t, expectedOutput, actual) -} - -func TestPostFixOrWithEqualsExample(t *testing.T) { - var infix = ".a==\"thing\", .b==.thongs" - var expectedOutput = `PathKey - a --------- -Value - thing (string) --------- -Operation - EQUALS --------- -PathKey - b --------- -PathKey - thongs --------- -Operation - EQUALS --------- -Operation - OR --------- -` - - actual, err := testExpression(infix) - if err != nil { - t.Error(err) - } - - test.AssertResultComplex(t, expectedOutput, actual) -} - -func TestPostFixOrWithEqualsPathExample(t *testing.T) { - var infix = ".apples.monkeys==\"thing\", .bogs.bobos==true" - var expectedOutput = `PathKey - apples --------- -PathKey - monkeys --------- -Operation - PIPE --------- -Value - thing (string) --------- -Operation - EQUALS --------- -PathKey - bogs --------- -PathKey - bobos --------- -Operation - PIPE --------- -Value - true (bool) --------- -Operation - EQUALS --------- -Operation - OR --------- -` - - actual, err := testExpression(infix) - if err != nil { - t.Error(err) - } - - test.AssertResultComplex(t, expectedOutput, actual) -} diff --git a/pkg/yqlib/treeops/path_tokeniser.go b/pkg/yqlib/treeops/path_tokeniser.go index 6cc94dac..88384dfe 100644 --- a/pkg/yqlib/treeops/path_tokeniser.go +++ b/pkg/yqlib/treeops/path_tokeniser.go @@ -11,12 +11,21 @@ func skip(*lex.Scanner, *machines.Match) (interface{}, error) { return nil, nil } +type TokenType uint32 + +const ( + Operation = 1 << iota + OpenBracket + CloseBracket + OpenCollect + CloseCollect +) + type Token struct { - PathElementType PathElementType - OperationType *OperationType - Value interface{} - StringValue string - PrefixSelf bool + TokenType TokenType + OperationType *OperationType + Value interface{} + StringValue string CheckForPostTraverse bool // e.g. [1]cat should really be [1].cat } @@ -28,7 +37,13 @@ func pathToken(wrapped bool) lex.Action { if wrapped { value = unwrap(value) } - return &Token{PathElementType: PathKey, OperationType: None, Value: value, StringValue: value, CheckForPostTraverse: true}, nil + return &Token{TokenType: Operation, OperationType: TraversePath, Value: value, StringValue: value, CheckForPostTraverse: true}, nil + } +} + +func literalPathToken(value string) lex.Action { + return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { + return &Token{TokenType: Operation, OperationType: TraversePath, Value: value, StringValue: value, CheckForPostTraverse: true}, nil } } @@ -40,20 +55,20 @@ func documentToken() lex.Action { if errParsingInt != nil { return nil, errParsingInt } - return &Token{PathElementType: DocumentKey, OperationType: None, Value: number, StringValue: numberString, CheckForPostTraverse: true}, nil + return &Token{TokenType: Operation, OperationType: DocumentFilter, Value: number, StringValue: numberString, CheckForPostTraverse: true}, nil } } func opToken(op *OperationType) lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { value := string(m.Bytes) - return &Token{PathElementType: Operation, OperationType: op, Value: op.Type, StringValue: value}, nil + return &Token{TokenType: Operation, OperationType: op, Value: op.Type, StringValue: value}, nil } } -func literalToken(pType PathElementType, literal string, checkForPost bool) lex.Action { +func literalToken(pType TokenType, literal string, checkForPost bool) lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { - return &Token{PathElementType: pType, OperationType: None, Value: literal, StringValue: literal, CheckForPostTraverse: checkForPost}, nil + return &Token{TokenType: Operation, OperationType: ValueOp, Value: literal, StringValue: literal, CheckForPostTraverse: checkForPost}, nil } } @@ -73,7 +88,7 @@ func arrayIndextoken(precedingDot bool) lex.Action { if errParsingInt != nil { return nil, errParsingInt } - return &Token{PathElementType: PathKey, OperationType: None, Value: number, StringValue: numberString, CheckForPostTraverse: true}, nil + return &Token{TokenType: Operation, OperationType: TraversePath, Value: number, StringValue: numberString, CheckForPostTraverse: true}, nil } } @@ -84,7 +99,7 @@ func numberValue() lex.Action { if errParsingInt != nil { return nil, errParsingInt } - return &Token{PathElementType: Value, OperationType: None, Value: number, StringValue: numberString}, nil + return &Token{TokenType: Operation, OperationType: ValueOp, Value: number, StringValue: numberString}, nil } } @@ -95,13 +110,13 @@ func floatValue() lex.Action { if errParsingInt != nil { return nil, errParsingInt } - return &Token{PathElementType: Value, OperationType: None, Value: number, StringValue: numberString}, nil + return &Token{TokenType: Operation, OperationType: ValueOp, Value: number, StringValue: numberString}, nil } } func booleanValue(val bool) lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { - return &Token{PathElementType: Value, OperationType: None, Value: val, StringValue: string(m.Bytes)}, nil + return &Token{TokenType: Operation, OperationType: ValueOp, Value: val, StringValue: string(m.Bytes)}, nil } } @@ -111,13 +126,13 @@ func stringValue(wrapped bool) lex.Action { if wrapped { value = unwrap(value) } - return &Token{PathElementType: Value, OperationType: None, Value: value, StringValue: value}, nil + return &Token{TokenType: Operation, OperationType: ValueOp, Value: value, StringValue: value}, nil } } func selfToken() lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { - return &Token{PathElementType: SelfReference, OperationType: None, Value: "SELF", StringValue: "SELF"}, nil + return &Token{TokenType: Operation, OperationType: SelfReference}, nil } } @@ -127,7 +142,7 @@ func initLexer() (*lex.Lexer, error) { lexer.Add([]byte(`\(`), literalToken(OpenBracket, "(", false)) lexer.Add([]byte(`\)`), literalToken(CloseBracket, ")", true)) - lexer.Add([]byte(`\.?\[\]`), literalToken(PathKey, "[]", true)) + lexer.Add([]byte(`\.?\[\]`), literalPathToken("[]")) lexer.Add([]byte(`\.\.`), opToken(RecursiveDescent)) lexer.Add([]byte(`,`), opToken(Union)) @@ -218,8 +233,8 @@ func (p *pathTokeniser) Tokenise(path string) ([]*Token, error) { postProcessedTokens = append(postProcessedTokens, token) if index != len(tokens)-1 && token.CheckForPostTraverse && - tokens[index+1].PathElementType == PathKey { - postProcessedTokens = append(postProcessedTokens, &Token{PathElementType: Operation, OperationType: Pipe, Value: "PIPE"}) + tokens[index+1].OperationType == TraversePath { + postProcessedTokens = append(postProcessedTokens, &Token{TokenType: Operation, OperationType: Pipe, Value: "PIPE"}) } } diff --git a/pkg/yqlib/treeops/path_tree.go b/pkg/yqlib/treeops/path_tree.go index 30c67ee9..e07cfa7c 100644 --- a/pkg/yqlib/treeops/path_tree.go +++ b/pkg/yqlib/treeops/path_tree.go @@ -46,7 +46,7 @@ func (p *pathTreeCreator) CreatePathTree(postFixPath []*PathElement) (*PathTreeN for _, pathElement := range postFixPath { var newNode = PathTreeNode{PathElement: pathElement} log.Debugf("pathTree %v ", pathElement.toString()) - if pathElement.PathElementType == Operation && pathElement.OperationType.NumArgs > 0 { + if pathElement.OperationType.NumArgs > 0 { numArgs := pathElement.OperationType.NumArgs if numArgs == 1 { remaining, rhs := stack[:len(stack)-1], stack[len(stack)-1] diff --git a/release_instructions.txt b/release_instructions.txt index 8e3727ad..8756ca9b 100644 --- a/release_instructions.txt +++ b/release_instructions.txt @@ -12,7 +12,6 @@ - snapcraft - will auto create a candidate, test it works then promote - - see https://build.snapcraft.io/user/mikefarah/yq sudo snap remove yq sudo snap install --edge yq