From 6829d8cb78cdc3daf35bfda7cbf7be7d7853d678 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 16 Oct 2020 12:29:26 +1100 Subject: [PATCH] JQ like syntax wip --- pkg/yqlib/treeops/candidate_node.go | 9 + pkg/yqlib/treeops/data_tree_navigator.go | 2 +- pkg/yqlib/treeops/data_tree_navigator_test.go | 37 ++- pkg/yqlib/treeops/equals_operator.go | 61 +++++ pkg/yqlib/treeops/leaf_traverser.go | 2 +- pkg/yqlib/treeops/lib.go | 33 ++- pkg/yqlib/treeops/operators.go | 144 ++++------- pkg/yqlib/treeops/path_postfix.go | 21 +- pkg/yqlib/treeops/path_postfix_test.go | 230 ++++++++++-------- pkg/yqlib/treeops/path_tokeniser.go | 118 +++++---- pkg/yqlib/treeops/path_tokeniser_test.go | 81 +++--- pkg/yqlib/treeops/path_tree.go | 8 +- test/utils.go | 8 + 13 files changed, 446 insertions(+), 308 deletions(-) create mode 100644 pkg/yqlib/treeops/equals_operator.go diff --git a/pkg/yqlib/treeops/candidate_node.go b/pkg/yqlib/treeops/candidate_node.go index 13576899..aa46a495 100644 --- a/pkg/yqlib/treeops/candidate_node.go +++ b/pkg/yqlib/treeops/candidate_node.go @@ -18,6 +18,15 @@ func (n *CandidateNode) GetKey() string { return fmt.Sprintf("%v - %v", n.Document, n.Path) } +// updates this candidate from the given candidate node +func (n *CandidateNode) UpdateFrom(other *CandidateNode) { + n.Node.Content = other.Node.Content + n.Node.Value = other.Node.Value + n.Node.Kind = other.Node.Kind + n.Node.Tag = other.Node.Tag + n.Node.Style = other.Node.Style +} + func (n *CandidateNode) PathStackToString() string { return mergePathStackToString(n.Path) } diff --git a/pkg/yqlib/treeops/data_tree_navigator.go b/pkg/yqlib/treeops/data_tree_navigator.go index a5cfaaa9..be30b8c5 100644 --- a/pkg/yqlib/treeops/data_tree_navigator.go +++ b/pkg/yqlib/treeops/data_tree_navigator.go @@ -70,7 +70,7 @@ func (d *dataTreeNavigator) getMatchingNodes(matchingNodes *orderedmap.OrderedMa log.Debugf("Processing Path: %v", pathNode.PathElement.toString()) if pathNode.PathElement.PathElementType == SelfReference { return matchingNodes, nil - } else if pathNode.PathElement.PathElementType == PathKey || pathNode.PathElement.PathElementType == ArrayIndex { + } else if pathNode.PathElement.PathElementType == PathKey { return d.traverse(matchingNodes, pathNode.PathElement) } else { handler := pathNode.PathElement.OperationType.Handler diff --git a/pkg/yqlib/treeops/data_tree_navigator_test.go b/pkg/yqlib/treeops/data_tree_navigator_test.go index cdfde2f6..1d8f52e2 100644 --- a/pkg/yqlib/treeops/data_tree_navigator_test.go +++ b/pkg/yqlib/treeops/data_tree_navigator_test.go @@ -240,14 +240,41 @@ func TestDataTreeNavigatorDeleteViaSelf(t *testing.T) { test.AssertResult(t, expected, resultsToString(results)) } -func TestDataTreeNavigatorCountWithFilter(t *testing.T) { +func TestDataTreeNavigatorFilterWithSplat(t *testing.T) { nodes := readDoc(t, `f: a: frog b: dally c: log`) - path, errPath := treeCreator.ParsePath("f(count(. == *og))") + path, errPath := treeCreator.ParsePath(".f | .[] == \"frog\"") + if errPath != nil { + t.Error(errPath) + } + results, errNav := treeNavigator.GetMatchingNodes(nodes, path) + + if errNav != nil { + t.Error(errNav) + } + + expected := ` +-- Node -- + Document 0, path: [f] + Tag: !!int, Kind: ScalarNode, Anchor: + 2 +` + + test.AssertResult(t, expected, resultsToString(results)) +} + +func TestDataTreeNavigatorCountAndCollectWithFilterCmd(t *testing.T) { + + nodes := readDoc(t, `f: + a: frog + b: dally + c: log`) + + path, errPath := treeCreator.ParsePath(".f | .[] == *og ") if errPath != nil { t.Error(errPath) } @@ -934,7 +961,7 @@ func TestDataTreeNavigatorEqualsSimple(t *testing.T) { pat: {b: banana} `) - path, errPath := treeCreator.ParsePath("a.(b == apple)") + path, errPath := treeCreator.ParsePath(".a | (.[].b == \"apple\")") if errPath != nil { t.Error(errPath) } @@ -1132,7 +1159,7 @@ func TestDataTreeNavigatorArrayEqualsDeep(t *testing.T) { test.AssertResult(t, expected, resultsToString(results)) } -func TestDataTreeNavigatorEqualsTrickey(t *testing.T) { +func xTestDataTreeNavigatorEqualsTrickey(t *testing.T) { nodes := readDoc(t, `a: cat: {b: apso, c: {d : yes}} @@ -1141,7 +1168,7 @@ func TestDataTreeNavigatorEqualsTrickey(t *testing.T) { fat: {b: apple} `) - path, errPath := treeCreator.ParsePath("a.(b == ap* and c.d == yes)") + path, errPath := treeCreator.ParsePath(".a(.b == ap* and .c.d == yes)") if errPath != nil { t.Error(errPath) } diff --git a/pkg/yqlib/treeops/equals_operator.go b/pkg/yqlib/treeops/equals_operator.go new file mode 100644 index 00000000..4a3aee42 --- /dev/null +++ b/pkg/yqlib/treeops/equals_operator.go @@ -0,0 +1,61 @@ +package treeops + +import ( + "github.com/elliotchance/orderedmap" + "gopkg.in/yaml.v3" +) + +func EqualsOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { + log.Debugf("-- equalsOperation") + var results = orderedmap.NewOrderedMap() + + for el := matchMap.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + log.Debug("equalsOperation checking %v", candidate) + + matches, errInChild := hasMatch(d, candidate, pathNode.Lhs, pathNode.Rhs) + if errInChild != nil { + return nil, errInChild + } + + matchString := "true" + if !matches { + matchString = "false" + } + + node := &yaml.Node{Kind: yaml.ScalarNode, Value: matchString, Tag: "!!bool"} + lengthCand := &CandidateNode{Node: node, Document: candidate.Document, Path: candidate.Path} + results.Set(candidate.GetKey(), lengthCand) + + } + + return results, nil +} + +func hasMatch(d *dataTreeNavigator, candidate *CandidateNode, lhs *PathTreeNode, rhs *PathTreeNode) (bool, error) { + childMap := orderedmap.NewOrderedMap() + childMap.Set(candidate.GetKey(), candidate) + childMatches, errChild := d.getMatchingNodes(childMap, lhs) + log.Debug("got the LHS") + if errChild != nil { + return false, errChild + } + + // TODO = handle other RHS types + return containsMatchingValue(childMatches, rhs.PathElement.StringValue), nil +} + +func containsMatchingValue(matchMap *orderedmap.OrderedMap, valuePattern string) bool { + log.Debugf("-- findMatchingValues") + + for el := matchMap.Front(); el != nil; el = el.Next() { + node := el.Value.(*CandidateNode) + log.Debugf("-- comparing %v to %v", node.Node.Value, valuePattern) + if Match(node.Node.Value, valuePattern) { + return true + } + } + log.Debugf("-- done findMatchingValues") + + return false +} diff --git a/pkg/yqlib/treeops/leaf_traverser.go b/pkg/yqlib/treeops/leaf_traverser.go index 1dee1740..679a08b2 100644 --- a/pkg/yqlib/treeops/leaf_traverser.go +++ b/pkg/yqlib/treeops/leaf_traverser.go @@ -53,7 +53,7 @@ func (t *traverser) traverseMap(candidate *CandidateNode, pathNode *PathElement) func (t *traverser) traverseArray(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) { log.Debug("pathNode Value %v", pathNode.Value) - if pathNode.Value == "[*]" || pathNode.Value == "*" { + if pathNode.Value == "[]" { var contents = candidate.Node.Content var newMatches = make([]*CandidateNode, len(contents)) diff --git a/pkg/yqlib/treeops/lib.go b/pkg/yqlib/treeops/lib.go index d2b53791..46d9ba2d 100644 --- a/pkg/yqlib/treeops/lib.go +++ b/pkg/yqlib/treeops/lib.go @@ -15,11 +15,13 @@ type PathElementType uint32 const ( PathKey PathElementType = 1 << iota - ArrayIndex Operation SelfReference OpenBracket CloseBracket + OpenCollect + CloseCollect + Value // e.g. string, int ) type OperationType struct { @@ -30,16 +32,23 @@ type OperationType struct { } var None = &OperationType{Type: "NONE", NumArgs: 0, Precedence: 0} -var Traverse = &OperationType{Type: "TRAVERSE", NumArgs: 2, Precedence: 35, Handler: TraverseOperator} + var Or = &OperationType{Type: "OR", NumArgs: 2, Precedence: 10, Handler: UnionOperator} var And = &OperationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: IntersectionOperator} -var Equals = &OperationType{Type: "EQUALS", NumArgs: 2, Precedence: 30, Handler: EqualsOperator} -var Assign = &OperationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: AssignOperator} -var DeleteChild = &OperationType{Type: "DELETE", NumArgs: 2, Precedence: 30, Handler: DeleteChildOperator} -var Count = &OperationType{Type: "COUNT", NumArgs: 1, Precedence: 40, Handler: CountOperator} +var Assign = &OperationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: AssignOperator} +var Equals = &OperationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: EqualsOperator} +var Pipe = &OperationType{Type: "PIPE", NumArgs: 2, Precedence: 45, Handler: PipeOperator} + +var Length = &OperationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: LengthOperator} + +// not sure yet + +var DeleteChild = &OperationType{Type: "DELETE", NumArgs: 2, Precedence: 40, Handler: DeleteChildOperator} var Collect = &OperationType{Type: "COLLECT", NumArgs: 1, Precedence: 40, Handler: CollectOperator} +// var Splat = &OperationType{Type: "SPLAT", NumArgs: 0, Precedence: 40, Handler: SplatOperator} + // var Exists = &OperationType{Type: "Length", NumArgs: 2, Precedence: 35} // filters matches if they have the existing path @@ -55,13 +64,15 @@ func (p *PathElement) toString() string { var result string = `` switch p.PathElementType { case PathKey: - result = result + fmt.Sprintf("PathKey - '%v'\n", p.Value) - case ArrayIndex: - result = result + fmt.Sprintf("ArrayIndex - '%v'\n", p.Value) + result = result + fmt.Sprintf("PathKey - %v", p.Value) case SelfReference: - result = result + fmt.Sprintf("SELF\n") + result = result + fmt.Sprintf("SELF") case Operation: - result = result + fmt.Sprintf("Operation - %v\n", p.OperationType.Type) + result = result + fmt.Sprintf("Operation - %v", p.OperationType.Type) + case Value: + result = result + fmt.Sprintf("Value - %v (%T)", p.Value, p.Value) + default: + result = result + "I HAVENT GOT A STRATEGY" } return result } diff --git a/pkg/yqlib/treeops/operators.go b/pkg/yqlib/treeops/operators.go index 5405bb4f..9e5afcaf 100644 --- a/pkg/yqlib/treeops/operators.go +++ b/pkg/yqlib/treeops/operators.go @@ -9,7 +9,7 @@ import ( type OperatorHandler func(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) -func TraverseOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { +func PipeOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { lhs, err := d.getMatchingNodes(matchingNodes, pathNode.Lhs) if err != nil { return nil, err @@ -17,15 +17,32 @@ func TraverseOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap return d.getMatchingNodes(lhs, pathNode.Rhs) } +func nodeToMap(candidate *CandidateNode) *orderedmap.OrderedMap { + elMap := orderedmap.NewOrderedMap() + elMap.Set(candidate.GetKey(), candidate) + return elMap +} + func AssignOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { lhs, err := d.getMatchingNodes(matchingNodes, pathNode.Lhs) if err != nil { return nil, err } for el := lhs.Front(); el != nil; el = el.Next() { - node := el.Value.(*CandidateNode) - log.Debugf("Assiging %v to %v", node.GetKey(), pathNode.Rhs.PathElement.StringValue) - node.Node.Value = pathNode.Rhs.PathElement.StringValue + candidate := el.Value.(*CandidateNode) + + rhs, err := d.getMatchingNodes(nodeToMap(candidate), pathNode.Rhs) + + if err != nil { + return nil, err + } + + // grab the first value + first := rhs.Front() + + if first != nil { + candidate.UpdateFrom(first.Value.(*CandidateNode)) + } } return lhs, nil } @@ -66,54 +83,36 @@ func IntersectionOperator(d *dataTreeNavigator, matchingNodes *orderedmap.Ordere } func splatNode(d *dataTreeNavigator, candidate *CandidateNode) (*orderedmap.OrderedMap, error) { - elMap := orderedmap.NewOrderedMap() - elMap.Set(candidate.GetKey(), candidate) //need to splat matching nodes, then search through them splatter := &PathTreeNode{PathElement: &PathElement{ PathElementType: PathKey, Value: "*", StringValue: "*", }} - return d.getMatchingNodes(elMap, splatter) + return d.getMatchingNodes(nodeToMap(candidate), splatter) } -func EqualsOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { - log.Debugf("-- equalsOperation") +func LengthOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { + log.Debugf("-- lengthOperation") var results = orderedmap.NewOrderedMap() for el := matchMap.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) - valuePattern := pathNode.Rhs.PathElement.StringValue - log.Debug("checking %v", candidate) - - errInChild := findMatchingChildren(d, results, candidate, pathNode.Lhs, valuePattern) - if errInChild != nil { - return nil, errInChild - } - } - - return results, nil -} - -func CountOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { - log.Debugf("-- countOperation") - var results = orderedmap.NewOrderedMap() - - for el := matchMap.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - elMap := orderedmap.NewOrderedMap() - elMap.Set(el.Key, el.Value) - childMatches, errChild := d.getMatchingNodes(elMap, pathNode.Rhs) - - if errChild != nil { - return nil, errChild + var length int + switch candidate.Node.Kind { + case yaml.ScalarNode: + length = len(candidate.Node.Value) + case yaml.MappingNode: + length = len(candidate.Node.Content) / 2 + case yaml.SequenceNode: + length = len(candidate.Node.Content) + default: + length = 0 } - length := childMatches.Len() node := &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%v", length), Tag: "!!int"} lengthCand := &CandidateNode{Node: node, Document: candidate.Document, Path: candidate.Path} results.Set(candidate.GetKey(), lengthCand) - } return results, nil @@ -122,76 +121,25 @@ func CountOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNo func CollectOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { log.Debugf("-- collectOperation") - log.Debugf("-- countOperation") var results = orderedmap.NewOrderedMap() + node := &yaml.Node{Kind: yaml.SequenceNode} + + var document uint = 0 + var path []interface{} + for el := matchMap.Front(); el != nil; el = el.Next() { candidate := el.Value.(*CandidateNode) - elMap := orderedmap.NewOrderedMap() - elMap.Set(el.Key, el.Value) - childMatches, errChild := d.getMatchingNodes(elMap, pathNode.Rhs) - - if errChild != nil { - return nil, errChild + if path == nil && candidate.Path != nil { + path = candidate.Path + document = candidate.Document } - - node := &yaml.Node{Kind: yaml.SequenceNode} - - for childEl := childMatches.Front(); childEl != nil; childEl = childEl.Next() { - childCandidate := childEl.Value.(*CandidateNode) - node.Content = append(node.Content, childCandidate.Node) - } - - lengthCand := &CandidateNode{Node: node, Document: candidate.Document, Path: candidate.Path} - results.Set(candidate.GetKey(), lengthCand) + node.Content = append(node.Content, candidate.Node) } + collectC := &CandidateNode{Node: node, Document: document, Path: path} + results.Set(collectC.GetKey(), collectC) + return results, nil } - -func findMatchingChildren(d *dataTreeNavigator, results *orderedmap.OrderedMap, candidate *CandidateNode, lhs *PathTreeNode, valuePattern string) error { - var children *orderedmap.OrderedMap - var err error - // don't splat scalars. - if candidate.Node.Kind != yaml.ScalarNode { - children, err = splatNode(d, candidate) - log.Debugf("-- splatted matches, ") - if err != nil { - return err - } - } else { - children = orderedmap.NewOrderedMap() - children.Set(candidate.GetKey(), candidate) - } - - for childEl := children.Front(); childEl != nil; childEl = childEl.Next() { - childMap := orderedmap.NewOrderedMap() - childMap.Set(childEl.Key, childEl.Value) - childMatches, errChild := d.getMatchingNodes(childMap, lhs) - log.Debug("got the LHS") - if errChild != nil { - return errChild - } - - if containsMatchingValue(childMatches, valuePattern) { - results.Set(childEl.Key, childEl.Value) - } - } - return nil -} - -func containsMatchingValue(matchMap *orderedmap.OrderedMap, valuePattern string) bool { - log.Debugf("-- findMatchingValues") - - for el := matchMap.Front(); el != nil; el = el.Next() { - node := el.Value.(*CandidateNode) - log.Debugf("-- comparing %v to %v", node.Node.Value, valuePattern) - if Match(node.Node.Value, valuePattern) { - return true - } - } - log.Debugf("-- done findMatchingValues") - - return false -} diff --git a/pkg/yqlib/treeops/path_postfix.go b/pkg/yqlib/treeops/path_postfix.go index 40419959..f1c46cd4 100644 --- a/pkg/yqlib/treeops/path_postfix.go +++ b/pkg/yqlib/treeops/path_postfix.go @@ -25,17 +25,28 @@ func popOpToResult(opStack []*Token, result []*PathElement) ([]*Token, []*PathEl func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*PathElement, error) { var result []*PathElement // surround the whole thing with quotes - var opStack = []*Token{&Token{PathElementType: OpenBracket, OperationType: None}} - var tokens = append(infixTokens, &Token{PathElementType: CloseBracket, OperationType: None}) + var opStack = []*Token{&Token{PathElementType: OpenBracket, OperationType: None, Value: "("}} + var tokens = append(infixTokens, &Token{PathElementType: CloseBracket, OperationType: None, Value: ")"}) for _, token := range tokens { + log.Debugf("postfix processing token %v", token.Value) switch token.PathElementType { - case PathKey, ArrayIndex, SelfReference: + case PathKey, SelfReference, Value: var pathElement = PathElement{PathElementType: token.PathElementType, Value: token.Value, StringValue: token.StringValue} result = append(result, &pathElement) - case OpenBracket: + case OpenBracket, OpenCollect: opStack = append(opStack, token) - + case CloseCollect: + for len(opStack) > 0 && opStack[len(opStack)-1].PathElementType != OpenCollect { + opStack, result = popOpToResult(opStack, result) + } + if len(opStack) == 0 { + return nil, errors.New("Bad path expression, got close collect brackets without matching opening bracket") + } + // now we should have ( as the last element on the opStack, get rid of it + opStack = opStack[0 : len(opStack)-1] + //and append a collect to the opStack + opStack = append(opStack, &Token{PathElementType: Operation, OperationType: Collect}) case CloseBracket: for len(opStack) > 0 && opStack[len(opStack)-1].PathElementType != OpenBracket { opStack, result = popOpToResult(opStack, result) diff --git a/pkg/yqlib/treeops/path_postfix_test.go b/pkg/yqlib/treeops/path_postfix_test.go index 89f112cf..0673424d 100644 --- a/pkg/yqlib/treeops/path_postfix_test.go +++ b/pkg/yqlib/treeops/path_postfix_test.go @@ -20,20 +20,20 @@ func testExpression(expression string) (string, error) { } formatted := "" for _, path := range results { - formatted = formatted + path.toString() + "--------\n" + formatted = formatted + path.toString() + "\n--------\n" } return formatted, nil } func TestPostFixTraverseBar(t *testing.T) { - var infix = "animals | collect(.)" - var expectedOutput = `PathKey - 'animals' + var infix = ".animals | [.]" + var expectedOutput = `PathKey - animals -------- SELF -------- Operation - COLLECT -------- -Operation - TRAVERSE +Operation - PIPE -------- ` @@ -45,17 +45,87 @@ Operation - TRAVERSE test.AssertResultComplex(t, expectedOutput, actual) } -func TestPostFixArrayEquals(t *testing.T) { - var infix = "animals(.== cat)" - var expectedOutput = `PathKey - 'animals' +func TestPostFixPipeEquals(t *testing.T) { + var infix = `.animals | (. == "cat") ` + var expectedOutput = `PathKey - animals -------- SELF -------- -PathKey - 'cat' +Value - cat (string) -------- Operation - EQUALS -------- -Operation - TRAVERSE +Operation - PIPE +-------- +` + + actual, err := testExpression(infix) + if err != nil { + t.Error(err) + } + + test.AssertResultComplex(t, expectedOutput, actual) +} + +func TestPostFixCollect(t *testing.T) { + var infix = "[.a]" + var expectedOutput = `PathKey - a +-------- +Operation - COLLECT +-------- +` + + actual, err := testExpression(infix) + if err != nil { + t.Error(err) + } + + test.AssertResultComplex(t, expectedOutput, actual) +} + +func TestPostFixSplatSearch(t *testing.T) { + var infix = `.a | (.[].b == "apple")` + var expectedOutput = `PathKey - a +-------- +PathKey - [] +-------- +PathKey - b +-------- +Operation - PIPE +-------- +Value - apple (string) +-------- +Operation - EQUALS +-------- +Operation - PIPE +-------- +` + + actual, err := testExpression(infix) + if err != nil { + t.Error(err) + } + + test.AssertResultComplex(t, expectedOutput, actual) +} + +func TestPostFixCollectWithExpression(t *testing.T) { + var infix = `[ (.a == "fred") | (.d, .f)]` + var expectedOutput = `PathKey - a +-------- +Value - fred (string) +-------- +Operation - EQUALS +-------- +PathKey - d +-------- +PathKey - f +-------- +Operation - OR +-------- +Operation - PIPE +-------- +Operation - COLLECT -------- ` @@ -68,10 +138,12 @@ Operation - TRAVERSE } func TestPostFixLength(t *testing.T) { - var infix = "len(a)" - var expectedOutput = `PathKey - 'a' + var infix = ".a | length" + var expectedOutput = `PathKey - a -------- -Operation - Length +Operation - LENGTH +-------- +Operation - PIPE -------- ` @@ -84,8 +156,8 @@ Operation - Length } func TestPostFixSimpleExample(t *testing.T) { - var infix = "a" - var expectedOutput = `PathKey - 'a' + var infix = ".a" + var expectedOutput = `PathKey - a -------- ` @@ -98,16 +170,16 @@ func TestPostFixSimpleExample(t *testing.T) { } func TestPostFixSimplePathExample(t *testing.T) { - var infix = "apples.bananas*.cat" - var expectedOutput = `PathKey - 'apples' + var infix = ".apples.bananas*.cat" + var expectedOutput = `PathKey - apples -------- -PathKey - 'bananas*' +PathKey - bananas* -------- -Operation - TRAVERSE +Operation - PIPE -------- -PathKey - 'cat' +PathKey - cat -------- -Operation - TRAVERSE +Operation - PIPE -------- ` @@ -120,14 +192,14 @@ Operation - TRAVERSE } func TestPostFixSimpleAssign(t *testing.T) { - var infix = "a.b := frog" - var expectedOutput = `PathKey - 'a' + var infix = ".a.b |= \"frog\"" + var expectedOutput = `PathKey - a -------- -PathKey - 'b' +PathKey - b -------- -Operation - TRAVERSE +Operation - PIPE -------- -PathKey - 'frog' +Value - frog (string) -------- Operation - ASSIGN -------- @@ -142,38 +214,16 @@ Operation - ASSIGN } func TestPostFixSimplePathNumbersExample(t *testing.T) { - var infix = "apples[0].cat" - var expectedOutput = `PathKey - 'apples' + var infix = ".apples[0].cat" + var expectedOutput = `PathKey - apples -------- -PathKey - '0' +PathKey - 0 -------- -Operation - TRAVERSE +Operation - PIPE -------- -PathKey - 'cat' +PathKey - cat -------- -Operation - TRAVERSE --------- -` - - actual, err := testExpression(infix) - if err != nil { - t.Error(err) - } - - test.AssertResultComplex(t, expectedOutput, actual) -} - -func TestPostFixSimplePathAppendArrayExample(t *testing.T) { - var infix = "apples[+].cat" - var expectedOutput = `PathKey - 'apples' --------- -PathKey - '[+]' --------- -Operation - TRAVERSE --------- -PathKey - 'cat' --------- -Operation - TRAVERSE +Operation - PIPE -------- ` @@ -186,38 +236,16 @@ Operation - TRAVERSE } func TestPostFixSimplePathSplatArrayExample(t *testing.T) { - var infix = "apples.[*]cat" - var expectedOutput = `PathKey - 'apples' + var infix = ".apples[].cat" + var expectedOutput = `PathKey - apples -------- -PathKey - '[*]' +PathKey - [] -------- -Operation - TRAVERSE +Operation - PIPE -------- -PathKey - 'cat' +PathKey - cat -------- -Operation - TRAVERSE --------- -` - - actual, err := testExpression(infix) - if err != nil { - t.Error(err) - } - - test.AssertResultComplex(t, expectedOutput, actual) -} - -func TestPostFixDeepMatchExample(t *testing.T) { - var infix = "apples.**.cat" - var expectedOutput = `PathKey - 'apples' --------- -PathKey - '**' --------- -Operation - TRAVERSE --------- -PathKey - 'cat' --------- -Operation - TRAVERSE +Operation - PIPE -------- ` @@ -230,10 +258,10 @@ Operation - TRAVERSE } func TestPostFixOrExample(t *testing.T) { - var infix = "a OR b" - var expectedOutput = `PathKey - 'a' + var infix = ".a, .b" + var expectedOutput = `PathKey - a -------- -PathKey - 'b' +PathKey - b -------- Operation - OR -------- @@ -248,10 +276,10 @@ Operation - OR } func TestPostFixEqualsNumberExample(t *testing.T) { - var infix = "(animal == 3)" - var expectedOutput = `PathKey - 'animal' + var infix = ".animal == 3" + var expectedOutput = `PathKey - animal -------- -PathKey - '3' +Value - 3 (int64) -------- Operation - EQUALS -------- @@ -266,16 +294,16 @@ Operation - EQUALS } func TestPostFixOrWithEqualsExample(t *testing.T) { - var infix = "a==thing OR b==thongs" - var expectedOutput = `PathKey - 'a' + var infix = ".a==\"thing\", .b==.thongs" + var expectedOutput = `PathKey - a -------- -PathKey - 'thing' +Value - thing (string) -------- Operation - EQUALS -------- -PathKey - 'b' +PathKey - b -------- -PathKey - 'thongs' +PathKey - thongs -------- Operation - EQUALS -------- @@ -292,24 +320,24 @@ Operation - OR } func TestPostFixOrWithEqualsPathExample(t *testing.T) { - var infix = "apples.monkeys==thing OR bogs.bobos==thongs" - var expectedOutput = `PathKey - 'apples' + var infix = ".apples.monkeys==\"thing\", .bogs.bobos==true" + var expectedOutput = `PathKey - apples -------- -PathKey - 'monkeys' +PathKey - monkeys -------- -Operation - TRAVERSE +Operation - PIPE -------- -PathKey - 'thing' +Value - thing (string) -------- Operation - EQUALS -------- -PathKey - 'bogs' +PathKey - bogs -------- -PathKey - 'bobos' +PathKey - bobos -------- -Operation - TRAVERSE +Operation - PIPE -------- -PathKey - 'thongs' +Value - true (bool) -------- Operation - EQUALS -------- diff --git a/pkg/yqlib/treeops/path_tokeniser.go b/pkg/yqlib/treeops/path_tokeniser.go index ccb52012..e9d91b8c 100644 --- a/pkg/yqlib/treeops/path_tokeniser.go +++ b/pkg/yqlib/treeops/path_tokeniser.go @@ -18,31 +18,30 @@ type Token struct { StringValue string PrefixSelf bool - CheckForPreTraverse bool // this token can sometimes have the traverse '.' missing in frnot of it - // e.g. a[1] should really be a.[1] - CheckForPostTraverse bool // samething but for post, e.g. [1]cat should really be [1].cat + CheckForPostTraverse bool // e.g. [1]cat should really be [1].cat } func pathToken(wrapped bool) lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { value := string(m.Bytes) + value = value[1:len(value)] if wrapped { value = unwrap(value) } - return &Token{PathElementType: PathKey, OperationType: None, Value: value, StringValue: value}, nil + return &Token{PathElementType: PathKey, OperationType: None, Value: value, StringValue: value, CheckForPostTraverse: true}, nil } } -func opToken(op *OperationType, againstSelf bool) lex.Action { +func opToken(op *OperationType) lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { value := string(m.Bytes) - return &Token{PathElementType: Operation, OperationType: op, Value: op.Type, StringValue: value, PrefixSelf: againstSelf}, nil + return &Token{PathElementType: Operation, OperationType: op, Value: op.Type, StringValue: value}, nil } } -func literalToken(pType PathElementType, literal string, checkForPre bool, checkForPost bool, againstSelf bool) lex.Action { +func literalToken(pType PathElementType, literal string, checkForPost bool) lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { - return &Token{PathElementType: pType, OperationType: None, Value: literal, StringValue: literal, CheckForPreTraverse: checkForPre, CheckForPostTraverse: checkForPost, PrefixSelf: againstSelf}, nil + return &Token{PathElementType: pType, OperationType: None, Value: literal, StringValue: literal, CheckForPostTraverse: checkForPost}, nil } } @@ -50,54 +49,96 @@ func unwrap(value string) string { return value[1 : len(value)-1] } -func arrayIndextoken(wrapped bool, checkForPre bool, checkForPost bool) lex.Action { +func arrayIndextoken(precedingDot bool) lex.Action { return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { var numberString = string(m.Bytes) - if wrapped { - numberString = unwrap(numberString) + startIndex := 1 + if precedingDot { + startIndex = 2 } + numberString = numberString[startIndex : len(numberString)-1] var number, errParsingInt = strconv.ParseInt(numberString, 10, 64) // nolint if errParsingInt != nil { return nil, errParsingInt } - return &Token{PathElementType: ArrayIndex, OperationType: None, Value: number, StringValue: numberString, CheckForPreTraverse: checkForPre, CheckForPostTraverse: checkForPost}, nil + return &Token{PathElementType: PathKey, OperationType: None, Value: number, StringValue: numberString, CheckForPostTraverse: true}, nil + } +} + +func numberValue() lex.Action { + return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { + var numberString = string(m.Bytes) + var number, errParsingInt = strconv.ParseInt(numberString, 10, 64) // nolint + if errParsingInt != nil { + return nil, errParsingInt + } + return &Token{PathElementType: Value, OperationType: None, Value: number, StringValue: numberString}, nil + } +} + +func booleanValue(val bool) lex.Action { + return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { + return &Token{PathElementType: Value, OperationType: None, Value: val, StringValue: string(m.Bytes)}, nil + } +} + +func stringValue(wrapped bool) lex.Action { + return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { + value := string(m.Bytes) + if wrapped { + value = unwrap(value) + } + return &Token{PathElementType: Value, OperationType: None, Value: value, StringValue: value}, nil + } +} + +func selfToken() lex.Action { + return func(s *lex.Scanner, m *machines.Match) (interface{}, error) { + return &Token{PathElementType: SelfReference, OperationType: None, Value: "SELF", StringValue: "SELF"}, nil } } // Creates the lexer object and compiles the NFA. func initLexer() (*lex.Lexer, error) { lexer := lex.NewLexer() - lexer.Add([]byte(`\(`), literalToken(OpenBracket, "(", true, false, false)) - lexer.Add([]byte(`\)`), literalToken(CloseBracket, ")", false, true, false)) - lexer.Add([]byte(`\.\s*\)`), literalToken(CloseBracket, ")", false, true, true)) + lexer.Add([]byte(`\(`), literalToken(OpenBracket, "(", false)) + lexer.Add([]byte(`\)`), literalToken(CloseBracket, ")", true)) - lexer.Add([]byte(`\[\+\]`), literalToken(PathKey, "[+]", true, true, false)) - lexer.Add([]byte(`\[\*\]`), literalToken(PathKey, "[*]", true, true, false)) - lexer.Add([]byte(`\*\*`), literalToken(PathKey, "**", false, false, false)) + lexer.Add([]byte(`\.?\[\]`), literalToken(PathKey, "[]", true)) + lexer.Add([]byte(`\.\.`), literalToken(PathKey, "..", true)) - lexer.Add([]byte(`([Oo][Rr])`), opToken(Or, false)) - lexer.Add([]byte(`([Aa][Nn][Dd])`), opToken(And, false)) - lexer.Add([]byte(`([Cc][Oo][Uu][Nn][Tt])`), opToken(Count, false)) - lexer.Add([]byte(`([Cc][Oo][Ll][Ll][Ee][Cc][Tt])`), opToken(Collect, false)) + lexer.Add([]byte(`,`), opToken(Or)) + lexer.Add([]byte(`length`), opToken(Length)) + lexer.Add([]byte(`([Cc][Oo][Ll][Ll][Ee][Cc][Tt])`), opToken(Collect)) - lexer.Add([]byte(`\.\s*==\s*`), opToken(Equals, true)) - lexer.Add([]byte(`\s*==\s*`), opToken(Equals, false)) + lexer.Add([]byte(`\s*==\s*`), opToken(Equals)) - lexer.Add([]byte(`\.\s*.-\s*`), opToken(DeleteChild, true)) - lexer.Add([]byte(`\s*.-\s*`), opToken(DeleteChild, false)) + lexer.Add([]byte(`\s*.-\s*`), opToken(DeleteChild)) - lexer.Add([]byte(`\.\s*:=\s*`), opToken(Assign, true)) - lexer.Add([]byte(`\s*:=\s*`), opToken(Assign, false)) + lexer.Add([]byte(`\s*\|=\s*`), opToken(Assign)) + + lexer.Add([]byte(`\[-?[0-9]+\]`), arrayIndextoken(false)) + lexer.Add([]byte(`\.\[-?[0-9]+\]`), arrayIndextoken(true)) - lexer.Add([]byte(`\[-?[0-9]+\]`), arrayIndextoken(true, true, true)) - lexer.Add([]byte(`-?[0-9]+`), arrayIndextoken(false, false, false)) lexer.Add([]byte("( |\t|\n|\r)+"), skip) - lexer.Add([]byte(`"[^ "]+"`), pathToken(true)) - lexer.Add([]byte(`[^ \|\.\[\(\)=]+`), pathToken(false)) + lexer.Add([]byte(`\."[^ "]+"`), pathToken(true)) + lexer.Add([]byte(`\.[^ \[\],\|\.\[\(\)=]+`), pathToken(false)) + lexer.Add([]byte(`\.`), selfToken()) - lexer.Add([]byte(`\|`), opToken(Traverse, false)) - lexer.Add([]byte(`\.`), opToken(Traverse, false)) + lexer.Add([]byte(`\|`), opToken(Pipe)) + + lexer.Add([]byte(`-?[0-9]+`), numberValue()) + + lexer.Add([]byte(`[Tt][Rr][Uu][Ee]`), booleanValue(true)) + lexer.Add([]byte(`[Ff][Aa][Ll][Ss][Ee]`), booleanValue(false)) + + lexer.Add([]byte(`"[^ "]+"`), stringValue(true)) + + lexer.Add([]byte(`\[`), literalToken(OpenCollect, "[", false)) + lexer.Add([]byte(`\]`), literalToken(CloseCollect, "]", true)) + + // lexer.Add([]byte(`[^ \,\|\.\[\(\)=]+`), stringValue(false)) err := lexer.Compile() if err != nil { return nil, err @@ -142,19 +183,12 @@ func (p *pathTokeniser) Tokenise(path string) ([]*Token, error) { var postProcessedTokens = make([]*Token, 0) for index, token := range tokens { - if index > 0 && token.CheckForPreTraverse && - (tokens[index-1].PathElementType == PathKey || tokens[index-1].PathElementType == CloseBracket) { - postProcessedTokens = append(postProcessedTokens, &Token{PathElementType: Operation, OperationType: Traverse, Value: "."}) - } - if token.PrefixSelf { - postProcessedTokens = append(postProcessedTokens, &Token{PathElementType: SelfReference, Value: "SELF"}) - } postProcessedTokens = append(postProcessedTokens, token) if index != len(tokens)-1 && token.CheckForPostTraverse && tokens[index+1].PathElementType == PathKey { - postProcessedTokens = append(postProcessedTokens, &Token{PathElementType: Operation, OperationType: Traverse, Value: "."}) + postProcessedTokens = append(postProcessedTokens, &Token{PathElementType: Operation, OperationType: Pipe, Value: "PIPE"}) } } diff --git a/pkg/yqlib/treeops/path_tokeniser_test.go b/pkg/yqlib/treeops/path_tokeniser_test.go index 6c56a5ea..731460e3 100644 --- a/pkg/yqlib/treeops/path_tokeniser_test.go +++ b/pkg/yqlib/treeops/path_tokeniser_test.go @@ -10,50 +10,53 @@ var tokeniserTests = []struct { path string expectedTokens []interface{} }{ // TODO: Ensure ALL documented examples have tests! sheesh - {"len(.)", append(make([]interface{}, 0), "LENGTH", "(", "SELF", ")")}, - {"\"len\"(.)", append(make([]interface{}, 0), "len", ".", "(", "SELF", ")")}, - // {"a OR (b OR c)", append(make([]interface{}, 0), "a", "OR", "(", "b", "OR", "c", ")")}, + // {"len(.)", append(make([]interface{}, 0), "LENGTH", "(", "SELF", ")")}, + // {"\"len\"(.)", append(make([]interface{}, 0), "len", "TRAVERSE", "(", "SELF", ")")}, + // {".a OR (.b OR .c)", append(make([]interface{}, 0), "a", "OR", "(", "b", "OR", "c", ")")}, // {"a OR (b OR c)", append(make([]interface{}, 0), "a", "OR", "(", "b", "OR", "c", ")")}, // {"a .- (b OR c)", append(make([]interface{}, 0), "a", " .- ", "(", "b", "OR", "c", ")")}, // {"(animal==3)", append(make([]interface{}, 0), "(", "animal", "==", int64(3), ")")}, // {"(animal==f3)", append(make([]interface{}, 0), "(", "animal", "==", "f3", ")")}, - // {"apples.BANANAS", append(make([]interface{}, 0), "apples", ".", "BANANAS")}, - // {"appl*.BANA*", append(make([]interface{}, 0), "appl*", ".", "BANA*")}, - // {"a.b.**", append(make([]interface{}, 0), "a", ".", "b", ".", "**")}, - // {"a.\"=\".frog", append(make([]interface{}, 0), "a", ".", "=", ".", "frog")}, - // {"a.b.*", append(make([]interface{}, 0), "a", ".", "b", ".", "*")}, - // {"a.b.thin*", append(make([]interface{}, 0), "a", ".", "b", ".", "thin*")}, - // {"a.b[0]", append(make([]interface{}, 0), "a", ".", "b", ".", int64(0))}, - // {"a.b.[0]", append(make([]interface{}, 0), "a", ".", "b", ".", int64(0))}, - // {"a.b[*]", append(make([]interface{}, 0), "a", ".", "b", ".", "[*]")}, - // {"a.b.[*]", append(make([]interface{}, 0), "a", ".", "b", ".", "[*]")}, - // {"a.b[+]", append(make([]interface{}, 0), "a", ".", "b", ".", "[+]")}, - // {"a.b.[+]", append(make([]interface{}, 0), "a", ".", "b", ".", "[+]")}, - // {"a.b[-12]", append(make([]interface{}, 0), "a", ".", "b", ".", int64(-12))}, - // {"a.b.0", append(make([]interface{}, 0), "a", ".", "b", ".", int64(0))}, - // // {"a.b.-12", append(make([]interface{}, 0), "a", ".", "b", ".", int64(-12))}, - // {"a", append(make([]interface{}, 0), "a")}, - // {"\"a.b\".c", append(make([]interface{}, 0), "a.b", ".", "c")}, - // {`b."foo.bar"`, append(make([]interface{}, 0), "b", ".", "foo.bar")}, - // {"animals(.==cat)", append(make([]interface{}, 0), "animals", ".", "(", "SELF", ".==", "cat", ")")}, - // {"animals.(.==cat)", append(make([]interface{}, 0), "animals", ".", "(", "SELF", ".==", "cat", ")")}, - // {"animals(. == cat)", append(make([]interface{}, 0), "animals", ".", "(", "SELF", ". == ", "cat", ")")}, - // {"animals(.==c*)", append(make([]interface{}, 0), "animals", ".", "(", "SELF", ".==", "c*", ")")}, - // {"animals(a.b==c*)", append(make([]interface{}, 0), "animals", ".", "(", "a", ".", "b", "==", "c*", ")")}, - // {"animals.(a.b==c*)", append(make([]interface{}, 0), "animals", ".", "(", "a", ".", "b", "==", "c*", ")")}, - // {"(a.b==c*).animals", append(make([]interface{}, 0), "(", "a", ".", "b", "==", "c*", ")", ".", "animals")}, - // {"(a.b==c*)animals", append(make([]interface{}, 0), "(", "a", ".", "b", "==", "c*", ")", ".", "animals")}, - // {"[1].a.d", append(make([]interface{}, 0), int64(1), ".", "a", ".", "d")}, - // {"[1]a.d", append(make([]interface{}, 0), int64(1), ".", "a", ".", "d")}, - // {"a[0]c", append(make([]interface{}, 0), "a", ".", int64(0), ".", "c")}, - // {"a.[0].c", append(make([]interface{}, 0), "a", ".", int64(0), ".", "c")}, + // {"apples.BANANAS", append(make([]interface{}, 0), "apples", "TRAVERSE", "BANANAS")}, + // {"appl*.BANA*", append(make([]interface{}, 0), "appl*", "TRAVERSE", "BANA*")}, + // {"a.b.**", append(make([]interface{}, 0), "a", "TRAVERSE", "b", "TRAVERSE", "**")}, + // {"a.\"=\".frog", append(make([]interface{}, 0), "a", "TRAVERSE", "=", "TRAVERSE", "frog")}, + // {"a.b.*", append(make([]interface{}, 0), "a", "TRAVERSE", "b", "TRAVERSE", "*")}, + // {"a.b.thin*", append(make([]interface{}, 0), "a", "TRAVERSE", "b", "TRAVERSE", "thin*")}, + // {".a.b.[0]", append(make([]interface{}, 0), "a", "TRAVERSE", "b", "TRAVERSE", int64(0))}, + // {".a.b.[]", append(make([]interface{}, 0), "a", "TRAVERSE", "b", "TRAVERSE", "[]")}, + // {".a.b.[+]", append(make([]interface{}, 0), "a", "TRAVERSE", "b", "TRAVERSE", "[+]")}, + // {".a.b.[-12]", append(make([]interface{}, 0), "a", "TRAVERSE", "b", "TRAVERSE", int64(-12))}, + // {".a.b.0", append(make([]interface{}, 0), "a", "TRAVERSE", "b", "TRAVERSE", "0")}, + // {".a", append(make([]interface{}, 0), "a")}, + // {".\"a.b\".c", append(make([]interface{}, 0), "a.b", "TRAVERSE", "c")}, + // {`.b."foo.bar"`, append(make([]interface{}, 0), "b", "TRAVERSE", "foo.bar")}, + // {`f | . == *og | length`, append(make([]interface{}, 0), "f", "TRAVERSE", "SELF", "EQUALS", "*og", "TRAVERSE", "LENGTH")}, + // {`.a, .b`, append(make([]interface{}, 0), "a", "OR", "b")}, + // {`[.a, .b]`, append(make([]interface{}, 0), "[", "a", "OR", "b", "]")}, + // {`."[a", ."b]"`, append(make([]interface{}, 0), "[a", "OR", "b]")}, + // {`.a[]`, append(make([]interface{}, 0), "a", "PIPE", "[]")}, + // {`.[].a`, append(make([]interface{}, 0), "[]", "PIPE", "a")}, + {`.a | (.[].b == "apple")`, append(make([]interface{}, 0), "a", "PIPE", "(", "[]", "PIPE", "b", "EQUALS", "apple", ")")}, + + // {".animals | .==cat", append(make([]interface{}, 0), "animals", "TRAVERSE", "SELF", "EQUALS", "cat")}, + // {".animals | (. == cat)", append(make([]interface{}, 0), "animals", "TRAVERSE", "(", "SELF", "EQUALS", "cat", ")")}, + // {".animals | (.==c*)", append(make([]interface{}, 0), "animals", "TRAVERSE", "(", "SELF", "EQUALS", "c*", ")")}, + // {"animals(a.b==c*)", append(make([]interface{}, 0), "animals", "TRAVERSE", "(", "a", "TRAVERSE", "b", "==", "c*", ")")}, + // {"animals.(a.b==c*)", append(make([]interface{}, 0), "animals", "TRAVERSE", "(", "a", "TRAVERSE", "b", "==", "c*", ")")}, + // {"(a.b==c*).animals", append(make([]interface{}, 0), "(", "a", "TRAVERSE", "b", "==", "c*", ")", "TRAVERSE", "animals")}, + // {"(a.b==c*)animals", append(make([]interface{}, 0), "(", "a", "TRAVERSE", "b", "==", "c*", ")", "TRAVERSE", "animals")}, + // {"[1].a.d", append(make([]interface{}, 0), int64(1), "TRAVERSE", "a", "TRAVERSE", "d")}, + // {"[1]a.d", append(make([]interface{}, 0), int64(1), "TRAVERSE", "a", "TRAVERSE", "d")}, + // {"a[0]c", append(make([]interface{}, 0), "a", "TRAVERSE", int64(0), "TRAVERSE", "c")}, + // {"a.[0].c", append(make([]interface{}, 0), "a", "TRAVERSE", int64(0), "TRAVERSE", "c")}, // {"[0]", append(make([]interface{}, 0), int64(0))}, // {"0", append(make([]interface{}, 0), int64(0))}, - // {"a.b[+]c", append(make([]interface{}, 0), "a", ".", "b", ".", "[+]", ".", "c")}, - // {"a.cool(s.d.f == cool)", append(make([]interface{}, 0), "a", ".", "cool", ".", "(", "s", ".", "d", ".", "f", " == ", "cool", ")")}, - // {"a.cool.(s.d.f==cool OR t.b.h==frog).caterpillar", append(make([]interface{}, 0), "a", ".", "cool", ".", "(", "s", ".", "d", ".", "f", "==", "cool", "OR", "t", ".", "b", ".", "h", "==", "frog", ")", ".", "caterpillar")}, - // {"a.cool(s.d.f==cool and t.b.h==frog)*", append(make([]interface{}, 0), "a", ".", "cool", ".", "(", "s", ".", "d", ".", "f", "==", "cool", "and", "t", ".", "b", ".", "h", "==", "frog", ")", ".", "*")}, - // {"a.cool(s.d.f==cool and t.b.h==frog).th*", append(make([]interface{}, 0), "a", ".", "cool", ".", "(", "s", ".", "d", ".", "f", "==", "cool", "and", "t", ".", "b", ".", "h", "==", "frog", ")", ".", "th*")}, + // {"a.b[+]c", append(make([]interface{}, 0), "a", "TRAVERSE", "b", "TRAVERSE", "[+]", "TRAVERSE", "c")}, + // {"a.cool(s.d.f == cool)", append(make([]interface{}, 0), "a", "TRAVERSE", "cool", "TRAVERSE", "(", "s", "TRAVERSE", "d", "TRAVERSE", "f", " == ", "cool", ")")}, + // {"a.cool.(s.d.f==cool OR t.b.h==frog).caterpillar", append(make([]interface{}, 0), "a", "TRAVERSE", "cool", "TRAVERSE", "(", "s", "TRAVERSE", "d", "TRAVERSE", "f", "==", "cool", "OR", "t", "TRAVERSE", "b", "TRAVERSE", "h", "==", "frog", ")", "TRAVERSE", "caterpillar")}, + // {"a.cool(s.d.f==cool and t.b.h==frog)*", append(make([]interface{}, 0), "a", "TRAVERSE", "cool", "TRAVERSE", "(", "s", "TRAVERSE", "d", "TRAVERSE", "f", "==", "cool", "and", "t", "TRAVERSE", "b", "TRAVERSE", "h", "==", "frog", ")", "TRAVERSE", "*")}, + // {"a.cool(s.d.f==cool and t.b.h==frog).th*", append(make([]interface{}, 0), "a", "TRAVERSE", "cool", "TRAVERSE", "(", "s", "TRAVERSE", "d", "TRAVERSE", "f", "==", "cool", "and", "t", "TRAVERSE", "b", "TRAVERSE", "h", "==", "frog", ")", "TRAVERSE", "th*")}, } var tokeniser = NewPathTokeniser() @@ -68,6 +71,6 @@ func TestTokeniser(t *testing.T) { for _, token := range tokens { tokenValues = append(tokenValues, token.Value) } - test.AssertResultComplex(t, tt.expectedTokens, tokenValues) + test.AssertResultComplexWithContext(t, tt.expectedTokens, tokenValues, tt.path) } } diff --git a/pkg/yqlib/treeops/path_tree.go b/pkg/yqlib/treeops/path_tree.go index 3ca09e32..ce7c6523 100644 --- a/pkg/yqlib/treeops/path_tree.go +++ b/pkg/yqlib/treeops/path_tree.go @@ -45,12 +45,10 @@ func (p *pathTreeCreator) CreatePathTree(postFixPath []*PathElement) (*PathTreeN for _, pathElement := range postFixPath { var newNode = PathTreeNode{PathElement: pathElement} - if pathElement.PathElementType == Operation { + log.Debugf("pathTree %v ", pathElement.toString()) + if pathElement.PathElementType == Operation && pathElement.OperationType.NumArgs > 0 { numArgs := pathElement.OperationType.NumArgs - if numArgs == 0 { - remaining := stack[:len(stack)-1] - stack = remaining - } else if numArgs == 1 { + if numArgs == 1 { remaining, rhs := stack[:len(stack)-1], stack[len(stack)-1] newNode.Rhs = rhs stack = remaining diff --git a/test/utils.go b/test/utils.go index de621519..3cd61385 100644 --- a/test/utils.go +++ b/test/utils.go @@ -54,6 +54,14 @@ func AssertResultComplex(t *testing.T, expectedValue interface{}, actualValue in } } +func AssertResultComplexWithContext(t *testing.T, expectedValue interface{}, actualValue interface{}, context interface{}) { + t.Helper() + if !reflect.DeepEqual(expectedValue, actualValue) { + t.Error(context) + t.Error("\nExpected <", expectedValue, ">\nbut got <", actualValue, ">", fmt.Sprintf("%T", actualValue)) + } +} + func AssertResultWithContext(t *testing.T, expectedValue interface{}, actualValue interface{}, context interface{}) { t.Helper() if expectedValue != actualValue {