refactoring, fixing

pkg/yqlib/treeops/candidate_node.go

@@ -15,7 +15,7 @@ type CandidateNode struct {
 }
 
 func (n *CandidateNode) GetKey() string {
-	return fmt.Sprintf("%v - %v", n.Document, n.Path)
+	return fmt.Sprintf("%v - %v - %v", n.Document, n.Path, n.Node.Value)
 }
 
 // updates this candidate from the given candidate node

pkg/yqlib/treeops/data_tree_navigator.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	"github.com/elliotchance/orderedmap"
+	"gopkg.in/op/go-logging.v1"
 )
 
 type dataTreeNavigator struct {
@@ -68,6 +69,13 @@ func (d *dataTreeNavigator) getMatchingNodes(matchingNodes *orderedmap.OrderedMa
 		return matchingNodes, nil
 	}
 	log.Debugf("Processing Path: %v", pathNode.PathElement.toString())
+	if log.IsEnabledFor(logging.DEBUG) {
+		for el := matchingNodes.Front(); el != nil; el = el.Next() {
+			log.Debug(NodeToString(el.Value.(*CandidateNode)))
+		}
+	}
+	log.Debug(">>")
+
 	if pathNode.PathElement.PathElementType == SelfReference {
 		return matchingNodes, nil
 	} else if pathNode.PathElement.PathElementType == PathKey {

pkg/yqlib/treeops/data_tree_navigator_test.go

@@ -12,6 +12,9 @@ var treeNavigator = NewDataTreeNavigator(NavigationPrefs{})
 var treeCreator = NewPathTreeCreator()
 
 func readDoc(t *testing.T, content string) []*CandidateNode {
+	if content == "" {
+		return []*CandidateNode{}
+	}
 	decoder := yaml.NewDecoder(strings.NewReader(content))
 	var dataBucket yaml.Node
 	err := decoder.Decode(&dataBucket)
@@ -21,10 +24,10 @@ func readDoc(t *testing.T, content string) []*CandidateNode {
 	return []*CandidateNode{&CandidateNode{Node: dataBucket.Content[0], Document: 0}}
 }
 
-func resultsToString(results []*CandidateNode) string {
-	var pretty string = ""
+func resultsToString(results []*CandidateNode) []string {
+	var pretty []string = make([]string, 0)
 	for _, n := range results {
-		pretty = pretty + "\n" + NodeToString(n)
+		pretty = append(pretty, NodeToString(n))
 	}
 	return pretty
 }
@@ -1052,242 +1055,3 @@ func TestDataTreeNavigatorAnd(t *testing.T) {
 
 	test.AssertResult(t, expected, resultsToString(results))
 }
-
-func TestDataTreeNavigatorEqualsSimple(t *testing.T) {
-
-	nodes := readDoc(t, `a: 
-  cat: {b: apple, c: yes}
-  pat: {b: banana}
-`)
-
-	path, errPath := treeCreator.ParsePath(".a | (.[].b == \"apple\")")
-	if errPath != nil {
-		t.Error(errPath)
-	}
-	results, errNav := treeNavigator.GetMatchingNodes(nodes, path)
-
-	if errNav != nil {
-		t.Error(errNav)
-	}
-
-	expected := `
--- Node --
-  Document 0, path: [a cat]
-  Tag: !!map, Kind: MappingNode, Anchor: 
-  {b: apple, c: yes}
-`
-
-	test.AssertResult(t, expected, resultsToString(results))
-}
-
-func TestDataTreeNavigatorEqualsSelf(t *testing.T) {
-
-	nodes := readDoc(t, `a: frog
-b: cat
-c: frog`)
-
-	path, errPath := treeCreator.ParsePath("(a or b).(. == frog)")
-	if errPath != nil {
-		t.Error(errPath)
-	}
-	results, errNav := treeNavigator.GetMatchingNodes(nodes, path)
-
-	if errNav != nil {
-		t.Error(errNav)
-	}
-
-	expected := `
--- Node --
-  Document 0, path: [a]
-  Tag: !!str, Kind: ScalarNode, Anchor: 
-  frog
-`
-
-	test.AssertResult(t, expected, resultsToString(results))
-}
-
-func TestDataTreeNavigatorEqualsNested(t *testing.T) {
-
-	nodes := readDoc(t, `a: {t: frog}
-b: {t: cat}
-c: {t: frog}`)
-
-	path, errPath := treeCreator.ParsePath("(t == frog)")
-	if errPath != nil {
-		t.Error(errPath)
-	}
-	results, errNav := treeNavigator.GetMatchingNodes(nodes, path)
-
-	if errNav != nil {
-		t.Error(errNav)
-	}
-
-	expected := `
--- Node --
-  Document 0, path: [a]
-  Tag: !!map, Kind: MappingNode, Anchor: 
-  {t: frog}
-
--- Node --
-  Document 0, path: [c]
-  Tag: !!map, Kind: MappingNode, Anchor: 
-  {t: frog}
-`
-
-	test.AssertResult(t, expected, resultsToString(results))
-}
-
-func TestDataTreeNavigatorArrayEqualsSelf(t *testing.T) {
-
-	nodes := readDoc(t, `- cat
-- dog
-- frog`)
-
-	path, errPath := treeCreator.ParsePath("(. == *og)")
-	if errPath != nil {
-		t.Error(errPath)
-	}
-	results, errNav := treeNavigator.GetMatchingNodes(nodes, path)
-
-	if errNav != nil {
-		t.Error(errNav)
-	}
-
-	expected := `
--- Node --
-  Document 0, path: [1]
-  Tag: !!str, Kind: ScalarNode, Anchor: 
-  dog
-
--- Node --
-  Document 0, path: [2]
-  Tag: !!str, Kind: ScalarNode, Anchor: 
-  frog
-`
-
-	test.AssertResult(t, expected, resultsToString(results))
-}
-
-func TestDataTreeNavigatorArrayEqualsSelfSplatFirst(t *testing.T) {
-
-	nodes := readDoc(t, `- cat
-- dog
-- frog`)
-
-	path, errPath := treeCreator.ParsePath("*(. == *og)")
-	if errPath != nil {
-		t.Error(errPath)
-	}
-	results, errNav := treeNavigator.GetMatchingNodes(nodes, path)
-
-	if errNav != nil {
-		t.Error(errNav)
-	}
-
-	expected := `
--- Node --
-  Document 0, path: [1]
-  Tag: !!str, Kind: ScalarNode, Anchor: 
-  dog
-
--- Node --
-  Document 0, path: [2]
-  Tag: !!str, Kind: ScalarNode, Anchor: 
-  frog
-`
-
-	test.AssertResult(t, expected, resultsToString(results))
-}
-
-func TestDataTreeNavigatorArrayEquals(t *testing.T) {
-
-	nodes := readDoc(t, `- { b: apple, animal: rabbit }
-- { b: banana, animal: cat }
-- { b: corn, animal: dog }`)
-
-	path, errPath := treeCreator.ParsePath("(b == apple or animal == dog)")
-	if errPath != nil {
-		t.Error(errPath)
-	}
-	results, errNav := treeNavigator.GetMatchingNodes(nodes, path)
-
-	if errNav != nil {
-		t.Error(errNav)
-	}
-
-	expected := `
--- Node --
-  Document 0, path: [0]
-  Tag: !!map, Kind: MappingNode, Anchor: 
-  {b: apple, animal: rabbit}
-
--- Node --
-  Document 0, path: [2]
-  Tag: !!map, Kind: MappingNode, Anchor: 
-  {b: corn, animal: dog}
-`
-
-	test.AssertResult(t, expected, resultsToString(results))
-}
-
-func TestDataTreeNavigatorArrayEqualsDeep(t *testing.T) {
-
-	nodes := readDoc(t, `apples:
-  - { b: apple, animal: {legs: 2} }
-  - { b: banana, animal: {legs: 4} }
-  - { b: corn, animal: {legs: 6} }
-`)
-
-	path, errPath := treeCreator.ParsePath("apples(animal.legs == 4)")
-	if errPath != nil {
-		t.Error(errPath)
-	}
-	results, errNav := treeNavigator.GetMatchingNodes(nodes, path)
-
-	if errNav != nil {
-		t.Error(errNav)
-	}
-
-	expected := `
--- Node --
-  Document 0, path: [apples 1]
-  Tag: !!map, Kind: MappingNode, Anchor: 
-  {b: banana, animal: {legs: 4}}
-`
-
-	test.AssertResult(t, expected, resultsToString(results))
-}
-
-func xTestDataTreeNavigatorEqualsTrickey(t *testing.T) {
-
-	nodes := readDoc(t, `a: 
-  cat: {b: apso, c: {d : yes}}
-  pat: {b: apple, c: {d : no}}
-  sat: {b: apsy, c: {d : yes}}
-  fat: {b: apple}
-`)
-
-	path, errPath := treeCreator.ParsePath(".a(.b == ap* and .c.d == yes)")
-	if errPath != nil {
-		t.Error(errPath)
-	}
-	results, errNav := treeNavigator.GetMatchingNodes(nodes, path)
-
-	if errNav != nil {
-		t.Error(errNav)
-	}
-
-	expected := `
--- Node --
-  Document 0, path: [a cat]
-  Tag: !!map, Kind: MappingNode, Anchor: 
-  {b: apso, c: {d: yes}}
-
--- Node --
-  Document 0, path: [a sat]
-  Tag: !!map, Kind: MappingNode, Anchor: 
-  {b: apsy, c: {d: yes}}
-`
-
-	test.AssertResult(t, expected, resultsToString(results))
-}

pkg/yqlib/treeops/leaf_traverser.go

@@ -19,7 +19,7 @@ func NewLeafTraverser(navigationPrefs NavigationPrefs) LeafTraverser {
 }
 
 func (t *traverser) keyMatches(key *yaml.Node, pathNode *PathElement) bool {
-	return Match(key.Value, pathNode.StringValue)
+	return pathNode.Value == "[]" || Match(key.Value, pathNode.StringValue)
 }
 
 func (t *traverser) traverseMap(candidate *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) {

pkg/yqlib/treeops/lib.go

@@ -33,19 +33,24 @@ type OperationType struct {
 
 var None = &OperationType{Type: "NONE", NumArgs: 0, Precedence: 0}
 
-var Or = &OperationType{Type: "OR", NumArgs: 2, Precedence: 10, Handler: UnionOperator}
-var And = &OperationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: IntersectionOperator}
+var Or = &OperationType{Type: "OR", NumArgs: 2, Precedence: 20, Handler: OrOperator}
+var And = &OperationType{Type: "AND", NumArgs: 2, Precedence: 20, Handler: AndOperator}
+
+var Union = &OperationType{Type: "UNION", NumArgs: 2, Precedence: 10, Handler: UnionOperator}
+var Intersection = &OperationType{Type: "INTERSECTION", NumArgs: 2, Precedence: 20, Handler: IntersectionOperator}
 
 var Assign = &OperationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: AssignOperator}
 var Equals = &OperationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: EqualsOperator}
 var Pipe = &OperationType{Type: "PIPE", NumArgs: 2, Precedence: 45, Handler: PipeOperator}
 
 var Length = &OperationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: LengthOperator}
+var Collect = &OperationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: CollectOperator}
 
 // not sure yet
 
+var Select = &OperationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Handler: SelectOperator}
+
 var DeleteChild = &OperationType{Type: "DELETE", NumArgs: 2, Precedence: 40, Handler: DeleteChildOperator}
-var Collect = &OperationType{Type: "COLLECT", NumArgs: 1, Precedence: 40, Handler: CollectOperator}
 
 // var Splat = &OperationType{Type: "SPLAT", NumArgs: 0, Precedence: 40, Handler: SplatOperator}
 
@@ -64,13 +69,13 @@ func (p *PathElement) toString() string {
 	var result string = ``
 	switch p.PathElementType {
 	case PathKey:
-		result = result + fmt.Sprintf("PathKey - %v", p.Value)
+		result = result + fmt.Sprintf("%v", p.Value)
 	case SelfReference:
 		result = result + fmt.Sprintf("SELF")
 	case Operation:
-		result = result + fmt.Sprintf("Operation - %v", p.OperationType.Type)
+		result = result + fmt.Sprintf("%v", p.OperationType.Type)
 	case Value:
-		result = result + fmt.Sprintf("Value - %v (%T)", p.Value, p.Value)
+		result = result + fmt.Sprintf("%v (%T)", p.Value, p.Value)
 	default:
 		result = result + "I HAVENT GOT A STRATEGY"
 	}
@@ -124,7 +129,7 @@ func NodeToString(node *CandidateNode) string {
 	}
 	value := node.Node
 	if value == nil {
-		return "-- node is nil --"
+		return "-- nil --"
 	}
 	buf := new(bytes.Buffer)
 	encoder := yaml.NewEncoder(buf)
@@ -133,10 +138,7 @@ func NodeToString(node *CandidateNode) string {
 		log.Error("Error debugging node, %v", errorEncoding.Error())
 	}
 	encoder.Close()
-	return fmt.Sprintf(`-- Node --
-  Document %v, path: %v
-  Tag: %v, Kind: %v, Anchor: %v
-  %v`, node.Document, node.Path, value.Tag, KindString(value.Kind), value.Anchor, buf.String())
+	return fmt.Sprintf(`D%v, P%v, (%v)::%v`, node.Document, node.Path, value.Tag, buf.String())
 }
 
 func KindString(kind yaml.Kind) string {

pkg/yqlib/treeops/operator_booleans.go

@@ -0,0 +1,72 @@
+package treeops
+
+import (
+	"github.com/elliotchance/orderedmap"
+	"gopkg.in/yaml.v3"
+)
+
+func isTruthy(c *CandidateNode) (bool, error) {
+	node := c.Node
+	value := true
+	if node.Kind == yaml.ScalarNode && node.Tag == "!!bool" {
+		errDecoding := node.Decode(&value)
+		if errDecoding != nil {
+			return false, errDecoding
+		}
+
+	}
+	return value, nil
+}
+
+type boolOp func(bool, bool) bool
+
+func booleanOp(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode, op boolOp) (*orderedmap.OrderedMap, error) {
+	var results = orderedmap.NewOrderedMap()
+
+	for el := matchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		lhs, err := d.getMatchingNodes(nodeToMap(candidate), pathNode.Lhs)
+		if err != nil {
+			return nil, err
+		}
+		rhs, err := d.getMatchingNodes(nodeToMap(candidate), pathNode.Rhs)
+		if err != nil {
+			return nil, err
+		}
+
+		for lhsChild := lhs.Front(); lhsChild != nil; lhsChild = lhsChild.Next() {
+			lhsCandidate := lhsChild.Value.(*CandidateNode)
+			lhsTrue, errDecoding := isTruthy(lhsCandidate)
+			if errDecoding != nil {
+				return nil, errDecoding
+			}
+
+			for rhsChild := rhs.Front(); rhsChild != nil; rhsChild = rhsChild.Next() {
+				rhsCandidate := rhsChild.Value.(*CandidateNode)
+				rhsTrue, errDecoding := isTruthy(rhsCandidate)
+				if errDecoding != nil {
+					return nil, errDecoding
+				}
+				boolResult := createBooleanCandidate(lhsCandidate, op(lhsTrue, rhsTrue))
+
+				results.Set(boolResult.GetKey(), boolResult)
+			}
+		}
+
+	}
+	return results, nil
+}
+
+func OrOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
+	log.Debugf("-- orOp")
+	return booleanOp(d, matchingNodes, pathNode, func(b1 bool, b2 bool) bool {
+		return b1 || b2
+	})
+}
+
+func AndOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
+	log.Debugf("-- AndOp")
+	return booleanOp(d, matchingNodes, pathNode, func(b1 bool, b2 bool) bool {
+		return b1 && b2
+	})
+}

pkg/yqlib/treeops/operator_booleans_test.go

@@ -0,0 +1,35 @@
+package treeops
+
+import (
+	"testing"
+)
+
+var booleanOperatorScenarios = []expressionScenario{
+	{
+		document:   `{}`,
+		expression: `true or false`,
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	}, {
+		document:   `{}`,
+		expression: `false or false`,
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
+		},
+	}, {
+		document:   `{a: true, b: false}`,
+		expression: `.[] or (false, true)`,
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+			"D0, P[b], (!!bool)::false\n",
+			"D0, P[b], (!!bool)::true\n",
+		},
+	},
+}
+
+func TestBooleanOperatorScenarios(t *testing.T) {
+	for _, tt := range booleanOperatorScenarios {
+		testScenario(t, &tt)
+	}
+}

pkg/yqlib/treeops/operator_collect.go

@@ -0,0 +1,32 @@
+package treeops
+
+import (
+	"github.com/elliotchance/orderedmap"
+	"gopkg.in/yaml.v3"
+)
+
+func CollectOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
+	log.Debugf("-- collectOperation")
+
+	var results = orderedmap.NewOrderedMap()
+
+	node := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+
+	var document uint = 0
+	var path []interface{}
+
+	for el := matchMap.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		log.Debugf("Collecting %v", NodeToString(candidate))
+		if path == nil && candidate.Path != nil && len(candidate.Path) > 1 {
+			path = candidate.Path[:len(candidate.Path)-1]
+			document = candidate.Document
+		}
+		node.Content = append(node.Content, candidate.Node)
+	}
+
+	collectC := &CandidateNode{Node: node, Document: document, Path: path}
+	results.Set(collectC.GetKey(), collectC)
+
+	return results, nil
+}

pkg/yqlib/treeops/operator_collect_test.go

@@ -0,0 +1,45 @@
+package treeops
+
+import (
+	"testing"
+)
+
+var collectOperatorScenarios = []expressionScenario{
+	{
+		document:   `{}`,
+		expression: `["cat"]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- cat\n",
+		},
+	}, {
+		document:   `{}`,
+		expression: `["cat", "dog"]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- cat\n- dog\n",
+		},
+	}, {
+		document:   `{}`,
+		expression: `1 | collect`,
+		expected: []string{
+			"D0, P[], (!!seq)::- 1\n",
+		},
+	}, {
+		document:   `[1,2,3]`,
+		expression: `[.[]]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- 1\n- 2\n- 3\n",
+		},
+	}, {
+		document:   `a: {b: [1,2,3]}`,
+		expression: `[.a.b[]]`,
+		expected: []string{
+			"D0, P[a b], (!!seq)::- 1\n- 2\n- 3\n",
+		},
+	},
+}
+
+func TestCollectOperatorScenarios(t *testing.T) {
+	for _, tt := range collectOperatorScenarios {
+		testScenario(t, &tt)
+	}
+}

pkg/yqlib/treeops/operator_equals.go

@@ -2,7 +2,6 @@ package treeops
 
 import (
 	"github.com/elliotchance/orderedmap"
-	"gopkg.in/yaml.v3"
 )
 
 func EqualsOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
@@ -18,15 +17,8 @@ func EqualsOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathN
 			return nil, errInChild
 		}
 
-		matchString := "true"
-		if !matches {
-			matchString = "false"
-		}
-
-		node := &yaml.Node{Kind: yaml.ScalarNode, Value: matchString, Tag: "!!bool"}
-		lengthCand := &CandidateNode{Node: node, Document: candidate.Document, Path: candidate.Path}
-		results.Set(candidate.GetKey(), lengthCand)
-
+		equalsCandidate := createBooleanCandidate(candidate, matches)
+		results.Set(equalsCandidate.GetKey(), equalsCandidate)
 	}
 
 	return results, nil

pkg/yqlib/treeops/operator_equals_test.go

@@ -0,0 +1,43 @@
+package treeops
+
+import (
+	"testing"
+)
+
+var equalsOperatorScenarios = []expressionScenario{
+	{
+		document:   `[cat,goat,dog]`,
+		expression: `(.[] == "*at")`,
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	}, {
+		document:   `[cat,goat,dog]`,
+		expression: `.[] | (. == "*at")`,
+		expected: []string{
+			"D0, P[0], (!!bool)::true\n",
+			"D0, P[1], (!!bool)::true\n",
+			"D0, P[2], (!!bool)::false\n",
+		},
+	}, {
+		document:   `[3, 4, 5]`,
+		expression: `.[] | (. == 4)`,
+		expected: []string{
+			"D0, P[0], (!!bool)::false\n",
+			"D0, P[1], (!!bool)::true\n",
+			"D0, P[2], (!!bool)::false\n",
+		},
+	}, {
+		document:   `a: { cat: {b: apple, c: whatever}, pat: {b: banana} }`,
+		expression: `.a | (.[].b == "apple")`,
+		expected: []string{
+			"D0, P[a], (!!bool)::true\n",
+		},
+	},
+}
+
+func TestEqualOperatorScenarios(t *testing.T) {
+	for _, tt := range equalsOperatorScenarios {
+		testScenario(t, &tt)
+	}
+}

pkg/yqlib/treeops/operator_select.go

@@ -0,0 +1,37 @@
+package treeops
+
+import (
+	"github.com/elliotchance/orderedmap"
+)
+
+func SelectOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
+
+	log.Debugf("-- selectOperation")
+	var results = orderedmap.NewOrderedMap()
+
+	for el := matchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+
+		rhs, err := d.getMatchingNodes(nodeToMap(candidate), pathNode.Rhs)
+
+		if err != nil {
+			return nil, err
+		}
+
+		// grab the first value
+		first := rhs.Front()
+
+		if first != nil {
+			result := first.Value.(*CandidateNode)
+			includeResult, errDecoding := isTruthy(result)
+			if errDecoding != nil {
+				return nil, errDecoding
+			}
+
+			if includeResult {
+				results.Set(candidate.GetKey(), candidate)
+			}
+		}
+	}
+	return results, nil
+}

pkg/yqlib/treeops/operator_select_test.go

@@ -0,0 +1,45 @@
+package treeops
+
+import (
+	"testing"
+)
+
+var selectOperatorScenarios = []expressionScenario{
+	{
+		document:   `[cat,goat,dog]`,
+		expression: `.[] | select(. == "*at")`,
+		expected: []string{
+			"D0, P[0], (!!str)::cat\n",
+			"D0, P[1], (!!str)::goat\n",
+		},
+	}, {
+		document:   `[hot, fot, dog]`,
+		expression: `.[] | select(. == "*at")`,
+		expected:   []string{},
+	}, {
+		document:   `a: [cat,goat,dog]`,
+		expression: `.a[] | select(. == "*at")`,
+		expected: []string{
+			"D0, P[a 0], (!!str)::cat\n",
+			"D0, P[a 1], (!!str)::goat\n"},
+	}, {
+		document:   `a: { things: cat, bob: goat, horse: dog }`,
+		expression: `.a[] | select(. == "*at")`,
+		expected: []string{
+			"D0, P[a things], (!!str)::cat\n",
+			"D0, P[a bob], (!!str)::goat\n"},
+	}, {
+		document:   `a: { things: {include: true}, notMe: {include: false}, andMe: {include: fold} }`,
+		expression: `.a[] | select(.include)`,
+		expected: []string{
+			"D0, P[a things], (!!map)::{include: true}\n",
+			"D0, P[a andMe], (!!map)::{include: fold}\n",
+		},
+	},
+}
+
+func TestSelectOperatorScenarios(t *testing.T) {
+	for _, tt := range selectOperatorScenarios {
+		testScenario(t, &tt)
+	}
+}

pkg/yqlib/treeops/operator_union.go

@@ -0,0 +1,19 @@
+package treeops
+
+import "github.com/elliotchance/orderedmap"
+
+func UnionOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
+	lhs, err := d.getMatchingNodes(matchingNodes, pathNode.Lhs)
+	if err != nil {
+		return nil, err
+	}
+	rhs, err := d.getMatchingNodes(matchingNodes, pathNode.Rhs)
+	if err != nil {
+		return nil, err
+	}
+	for el := rhs.Front(); el != nil; el = el.Next() {
+		node := el.Value.(*CandidateNode)
+		lhs.Set(node.GetKey(), node)
+	}
+	return lhs, nil
+}

pkg/yqlib/treeops/operator_union_test.go

@@ -0,0 +1,31 @@
+package treeops
+
+import (
+	"testing"
+)
+
+var unionOperatorScenarios = []expressionScenario{
+	{
+		document:   `{}`,
+		expression: `"cat", "dog"`,
+		expected: []string{
+			"D0, P[], (!!str)::cat\n",
+			"D0, P[], (!!str)::dog\n",
+		},
+	}, {
+		document:   `{a: frog}`,
+		expression: `1, true, "cat", .a`,
+		expected: []string{
+			"D0, P[], (!!int)::1\n",
+			"D0, P[], (!!bool)::true\n",
+			"D0, P[], (!!str)::cat\n",
+			"D0, P[a], (!!str)::frog\n",
+		},
+	},
+}
+
+func TestUnionOperatorScenarios(t *testing.T) {
+	for _, tt := range unionOperatorScenarios {
+		testScenario(t, &tt)
+	}
+}

pkg/yqlib/treeops/operators.go

@@ -17,6 +17,15 @@ func PipeOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pa
 	return d.getMatchingNodes(lhs, pathNode.Rhs)
 }
 
+func createBooleanCandidate(owner *CandidateNode, value bool) *CandidateNode {
+	valString := "true"
+	if !value {
+		valString = "false"
+	}
+	node := &yaml.Node{Kind: yaml.ScalarNode, Value: valString, Tag: "!!bool"}
+	return &CandidateNode{Node: node, Document: owner.Document, Path: owner.Path}
+}
+
 func nodeToMap(candidate *CandidateNode) *orderedmap.OrderedMap {
 	elMap := orderedmap.NewOrderedMap()
 	elMap.Set(candidate.GetKey(), candidate)
@@ -47,22 +56,6 @@ func AssignOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap,
 	return lhs, nil
 }
 
-func UnionOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
-	lhs, err := d.getMatchingNodes(matchingNodes, pathNode.Lhs)
-	if err != nil {
-		return nil, err
-	}
-	rhs, err := d.getMatchingNodes(matchingNodes, pathNode.Rhs)
-	if err != nil {
-		return nil, err
-	}
-	for el := rhs.Front(); el != nil; el = el.Next() {
-		node := el.Value.(*CandidateNode)
-		lhs.Set(node.GetKey(), node)
-	}
-	return lhs, nil
-}
-
 func IntersectionOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
 	lhs, err := d.getMatchingNodes(matchingNodes, pathNode.Lhs)
 	if err != nil {
@@ -82,16 +75,6 @@ func IntersectionOperator(d *dataTreeNavigator, matchingNodes *orderedmap.Ordere
 	return matchingNodeMap, nil
 }
 
-func splatNode(d *dataTreeNavigator, candidate *CandidateNode) (*orderedmap.OrderedMap, error) {
-	//need to splat matching nodes, then search through them
-	splatter := &PathTreeNode{PathElement: &PathElement{
-		PathElementType: PathKey,
-		Value:           "*",
-		StringValue:     "*",
-	}}
-	return d.getMatchingNodes(nodeToMap(candidate), splatter)
-}
-
 func LengthOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
 	log.Debugf("-- lengthOperation")
 	var results = orderedmap.NewOrderedMap()
@@ -117,29 +100,3 @@ func LengthOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathN
 
 	return results, nil
 }
-
-func CollectOperator(d *dataTreeNavigator, matchMap *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
-	log.Debugf("-- collectOperation")
-
-	var results = orderedmap.NewOrderedMap()
-
-	node := &yaml.Node{Kind: yaml.SequenceNode}
-
-	var document uint = 0
-	var path []interface{}
-
-	for el := matchMap.Front(); el != nil; el = el.Next() {
-		candidate := el.Value.(*CandidateNode)
-		if path == nil && candidate.Path != nil {
-			path = candidate.Path
-			document = candidate.Document
-		}
-		node.Content = append(node.Content, candidate.Node)
-	}
-
-	collectC := &CandidateNode{Node: node, Document: document, Path: path}
-	results.Set(collectC.GetKey(), collectC)
-
-	return results, nil
-
-}

pkg/yqlib/treeops/operators_test.go

@@ -0,0 +1,28 @@
+package treeops
+
+import (
+	"testing"
+
+	"github.com/mikefarah/yq/v3/test"
+)
+
+type expressionScenario struct {
+	document   string
+	expression string
+	expected   []string
+}
+
+func testScenario(t *testing.T, s *expressionScenario) {
+
+	nodes := readDoc(t, s.document)
+	path, errPath := treeCreator.ParsePath(s.expression)
+	if errPath != nil {
+		t.Error(errPath)
+	}
+	results, errNav := treeNavigator.GetMatchingNodes(nodes, path)
+
+	if errNav != nil {
+		t.Error(errNav)
+	}
+	test.AssertResultComplexWithContext(t, s.expected, resultsToString(results), s.expression)
+}

pkg/yqlib/treeops/path_parse_test.go

@@ -1,14 +1,16 @@
 package treeops
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/mikefarah/yq/v3/test"
 )
 
-var tokeniserTests = []struct {
+var pathTests = []struct {
 	path            string
 	expectedTokens  []interface{}
+	expectedPostFix []interface{}
 }{ // TODO: Ensure ALL documented examples have tests! sheesh
 	// {"len(.)", append(make([]interface{}, 0), "LENGTH", "(", "SELF", ")")},
 	// {"\"len\"(.)", append(make([]interface{}, 0), "len", "TRAVERSE", "(", "SELF", ")")},
@@ -37,7 +39,21 @@ var tokeniserTests = []struct {
 	// {`."[a", ."b]"`, append(make([]interface{}, 0), "[a", "OR", "b]")},
 	// {`.a[]`, append(make([]interface{}, 0), "a", "PIPE", "[]")},
 	// {`.[].a`, append(make([]interface{}, 0), "[]", "PIPE", "a")},
-	{`.a | (.[].b == "apple")`, append(make([]interface{}, 0), "a", "PIPE", "(", "[]", "PIPE", "b", "EQUALS", "apple", ")")},
+	{
+		`.a | (.[].b == "apple")`,
+		append(make([]interface{}, 0), "a", "PIPE", "(", "[]", "PIPE", "b", "EQUALS", "apple", ")"),
+		append(make([]interface{}, 0), "a", "[]", "b", "PIPE", "apple (string)", "EQUALS", "PIPE"),
+	},
+	{
+		`.[] | select(. == "*at")`,
+		append(make([]interface{}, 0), "[]", "PIPE", "SELECT", "(", "SELF", "EQUALS", "*at", ")"),
+		append(make([]interface{}, 0), "[]", "SELF", "*at (string)", "EQUALS", "SELECT", "PIPE"),
+	},
+	{
+		`[true]`,
+		append(make([]interface{}, 0), "[", true, "]"),
+		append(make([]interface{}, 0), "true (bool)", "COLLECT"),
+	},
 
 	// {".animals | .==cat", append(make([]interface{}, 0), "animals", "TRAVERSE", "SELF", "EQUALS", "cat")},
 	// {".animals | (. == cat)", append(make([]interface{}, 0), "animals", "TRAVERSE", "(", "SELF", "EQUALS", "cat", ")")},
@@ -61,8 +77,8 @@ var tokeniserTests = []struct {
 
 var tokeniser = NewPathTokeniser()
 
-func TestTokeniser(t *testing.T) {
-	for _, tt := range tokeniserTests {
+func TestPathParsing(t *testing.T) {
+	for _, tt := range pathTests {
 		tokens, err := tokeniser.Tokenise(tt.path)
 		if err != nil {
 			t.Error(tt.path, err)
@@ -71,6 +87,20 @@ func TestTokeniser(t *testing.T) {
 		for _, token := range tokens {
 			tokenValues = append(tokenValues, token.Value)
 		}
-		test.AssertResultComplexWithContext(t, tt.expectedTokens, tokenValues, tt.path)
+		test.AssertResultComplexWithContext(t, tt.expectedTokens, tokenValues, fmt.Sprintf("tokenise: %v", tt.path))
+
+		results, errorP := postFixer.ConvertToPostfix(tokens)
+
+		var readableResults []interface{}
+		for _, token := range results {
+			readableResults = append(readableResults, token.toString())
+		}
+
+		if errorP != nil {
+			t.Error(tt.path, err)
+		}
+
+		test.AssertResultComplexWithContext(t, tt.expectedPostFix, readableResults, fmt.Sprintf("postfix: %v", tt.path))
+
 	}
 }

pkg/yqlib/treeops/path_postfix.go

@@ -2,6 +2,8 @@ package treeops
 
 import (
 	"errors"
+
+	"gopkg.in/op/go-logging.v1"
 )
 
 type PathPostFixer interface {
@@ -43,9 +45,10 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*PathElement,
 			if len(opStack) == 0 {
 				return nil, errors.New("Bad path expression, got close collect brackets without matching opening bracket")
 			}
-			// now we should have ( as the last element on the opStack, get rid of it
+			// now we should have [] as the last element on the opStack, get rid of it
 			opStack = opStack[0 : len(opStack)-1]
 			//and append a collect to the opStack
+			opStack = append(opStack, &Token{PathElementType: Operation, OperationType: Pipe})
 			opStack = append(opStack, &Token{PathElementType: Operation, OperationType: Collect})
 		case CloseBracket:
 			for len(opStack) > 0 && opStack[len(opStack)-1].PathElementType != OpenBracket {
@@ -67,5 +70,13 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*PathElement,
 			opStack = append(opStack, token)
 		}
 	}
+
+	if log.IsEnabledFor(logging.DEBUG) {
+		log.Debugf("PostFix Result:")
+		for _, token := range result {
+			log.Debugf("> %v", token.toString())
+		}
+	}
+
 	return result, nil
 }

pkg/yqlib/treeops/path_postfix_test.go

@@ -20,11 +20,12 @@ func testExpression(expression string) (string, error) {
 	}
 	formatted := ""
 	for _, path := range results {
-		formatted = formatted + path.toString() + "\n--------\n"
+		formatted = formatted + path.toString() + ", "
 	}
 	return formatted, nil
 }
 
+
 func TestPostFixTraverseBar(t *testing.T) {
 	var infix = ".animals | [.]"
 	var expectedOutput = `PathKey - animals

pkg/yqlib/treeops/path_tokeniser.go

@@ -107,9 +107,12 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`\.?\[\]`), literalToken(PathKey, "[]", true))
 	lexer.Add([]byte(`\.\.`), literalToken(PathKey, "..", true))
 
-	lexer.Add([]byte(`,`), opToken(Or))
+	lexer.Add([]byte(`,`), opToken(Union))
 	lexer.Add([]byte(`length`), opToken(Length))
-	lexer.Add([]byte(`([Cc][Oo][Ll][Ll][Ee][Cc][Tt])`), opToken(Collect))
+	lexer.Add([]byte(`select`), opToken(Select))
+	lexer.Add([]byte(`or`), opToken(Or))
+	lexer.Add([]byte(`and`), opToken(And))
+	lexer.Add([]byte(`collect`), opToken(Collect))
 
 	lexer.Add([]byte(`\s*==\s*`), opToken(Equals))
 

pkg/yqlib/treeops/path_tree.go

@@ -52,7 +52,7 @@ func (p *pathTreeCreator) CreatePathTree(postFixPath []*PathElement) (*PathTreeN
 				remaining, rhs := stack[:len(stack)-1], stack[len(stack)-1]
 				newNode.Rhs = rhs
 				stack = remaining
-			} else {
+			} else if numArgs == 2 {
 				remaining, lhs, rhs := stack[:len(stack)-2], stack[len(stack)-2], stack[len(stack)-1]
 				newNode.Lhs = lhs
 				newNode.Rhs = rhs