autovivification, merge!

go.mod

@@ -4,7 +4,7 @@ require (
 	github.com/elliotchance/orderedmap v1.3.0
 	github.com/fatih/color v1.9.0
 	github.com/goccy/go-yaml v1.8.1
-	github.com/kylelemons/godebug v1.1.0
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.7 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.0.0

pkg/yqlib/treeops/candidate_node.go

@@ -20,11 +20,19 @@ func (n *CandidateNode) GetKey() string {
 
 // updates this candidate from the given candidate node
 func (n *CandidateNode) UpdateFrom(other *CandidateNode) {
+
 	n.Node.Content = other.Node.Content
 	n.Node.Value = other.Node.Value
+	n.UpdateAttributesFrom(other)
+}
+
+func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode) {
 	n.Node.Kind = other.Node.Kind
 	n.Node.Tag = other.Node.Tag
 	n.Node.Style = other.Node.Style
+	n.Node.FootComment = other.Node.FootComment
+	n.Node.HeadComment = other.Node.HeadComment
+	n.Node.LineComment = other.Node.LineComment
 }
 
 func (n *CandidateNode) PathStackToString() string {

pkg/yqlib/treeops/data_tree_navigator.go

@@ -81,7 +81,7 @@ func (d *dataTreeNavigator) getMatchingNodes(matchingNodes *orderedmap.OrderedMa
 	} else if pathNode.PathElement.PathElementType == PathKey {
 		return d.traverse(matchingNodes, pathNode.PathElement)
 	} else if pathNode.PathElement.PathElementType == Value {
-		return nodeToMap(BuildCandidateNodeFrom(pathNode.PathElement)), nil
+		return nodeToMap(pathNode.PathElement.CandidateNode), nil
 	} else {
 		handler := pathNode.PathElement.OperationType.Handler
 		if handler != nil {

pkg/yqlib/treeops/leaf_traverser.go

@@ -48,6 +48,18 @@ func (t *traverser) traverseMap(candidate *CandidateNode, pathNode *PathElement)
 			})
 		}
 	}
+	if len(newMatches) == 0 {
+		//no matches, create one automagically
+		valueNode := &yaml.Node{}
+		node.Content = append(node.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: pathNode.StringValue}, valueNode)
+		newMatches = append(newMatches, &CandidateNode{
+			Node:     valueNode,
+			Path:     append(candidate.Path, pathNode.StringValue),
+			Document: candidate.Document,
+		})
+
+	}
+
 	return newMatches, nil
 }
 
@@ -72,9 +84,9 @@ func (t *traverser) traverseArray(candidate *CandidateNode, pathNode *PathElemen
 	index := pathNode.Value.(int64)
 	indexToUse := index
 	contentLength := int64(len(candidate.Node.Content))
-	if contentLength <= index {
+	for contentLength <= index {
-		// handle auto append here
+		candidate.Node.Content = append(candidate.Node.Content, &yaml.Node{})
-		return make([]*CandidateNode, 0), nil
+		contentLength = int64(len(candidate.Node.Content))
 	}
 
 	if indexToUse < 0 {
@@ -94,8 +106,22 @@ func (t *traverser) traverseArray(candidate *CandidateNode, pathNode *PathElemen
 }
 
 func (t *traverser) Traverse(matchingNode *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) {
-	log.Debug(NodeToString(matchingNode))
+	log.Debug("Traversing %v", NodeToString(matchingNode))
 	value := matchingNode.Node
+
+	if value.Kind == 0 {
+		log.Debugf("Guessing kind")
+		// we must ahve added this automatically, lets guess what it should be now
+		switch pathNode.Value.(type) {
+		case int, int64:
+			log.Debugf("probably an array")
+			value.Kind = yaml.SequenceNode
+		default:
+			log.Debugf("probabel a map")
+			value.Kind = yaml.MappingNode
+		}
+	}
+
 	switch value.Kind {
 	case yaml.MappingNode:
 		log.Debug("its a map with %v entries", len(value.Content)/2)

pkg/yqlib/treeops/lib.go

@@ -40,6 +40,7 @@ var Union = &OperationType{Type: "UNION", NumArgs: 2, Precedence: 10, Handler: U
 var Intersection = &OperationType{Type: "INTERSECTION", NumArgs: 2, Precedence: 20, Handler: IntersectionOperator}
 
 var Assign = &OperationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: AssignOperator}
+var AssignAttributes = &OperationType{Type: "ASSIGN_ATTRIBUTES", NumArgs: 2, Precedence: 40, Handler: AssignAttributesOperator}
 var Multiply = &OperationType{Type: "MULTIPLY", NumArgs: 2, Precedence: 40, Handler: MultiplyOperator}
 
 var Equals = &OperationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: EqualsOperator}
@@ -65,6 +66,7 @@ type PathElement struct {
 	OperationType   *OperationType
 	Value           interface{}
 	StringValue     string
+	CandidateNode   *CandidateNode // used for Value Path elements
 }
 
 // debugging purposes only

pkg/yqlib/treeops/operator_assign.go

@@ -25,3 +25,28 @@ func AssignOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap,
 	}
 	return matchingNodes, nil
 }
+
+// does not update content or values
+func AssignAttributesOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
+	lhs, err := d.getMatchingNodes(matchingNodes, pathNode.Lhs)
+	if err != nil {
+		return nil, err
+	}
+	for el := lhs.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+
+		rhs, err := d.getMatchingNodes(nodeToMap(candidate), pathNode.Rhs)
+
+		if err != nil {
+			return nil, err
+		}
+
+		// grab the first value
+		first := rhs.Front()
+
+		if first != nil {
+			candidate.UpdateAttributesFrom(first.Value.(*CandidateNode))
+		}
+	}
+	return matchingNodes, nil
+}

pkg/yqlib/treeops/operator_assign_test.go

@@ -47,6 +47,24 @@ var assignOperatorScenarios = []expressionScenario{
 		expected: []string{
 			"D0, P[], (!!seq)::[bogs, apple, bogs]\n",
 		},
+	}, {
+		document:   `{}`,
+		expression: `.a.b |= "bogs"`,
+		expected: []string{
+			"D0, P[], (!!map)::{a: {b: bogs}}\n",
+		},
+	}, {
+		document:   `{}`,
+		expression: `.a.b[0] |= "bogs"`,
+		expected: []string{
+			"D0, P[], (!!map)::{a: {b: [bogs]}}\n",
+		},
+	}, {
+		document:   `{}`,
+		expression: `.a.b[1].c |= "bogs"`,
+		expected: []string{
+			"D0, P[], (!!map)::{a: {b: [null, {c: bogs}]}}\n",
+		},
 	},
 }
 

pkg/yqlib/treeops/operator_merge.go

@@ -1,20 +0,0 @@
-package treeops
-
-import "github.com/elliotchance/orderedmap"
-
-func MultiplyOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
-	lhs, err := d.getMatchingNodes(matchingNodes, pathNode.Lhs)
-	if err != nil {
-		return nil, err
-	}
-	for el := lhs.Front(); el != nil; el = el.Next() {
-		candidate := el.Value.(*CandidateNode)
-
-		// TODO handle scalar mulitplication
-		switch candidate.Node.Kind {
-			case 
-		}
-		
-	}
-	return matchingNodes, nil
-}

pkg/yqlib/treeops/operator_merge_test.go

@@ -1,27 +0,0 @@
-package treeops
-
-import (
-	"testing"
-)
-
-var mergeOperatorScenarios = []expressionScenario{
-	{
-		document:   `{a: frog, b: cat}`,
-		expression: `.a * .b`,
-		expected: []string{
-			"D0, P[], (!!map)::{a: cat, b: cat}\n",
-		},
-	}, {
-		document:   `{a: {things: great}, b: {also: me}}`,
-		expression: `.a * .b`,
-		expected: []string{
-			"D0, P[], (!!map)::{a: {also: me, things: great}, b: {also: me}}\n",
-		},
-	},
-}
-
-func TestMergeOperatorScenarios(t *testing.T) {
-	for _, tt := range mergeOperatorScenarios {
-		testScenario(t, &tt)
-	}
-}

pkg/yqlib/treeops/operator_multilpy.go

@@ -0,0 +1,90 @@
+package treeops
+
+import (
+	"fmt"
+
+	"github.com/elliotchance/orderedmap"
+	"gopkg.in/yaml.v3"
+)
+
+func MultiplyOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) {
+	lhs, err := d.getMatchingNodes(matchingNodes, pathNode.Lhs)
+	if err != nil {
+		return nil, err
+	}
+
+	rhs, err := d.getMatchingNodes(matchingNodes, pathNode.Rhs)
+
+	if err != nil {
+		return nil, err
+	}
+
+	var results = orderedmap.NewOrderedMap()
+
+	for el := lhs.Front(); el != nil; el = el.Next() {
+		lhsCandidate := el.Value.(*CandidateNode)
+
+		for rightEl := rhs.Front(); rightEl != nil; rightEl = rightEl.Next() {
+			rhsCandidate := rightEl.Value.(*CandidateNode)
+			resultCandidate, err := multiply(d, lhsCandidate, rhsCandidate)
+			if err != nil {
+				return nil, err
+			}
+			results.Set(resultCandidate.GetKey(), resultCandidate)
+		}
+
+	}
+	return matchingNodes, nil
+}
+
+func multiply(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode {
+		var results = orderedmap.NewOrderedMap()
+		recursiveDecent(d, results, nodeToMap(rhs))
+
+		var pathIndexToStartFrom int = 0
+		if results.Front() != nil {
+			pathIndexToStartFrom = len(results.Front().Value.(*CandidateNode).Path)
+		}
+
+		for el := results.Front(); el != nil; el = el.Next() {
+			err := applyAssignment(d, pathIndexToStartFrom, lhs, el.Value.(*CandidateNode))
+			if err != nil {
+				return nil, err
+			}
+		}
+		return lhs, nil
+	}
+	return nil, fmt.Errorf("Cannot multiply %v with %v", NodeToString(lhs), NodeToString(rhs))
+}
+
+func createTraversalTree(path []interface{}) *PathTreeNode {
+	if len(path) == 0 {
+		return &PathTreeNode{PathElement: &PathElement{PathElementType: SelfReference}}
+	} else if len(path) == 1 {
+		return &PathTreeNode{PathElement: &PathElement{PathElementType: PathKey, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}}
+	}
+	return &PathTreeNode{
+		PathElement: &PathElement{PathElementType: Operation, OperationType: Pipe},
+		Lhs:         createTraversalTree(path[0:1]),
+		Rhs:         createTraversalTree(path[1:])}
+
+}
+
+func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode) error {
+	log.Debugf("merge - applyAssignment lhs %v, rhs: %v", NodeToString(lhs), NodeToString(rhs))
+
+	lhsPath := rhs.Path[pathIndexToStartFrom:]
+
+	assignmentOp := &PathElement{PathElementType: Operation, OperationType: AssignAttributes}
+	if rhs.Node.Kind == yaml.ScalarNode {
+		assignmentOp.OperationType = Assign
+	}
+	rhsOp := &PathElement{PathElementType: Value, CandidateNode: rhs}
+
+	assignmentOpNode := &PathTreeNode{PathElement: assignmentOp, Lhs: createTraversalTree(lhsPath), Rhs: &PathTreeNode{PathElement: rhsOp}}
+
+	_, err := d.getMatchingNodes(nodeToMap(lhs), assignmentOpNode)
+
+	return err
+}

pkg/yqlib/treeops/operator_multiply_test.go

@@ -0,0 +1,27 @@
+package treeops
+
+import (
+	"testing"
+)
+
+var multiplyOperatorScenarios = []expressionScenario{
+	{
+		// document:   `{a: frog, b: cat}`,
+		// expression: `.a * .b`,
+		// expected: []string{
+		// 	"D0, P[], (!!map)::{a: cat, b: cat}\n",
+		// },
+		// }, {
+		document:   `{a: {things: great}, b: {also: me}}`,
+		expression: `.a * .b`,
+		expected: []string{
+			"D0, P[], (!!map)::{a: {things: great, also: me}, b: {also: me}}\n",
+		},
+	},
+}
+
+func TestMultiplyOperatorScenarios(t *testing.T) {
+	for _, tt := range multiplyOperatorScenarios {
+		testScenario(t, &tt)
+	}
+}

pkg/yqlib/treeops/path_postfix.go

@@ -33,7 +33,11 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*PathElement,
 	for _, token := range tokens {
 		log.Debugf("postfix processing token %v", token.Value)
 		switch token.PathElementType {
-		case PathKey, SelfReference, Value:
+		case Value:
+			var candidateNode = BuildCandidateNodeFrom(token)
+			var pathElement = PathElement{PathElementType: token.PathElementType, Value: token.Value, StringValue: token.StringValue, CandidateNode: candidateNode}
+			result = append(result, &pathElement)
+		case PathKey, SelfReference:
 			var pathElement = PathElement{PathElementType: token.PathElementType, Value: token.Value, StringValue: token.StringValue}
 			result = append(result, &pathElement)
 		case OpenBracket, OpenCollect:

pkg/yqlib/treeops/value_node_builder.go

@@ -2,11 +2,11 @@ package treeops
 
 import "gopkg.in/yaml.v3"
 
-func BuildCandidateNodeFrom(p *PathElement) *CandidateNode {
+func BuildCandidateNodeFrom(token *Token) *CandidateNode {
 	var node yaml.Node = yaml.Node{Kind: yaml.ScalarNode}
-	node.Value = p.StringValue
+	node.Value = token.StringValue
 
-	switch p.Value.(type) {
+	switch token.Value.(type) {
 	case float32, float64:
 		node.Tag = "!!float"
 	case int, int64, int32: