From 2ddf8dd4ed185428f16f7a9c39dba262d6104b4f Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Mon, 19 Oct 2020 16:14:29 +1100 Subject: [PATCH] autovivification, merge! --- go.mod | 2 +- pkg/yqlib/treeops/candidate_node.go | 8 ++ pkg/yqlib/treeops/data_tree_navigator.go | 2 +- pkg/yqlib/treeops/leaf_traverser.go | 34 +++++++- pkg/yqlib/treeops/lib.go | 2 + pkg/yqlib/treeops/operator_assign.go | 25 ++++++ pkg/yqlib/treeops/operator_assign_test.go | 18 +++++ pkg/yqlib/treeops/operator_merge.go | 20 ----- pkg/yqlib/treeops/operator_merge_test.go | 27 ------- pkg/yqlib/treeops/operator_multilpy.go | 90 +++++++++++++++++++++ pkg/yqlib/treeops/operator_multiply_test.go | 27 +++++++ pkg/yqlib/treeops/path_postfix.go | 6 +- pkg/yqlib/treeops/value_node_builder.go | 6 +- 13 files changed, 210 insertions(+), 57 deletions(-) delete mode 100644 pkg/yqlib/treeops/operator_merge.go delete mode 100644 pkg/yqlib/treeops/operator_merge_test.go create mode 100644 pkg/yqlib/treeops/operator_multilpy.go create mode 100644 pkg/yqlib/treeops/operator_multiply_test.go diff --git a/go.mod b/go.mod index 19831915..a77d1990 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ require ( github.com/elliotchance/orderedmap v1.3.0 github.com/fatih/color v1.9.0 github.com/goccy/go-yaml v1.8.1 - github.com/kylelemons/godebug v1.1.0 + github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-colorable v0.1.7 // indirect github.com/pkg/errors v0.9.1 github.com/spf13/cobra v1.0.0 diff --git a/pkg/yqlib/treeops/candidate_node.go b/pkg/yqlib/treeops/candidate_node.go index 961e9e29..4d98e8ac 100644 --- a/pkg/yqlib/treeops/candidate_node.go +++ b/pkg/yqlib/treeops/candidate_node.go @@ -20,11 +20,19 @@ func (n *CandidateNode) GetKey() string { // updates this candidate from the given candidate node func (n *CandidateNode) UpdateFrom(other *CandidateNode) { + n.Node.Content = other.Node.Content n.Node.Value = other.Node.Value + n.UpdateAttributesFrom(other) +} + +func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode) { n.Node.Kind = other.Node.Kind n.Node.Tag = other.Node.Tag n.Node.Style = other.Node.Style + n.Node.FootComment = other.Node.FootComment + n.Node.HeadComment = other.Node.HeadComment + n.Node.LineComment = other.Node.LineComment } func (n *CandidateNode) PathStackToString() string { diff --git a/pkg/yqlib/treeops/data_tree_navigator.go b/pkg/yqlib/treeops/data_tree_navigator.go index 4a798ead..8076647a 100644 --- a/pkg/yqlib/treeops/data_tree_navigator.go +++ b/pkg/yqlib/treeops/data_tree_navigator.go @@ -81,7 +81,7 @@ func (d *dataTreeNavigator) getMatchingNodes(matchingNodes *orderedmap.OrderedMa } else if pathNode.PathElement.PathElementType == PathKey { return d.traverse(matchingNodes, pathNode.PathElement) } else if pathNode.PathElement.PathElementType == Value { - return nodeToMap(BuildCandidateNodeFrom(pathNode.PathElement)), nil + return nodeToMap(pathNode.PathElement.CandidateNode), nil } else { handler := pathNode.PathElement.OperationType.Handler if handler != nil { diff --git a/pkg/yqlib/treeops/leaf_traverser.go b/pkg/yqlib/treeops/leaf_traverser.go index 3c19dc76..1b60b22b 100644 --- a/pkg/yqlib/treeops/leaf_traverser.go +++ b/pkg/yqlib/treeops/leaf_traverser.go @@ -48,6 +48,18 @@ func (t *traverser) traverseMap(candidate *CandidateNode, pathNode *PathElement) }) } } + if len(newMatches) == 0 { + //no matches, create one automagically + valueNode := &yaml.Node{} + node.Content = append(node.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: pathNode.StringValue}, valueNode) + newMatches = append(newMatches, &CandidateNode{ + Node: valueNode, + Path: append(candidate.Path, pathNode.StringValue), + Document: candidate.Document, + }) + + } + return newMatches, nil } @@ -72,9 +84,9 @@ func (t *traverser) traverseArray(candidate *CandidateNode, pathNode *PathElemen index := pathNode.Value.(int64) indexToUse := index contentLength := int64(len(candidate.Node.Content)) - if contentLength <= index { - // handle auto append here - return make([]*CandidateNode, 0), nil + for contentLength <= index { + candidate.Node.Content = append(candidate.Node.Content, &yaml.Node{}) + contentLength = int64(len(candidate.Node.Content)) } if indexToUse < 0 { @@ -94,8 +106,22 @@ func (t *traverser) traverseArray(candidate *CandidateNode, pathNode *PathElemen } func (t *traverser) Traverse(matchingNode *CandidateNode, pathNode *PathElement) ([]*CandidateNode, error) { - log.Debug(NodeToString(matchingNode)) + log.Debug("Traversing %v", NodeToString(matchingNode)) value := matchingNode.Node + + if value.Kind == 0 { + log.Debugf("Guessing kind") + // we must ahve added this automatically, lets guess what it should be now + switch pathNode.Value.(type) { + case int, int64: + log.Debugf("probably an array") + value.Kind = yaml.SequenceNode + default: + log.Debugf("probabel a map") + value.Kind = yaml.MappingNode + } + } + switch value.Kind { case yaml.MappingNode: log.Debug("its a map with %v entries", len(value.Content)/2) diff --git a/pkg/yqlib/treeops/lib.go b/pkg/yqlib/treeops/lib.go index 34be9b08..fca412fb 100644 --- a/pkg/yqlib/treeops/lib.go +++ b/pkg/yqlib/treeops/lib.go @@ -40,6 +40,7 @@ var Union = &OperationType{Type: "UNION", NumArgs: 2, Precedence: 10, Handler: U var Intersection = &OperationType{Type: "INTERSECTION", NumArgs: 2, Precedence: 20, Handler: IntersectionOperator} var Assign = &OperationType{Type: "ASSIGN", NumArgs: 2, Precedence: 40, Handler: AssignOperator} +var AssignAttributes = &OperationType{Type: "ASSIGN_ATTRIBUTES", NumArgs: 2, Precedence: 40, Handler: AssignAttributesOperator} var Multiply = &OperationType{Type: "MULTIPLY", NumArgs: 2, Precedence: 40, Handler: MultiplyOperator} var Equals = &OperationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Handler: EqualsOperator} @@ -65,6 +66,7 @@ type PathElement struct { OperationType *OperationType Value interface{} StringValue string + CandidateNode *CandidateNode // used for Value Path elements } // debugging purposes only diff --git a/pkg/yqlib/treeops/operator_assign.go b/pkg/yqlib/treeops/operator_assign.go index 0dc60900..60ee08ea 100644 --- a/pkg/yqlib/treeops/operator_assign.go +++ b/pkg/yqlib/treeops/operator_assign.go @@ -25,3 +25,28 @@ func AssignOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, } return matchingNodes, nil } + +// does not update content or values +func AssignAttributesOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { + lhs, err := d.getMatchingNodes(matchingNodes, pathNode.Lhs) + if err != nil { + return nil, err + } + for el := lhs.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + + rhs, err := d.getMatchingNodes(nodeToMap(candidate), pathNode.Rhs) + + if err != nil { + return nil, err + } + + // grab the first value + first := rhs.Front() + + if first != nil { + candidate.UpdateAttributesFrom(first.Value.(*CandidateNode)) + } + } + return matchingNodes, nil +} diff --git a/pkg/yqlib/treeops/operator_assign_test.go b/pkg/yqlib/treeops/operator_assign_test.go index 87a01a64..4ce1f0de 100644 --- a/pkg/yqlib/treeops/operator_assign_test.go +++ b/pkg/yqlib/treeops/operator_assign_test.go @@ -47,6 +47,24 @@ var assignOperatorScenarios = []expressionScenario{ expected: []string{ "D0, P[], (!!seq)::[bogs, apple, bogs]\n", }, + }, { + document: `{}`, + expression: `.a.b |= "bogs"`, + expected: []string{ + "D0, P[], (!!map)::{a: {b: bogs}}\n", + }, + }, { + document: `{}`, + expression: `.a.b[0] |= "bogs"`, + expected: []string{ + "D0, P[], (!!map)::{a: {b: [bogs]}}\n", + }, + }, { + document: `{}`, + expression: `.a.b[1].c |= "bogs"`, + expected: []string{ + "D0, P[], (!!map)::{a: {b: [null, {c: bogs}]}}\n", + }, }, } diff --git a/pkg/yqlib/treeops/operator_merge.go b/pkg/yqlib/treeops/operator_merge.go deleted file mode 100644 index 86ebbc14..00000000 --- a/pkg/yqlib/treeops/operator_merge.go +++ /dev/null @@ -1,20 +0,0 @@ -package treeops - -import "github.com/elliotchance/orderedmap" - -func MultiplyOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { - lhs, err := d.getMatchingNodes(matchingNodes, pathNode.Lhs) - if err != nil { - return nil, err - } - for el := lhs.Front(); el != nil; el = el.Next() { - candidate := el.Value.(*CandidateNode) - - // TODO handle scalar mulitplication - switch candidate.Node.Kind { - case - } - - } - return matchingNodes, nil -} diff --git a/pkg/yqlib/treeops/operator_merge_test.go b/pkg/yqlib/treeops/operator_merge_test.go deleted file mode 100644 index c0cc75fc..00000000 --- a/pkg/yqlib/treeops/operator_merge_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package treeops - -import ( - "testing" -) - -var mergeOperatorScenarios = []expressionScenario{ - { - document: `{a: frog, b: cat}`, - expression: `.a * .b`, - expected: []string{ - "D0, P[], (!!map)::{a: cat, b: cat}\n", - }, - }, { - document: `{a: {things: great}, b: {also: me}}`, - expression: `.a * .b`, - expected: []string{ - "D0, P[], (!!map)::{a: {also: me, things: great}, b: {also: me}}\n", - }, - }, -} - -func TestMergeOperatorScenarios(t *testing.T) { - for _, tt := range mergeOperatorScenarios { - testScenario(t, &tt) - } -} diff --git a/pkg/yqlib/treeops/operator_multilpy.go b/pkg/yqlib/treeops/operator_multilpy.go new file mode 100644 index 00000000..fa919cb5 --- /dev/null +++ b/pkg/yqlib/treeops/operator_multilpy.go @@ -0,0 +1,90 @@ +package treeops + +import ( + "fmt" + + "github.com/elliotchance/orderedmap" + "gopkg.in/yaml.v3" +) + +func MultiplyOperator(d *dataTreeNavigator, matchingNodes *orderedmap.OrderedMap, pathNode *PathTreeNode) (*orderedmap.OrderedMap, error) { + lhs, err := d.getMatchingNodes(matchingNodes, pathNode.Lhs) + if err != nil { + return nil, err + } + + rhs, err := d.getMatchingNodes(matchingNodes, pathNode.Rhs) + + if err != nil { + return nil, err + } + + var results = orderedmap.NewOrderedMap() + + for el := lhs.Front(); el != nil; el = el.Next() { + lhsCandidate := el.Value.(*CandidateNode) + + for rightEl := rhs.Front(); rightEl != nil; rightEl = rightEl.Next() { + rhsCandidate := rightEl.Value.(*CandidateNode) + resultCandidate, err := multiply(d, lhsCandidate, rhsCandidate) + if err != nil { + return nil, err + } + results.Set(resultCandidate.GetKey(), resultCandidate) + } + + } + return matchingNodes, nil +} + +func multiply(d *dataTreeNavigator, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) { + if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode { + var results = orderedmap.NewOrderedMap() + recursiveDecent(d, results, nodeToMap(rhs)) + + var pathIndexToStartFrom int = 0 + if results.Front() != nil { + pathIndexToStartFrom = len(results.Front().Value.(*CandidateNode).Path) + } + + for el := results.Front(); el != nil; el = el.Next() { + err := applyAssignment(d, pathIndexToStartFrom, lhs, el.Value.(*CandidateNode)) + if err != nil { + return nil, err + } + } + return lhs, nil + } + return nil, fmt.Errorf("Cannot multiply %v with %v", NodeToString(lhs), NodeToString(rhs)) +} + +func createTraversalTree(path []interface{}) *PathTreeNode { + if len(path) == 0 { + return &PathTreeNode{PathElement: &PathElement{PathElementType: SelfReference}} + } else if len(path) == 1 { + return &PathTreeNode{PathElement: &PathElement{PathElementType: PathKey, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}} + } + return &PathTreeNode{ + PathElement: &PathElement{PathElementType: Operation, OperationType: Pipe}, + Lhs: createTraversalTree(path[0:1]), + Rhs: createTraversalTree(path[1:])} + +} + +func applyAssignment(d *dataTreeNavigator, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode) error { + log.Debugf("merge - applyAssignment lhs %v, rhs: %v", NodeToString(lhs), NodeToString(rhs)) + + lhsPath := rhs.Path[pathIndexToStartFrom:] + + assignmentOp := &PathElement{PathElementType: Operation, OperationType: AssignAttributes} + if rhs.Node.Kind == yaml.ScalarNode { + assignmentOp.OperationType = Assign + } + rhsOp := &PathElement{PathElementType: Value, CandidateNode: rhs} + + assignmentOpNode := &PathTreeNode{PathElement: assignmentOp, Lhs: createTraversalTree(lhsPath), Rhs: &PathTreeNode{PathElement: rhsOp}} + + _, err := d.getMatchingNodes(nodeToMap(lhs), assignmentOpNode) + + return err +} diff --git a/pkg/yqlib/treeops/operator_multiply_test.go b/pkg/yqlib/treeops/operator_multiply_test.go new file mode 100644 index 00000000..14101621 --- /dev/null +++ b/pkg/yqlib/treeops/operator_multiply_test.go @@ -0,0 +1,27 @@ +package treeops + +import ( + "testing" +) + +var multiplyOperatorScenarios = []expressionScenario{ + { + // document: `{a: frog, b: cat}`, + // expression: `.a * .b`, + // expected: []string{ + // "D0, P[], (!!map)::{a: cat, b: cat}\n", + // }, + // }, { + document: `{a: {things: great}, b: {also: me}}`, + expression: `.a * .b`, + expected: []string{ + "D0, P[], (!!map)::{a: {things: great, also: me}, b: {also: me}}\n", + }, + }, +} + +func TestMultiplyOperatorScenarios(t *testing.T) { + for _, tt := range multiplyOperatorScenarios { + testScenario(t, &tt) + } +} diff --git a/pkg/yqlib/treeops/path_postfix.go b/pkg/yqlib/treeops/path_postfix.go index 3ed3e009..3688dc9f 100644 --- a/pkg/yqlib/treeops/path_postfix.go +++ b/pkg/yqlib/treeops/path_postfix.go @@ -33,7 +33,11 @@ func (p *pathPostFixer) ConvertToPostfix(infixTokens []*Token) ([]*PathElement, for _, token := range tokens { log.Debugf("postfix processing token %v", token.Value) switch token.PathElementType { - case PathKey, SelfReference, Value: + case Value: + var candidateNode = BuildCandidateNodeFrom(token) + var pathElement = PathElement{PathElementType: token.PathElementType, Value: token.Value, StringValue: token.StringValue, CandidateNode: candidateNode} + result = append(result, &pathElement) + case PathKey, SelfReference: var pathElement = PathElement{PathElementType: token.PathElementType, Value: token.Value, StringValue: token.StringValue} result = append(result, &pathElement) case OpenBracket, OpenCollect: diff --git a/pkg/yqlib/treeops/value_node_builder.go b/pkg/yqlib/treeops/value_node_builder.go index 2a7203f7..7c3e7d21 100644 --- a/pkg/yqlib/treeops/value_node_builder.go +++ b/pkg/yqlib/treeops/value_node_builder.go @@ -2,11 +2,11 @@ package treeops import "gopkg.in/yaml.v3" -func BuildCandidateNodeFrom(p *PathElement) *CandidateNode { +func BuildCandidateNodeFrom(token *Token) *CandidateNode { var node yaml.Node = yaml.Node{Kind: yaml.ScalarNode} - node.Value = p.StringValue + node.Value = token.StringValue - switch p.Value.(type) { + switch token.Value.(type) { case float32, float64: node.Tag = "!!float" case int, int64, int32: