From 461c3e719c299fe98550886f47e8b9c0b8a34967 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 30 Oct 2020 12:00:48 +1100 Subject: [PATCH] merge anchors! --- examples/merge-anchor.yaml | 20 ++-- go.mod | 4 +- pkg/yqlib/treeops/candidate_node.go | 2 +- pkg/yqlib/treeops/operator_traverse_path.go | 84 ++++++++++---- .../treeops/operator_traverse_path_test.go | 104 ++++++++++++++++++ 5 files changed, 178 insertions(+), 36 deletions(-) diff --git a/examples/merge-anchor.yaml b/examples/merge-anchor.yaml index 6d3f426d..61041bd5 100644 --- a/examples/merge-anchor.yaml +++ b/examples/merge-anchor.yaml @@ -1,19 +1,19 @@ foo: &foo - a: original - thing: coolasdf - thirsty: yep + a: foo_a + thing: foo_thing + c: foo_c bar: &bar - b: 2 - thing: coconut - c: oldbar + b: bar_b + thing: bar_thing + c: bar_c foobarList: + b: foobarList_b <<: [*foo,*bar] - c: newbar + c: foobarList_c foobar: + c: foobar_c <<: *foo - thirty: well beyond - thing: ice - c: 3 + thing: foobar_thing \ No newline at end of file diff --git a/go.mod b/go.mod index 11dab62f..d0f3f311 100644 --- a/go.mod +++ b/go.mod @@ -1,13 +1,13 @@ module github.com/mikefarah/yq/v4 require ( - github.com/elliotchance/orderedmap v1.3.0 // indirect + github.com/elliotchance/orderedmap v1.3.0 github.com/fatih/color v1.9.0 github.com/goccy/go-yaml v1.8.1 github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-colorable v0.1.7 // indirect - github.com/pkg/errors v0.9.1 + github.com/pkg/errors v0.9.1 // indirect github.com/spf13/cobra v1.0.0 github.com/spf13/pflag v1.0.5 // indirect github.com/timtadh/data-structures v0.5.3 // indirect diff --git a/pkg/yqlib/treeops/candidate_node.go b/pkg/yqlib/treeops/candidate_node.go index c8d40eea..74096f58 100644 --- a/pkg/yqlib/treeops/candidate_node.go +++ b/pkg/yqlib/treeops/candidate_node.go @@ -17,7 +17,7 @@ type CandidateNode struct { } func (n *CandidateNode) GetKey() string { - return fmt.Sprintf("%v - %v - %v", n.Document, n.Path, n.Node.Value) + return fmt.Sprintf("%v - %v", n.Document, n.Path) } func (n *CandidateNode) Copy() *CandidateNode { diff --git a/pkg/yqlib/treeops/operator_traverse_path.go b/pkg/yqlib/treeops/operator_traverse_path.go index 1bfa14a0..a2bcc1de 100644 --- a/pkg/yqlib/treeops/operator_traverse_path.go +++ b/pkg/yqlib/treeops/operator_traverse_path.go @@ -5,6 +5,8 @@ import ( "container/list" + "github.com/elliotchance/orderedmap" + "gopkg.in/yaml.v3" ) @@ -65,7 +67,34 @@ func traverse(d *dataTreeNavigator, matchingNode *CandidateNode, operation *Oper switch value.Kind { case yaml.MappingNode: log.Debug("its a map with %v entries", len(value.Content)/2) - return traverseMap(matchingNode, operation) + var newMatches = orderedmap.NewOrderedMap() + err := traverseMap(newMatches, matchingNode, operation) + + if err != nil { + return nil, err + } + + if newMatches.Len() == 0 { + //no matches, create one automagically + valueNode := &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"} + node := matchingNode.Node + node.Content = append(node.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: operation.StringValue}, valueNode) + candidateNode := &CandidateNode{ + Node: valueNode, + Path: append(matchingNode.Path, operation.StringValue), + Document: matchingNode.Document, + } + newMatches.Set(candidateNode.GetKey(), candidateNode) + + } + + arrayMatches := make([]*CandidateNode, newMatches.Len()) + i := 0 + for el := newMatches.Front(); el != nil; el = el.Next() { + arrayMatches[i] = el.Value.(*CandidateNode) + i++ + } + return arrayMatches, nil case yaml.SequenceNode: log.Debug("its a sequence of %v things!", len(value.Content)) @@ -92,15 +121,13 @@ func keyMatches(key *yaml.Node, pathNode *Operation) bool { return pathNode.Value == "[]" || Match(key.Value, pathNode.StringValue) } -func traverseMap(candidate *CandidateNode, pathNode *Operation) ([]*CandidateNode, error) { +func traverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, operation *Operation) error { // value.Content is a concatenated array of key, value, // so keys are in the even indexes, values in odd. // merge aliases are defined first, but we only want to traverse them // if we don't find a match directly on this node first. //TODO ALIASES, auto creation? - var newMatches = make([]*CandidateNode, 0) - node := candidate.Node var contents = node.Content @@ -109,33 +136,44 @@ func traverseMap(candidate *CandidateNode, pathNode *Operation) ([]*CandidateNod value := contents[index+1] log.Debug("checking %v (%v)", key.Value, key.Tag) - if keyMatches(key, pathNode) { + //skip the 'merge' tag, find a direct match first + if key.Tag == "!!merge" { + log.Debug("Merge anchor") + traverseMergeAnchor(newMatches, candidate, value, operation) + } else if keyMatches(key, operation) { log.Debug("MATCHED") - newMatches = append(newMatches, &CandidateNode{ + candidateNode := &CandidateNode{ Node: value, Path: append(candidate.Path, key.Value), Document: candidate.Document, - }) + } + newMatches.Set(candidateNode.GetKey(), candidateNode) } } - if len(newMatches) == 0 { - //no matches, create one automagically - valueNode := &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"} - node.Content = append(node.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: pathNode.StringValue}, valueNode) - newMatches = append(newMatches, &CandidateNode{ - Node: valueNode, - Path: append(candidate.Path, pathNode.StringValue), - Document: candidate.Document, - }) - } - - return newMatches, nil + return nil } -func traverseArray(candidate *CandidateNode, pathNode *Operation) ([]*CandidateNode, error) { - log.Debug("pathNode Value %v", pathNode.Value) - if pathNode.Value == "[]" { +func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *yaml.Node, operation *Operation) { + switch value.Kind { + case yaml.AliasNode: + candidateNode := &CandidateNode{ + Node: value.Alias, + Path: originalCandidate.Path, + Document: originalCandidate.Document, + } + traverseMap(newMatches, candidateNode, operation) + case yaml.SequenceNode: + for _, childValue := range value.Content { + traverseMergeAnchor(newMatches, originalCandidate, childValue, operation) + } + } + return +} + +func traverseArray(candidate *CandidateNode, operation *Operation) ([]*CandidateNode, error) { + log.Debug("operation Value %v", operation.Value) + if operation.Value == "[]" { var contents = candidate.Node.Content var newMatches = make([]*CandidateNode, len(contents)) @@ -151,7 +189,7 @@ func traverseArray(candidate *CandidateNode, pathNode *Operation) ([]*CandidateN } - index := pathNode.Value.(int64) + index := operation.Value.(int64) indexToUse := index contentLength := int64(len(candidate.Node.Content)) for contentLength <= index { diff --git a/pkg/yqlib/treeops/operator_traverse_path_test.go b/pkg/yqlib/treeops/operator_traverse_path_test.go index 8b58573a..c5bbd94d 100644 --- a/pkg/yqlib/treeops/operator_traverse_path_test.go +++ b/pkg/yqlib/treeops/operator_traverse_path_test.go @@ -4,6 +4,28 @@ import ( "testing" ) +var mergeDocSample = ` +foo: &foo + a: foo_a + thing: foo_thing + c: foo_c + +bar: &bar + b: bar_b + thing: bar_thing + c: bar_c + +foobarList: + b: foobarList_b + <<: [*foo,*bar] + c: foobarList_c + +foobar: + c: foobar_c + <<: *foo + thing: foobar_thing +` + var traversePathOperatorScenarios = []expressionScenario{ { document: `{a: {b: apple}}`, @@ -104,6 +126,88 @@ var traversePathOperatorScenarios = []expressionScenario{ "D0, P[b c], (!!str)::frog\n", }, }, + { + document: mergeDocSample, + expression: `.foobar`, + expected: []string{ + "D0, P[foobar], (!!map)::c: foobar_c\n!!merge <<: *foo\nthing: foobar_thing\n", + }, + }, + { + document: mergeDocSample, + expression: `.foobar.a`, + expected: []string{ + "D0, P[foobar a], (!!str)::foo_a\n", + }, + }, + { + document: mergeDocSample, + expression: `.foobar.c`, + expected: []string{ + "D0, P[foobar c], (!!str)::foo_c\n", + }, + }, + { + document: mergeDocSample, + expression: `.foobar.thing`, + expected: []string{ + "D0, P[foobar thing], (!!str)::foobar_thing\n", + }, + }, + { + document: mergeDocSample, + expression: `.foobar.[]`, + expected: []string{ + "D0, P[foobar c], (!!str)::foo_c\n", + "D0, P[foobar a], (!!str)::foo_a\n", + "D0, P[foobar thing], (!!str)::foobar_thing\n", + }, + }, + { + document: mergeDocSample, + expression: `.foobarList`, + expected: []string{ + "D0, P[foobarList], (!!map)::b: foobarList_b\n!!merge <<: [*foo, *bar]\nc: foobarList_c\n", + }, + }, + { + document: mergeDocSample, + expression: `.foobarList.a`, + expected: []string{ + "D0, P[foobarList a], (!!str)::foo_a\n", + }, + }, + { + document: mergeDocSample, + expression: `.foobarList.thing`, + expected: []string{ + "D0, P[foobarList thing], (!!str)::bar_thing\n", + }, + }, + { + document: mergeDocSample, + expression: `.foobarList.c`, + expected: []string{ + "D0, P[foobarList c], (!!str)::foobarList_c\n", + }, + }, + { + document: mergeDocSample, + expression: `.foobarList.b`, + expected: []string{ + "D0, P[foobarList b], (!!str)::bar_b\n", + }, + }, + { + document: mergeDocSample, + expression: `.foobarList.[]`, + expected: []string{ + "D0, P[foobarList b], (!!str)::bar_b\n", + "D0, P[foobarList a], (!!str)::foo_a\n", + "D0, P[foobarList thing], (!!str)::bar_thing\n", + "D0, P[foobarList c], (!!str)::foobarList_c\n", + }, + }, } func TestTraversePathOperatorScenarios(t *testing.T) {