From c4c8e5e7b0f50efb2fceda1eb27d5bc8dc1597a3 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Mon, 8 Feb 2021 13:58:46 +1100 Subject: [PATCH] Preserve comments on map keys --- go.mod | 2 +- go.sum | 7 ++-- pkg/yqlib/candidate_node.go | 53 ++++------------------------- pkg/yqlib/doc/Multiply.md | 1 - pkg/yqlib/operator_assign.go | 1 + pkg/yqlib/operator_delete.go | 2 +- pkg/yqlib/operator_multiply.go | 13 ++++--- pkg/yqlib/operator_multiply_test.go | 30 ++++++++++------ pkg/yqlib/operator_traverse_path.go | 33 +++++++++++++----- pkg/yqlib/operators.go | 18 +++++++--- 10 files changed, 83 insertions(+), 77 deletions(-) diff --git a/go.mod b/go.mod index 0d1dd6c8..19ba9dfb 100644 --- a/go.mod +++ b/go.mod @@ -1,7 +1,7 @@ module github.com/mikefarah/yq/v4 require ( - github.com/elliotchance/orderedmap v1.3.0 + github.com/elliotchance/orderedmap v1.4.0 github.com/fatih/color v1.10.0 github.com/goccy/go-yaml v1.8.8 github.com/jinzhu/copier v0.2.3 diff --git a/go.sum b/go.sum index 10f9eb3a..5ae484d4 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= -github.com/elliotchance/orderedmap v1.3.0 h1:k6m77/d0zCXTjsk12nX40TkEBkSICq8T4s6R6bpCqU0= -github.com/elliotchance/orderedmap v1.3.0/go.mod h1:8hdSl6jmveQw8ScByd3AaNHNk51RhbTazdqtTty+NFw= +github.com/elliotchance/orderedmap v1.4.0 h1:wZtfeEONCbx6in1CZyE6bELEt/vFayMvsxqI5SgsR+A= +github.com/elliotchance/orderedmap v1.4.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= @@ -182,6 +182,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/timtadh/data-structures v0.5.3 h1:F2tEjoG9qWIyUjbvXVgJqEOGJPMIiYn7U5W5mE+i/vQ= github.com/timtadh/data-structures v0.5.3/go.mod h1:9R4XODhJ8JdWFEI8P/HJKqxuJctfBQw6fDibMQny2oU= @@ -324,6 +326,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/yqlib/candidate_node.go b/pkg/yqlib/candidate_node.go index c868b61e..da037a75 100644 --- a/pkg/yqlib/candidate_node.go +++ b/pkg/yqlib/candidate_node.go @@ -2,8 +2,6 @@ package yqlib import ( "fmt" - "strconv" - "strings" "github.com/jinzhu/copier" yaml "gopkg.in/yaml.v3" @@ -18,10 +16,15 @@ type CandidateNode struct { // when performing op against all nodes given, this will treat all the nodes as one // (e.g. top level cross document merge). This property does not propegate to child nodes. EvaluateTogether bool + IsMapKey bool } func (n *CandidateNode) GetKey() string { - return fmt.Sprintf("%v - %v", n.Document, n.Path) + keyPrefix := "" + if n.IsMapKey { + keyPrefix = "key-" + } + return fmt.Sprintf("%v%v - %v", keyPrefix, n.Document, n.Path) } func (n *CandidateNode) CreateChild(path interface{}, node *yaml.Node) *CandidateNode { @@ -66,6 +69,7 @@ func (n *CandidateNode) UpdateFrom(other *CandidateNode) { } func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode) { + log.Debug("UpdateAttributesFrom: n: %v other: %v", n.GetKey(), other.GetKey()) if n.Node.Kind != other.Node.Kind { // clear out the contents when switching to a different type // e.g. map to array @@ -86,46 +90,3 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode) { n.Node.HeadComment = n.Node.HeadComment + other.Node.HeadComment n.Node.LineComment = n.Node.LineComment + other.Node.LineComment } - -func (n *CandidateNode) PathStackToString() string { - return mergePathStackToString(n.Path) -} - -func mergePathStackToString(pathStack []interface{}) string { - var sb strings.Builder - for index, path := range pathStack { - switch path.(type) { - case int, int64: - // if arrayMergeStrategy == AppendArrayMergeStrategy { - // sb.WriteString("[+]") - // } else { - sb.WriteString(fmt.Sprintf("[%v]", path)) - // } - - default: - s := fmt.Sprintf("%v", path) - var _, errParsingInt = strconv.ParseInt(s, 10, 64) // nolint - - hasSpecial := strings.Contains(s, ".") || strings.Contains(s, "[") || strings.Contains(s, "]") || strings.Contains(s, "\"") - hasDoubleQuotes := strings.Contains(s, "\"") - wrappingCharacterStart := "\"" - wrappingCharacterEnd := "\"" - if hasDoubleQuotes { - wrappingCharacterStart = "(" - wrappingCharacterEnd = ")" - } - if hasSpecial || errParsingInt == nil { - sb.WriteString(wrappingCharacterStart) - } - sb.WriteString(s) - if hasSpecial || errParsingInt == nil { - sb.WriteString(wrappingCharacterEnd) - } - } - - if index < len(pathStack)-1 { - sb.WriteString(".") - } - } - return sb.String() -} diff --git a/pkg/yqlib/doc/Multiply.md b/pkg/yqlib/doc/Multiply.md index 6d8b14dc..0bbc0626 100644 --- a/pkg/yqlib/doc/Multiply.md +++ b/pkg/yqlib/doc/Multiply.md @@ -71,7 +71,6 @@ Given a sample.yml file of: a: {things: great} b: also: "me" - ``` then ```bash diff --git a/pkg/yqlib/operator_assign.go b/pkg/yqlib/operator_assign.go index 85744f5c..ff13547a 100644 --- a/pkg/yqlib/operator_assign.go +++ b/pkg/yqlib/operator_assign.go @@ -36,6 +36,7 @@ func assignUpdateOperator(d *dataTreeNavigator, context Context, expressionNode // does not update content or values func assignAttributesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { + log.Debug("getting lhs matching nodes for update") lhs, err := d.GetMatchingNodes(context, expressionNode.Lhs) if err != nil { return Context{}, err diff --git a/pkg/yqlib/operator_delete.go b/pkg/yqlib/operator_delete.go index b9648895..d517de2a 100644 --- a/pkg/yqlib/operator_delete.go +++ b/pkg/yqlib/operator_delete.go @@ -26,7 +26,7 @@ func deleteChildOperator(d *dataTreeNavigator, context Context, expressionNode * deleteImmediateChildOpNode := &ExpressionNode{ Operation: deleteImmediateChildOp, - Rhs: createTraversalTree(candidate.Path[0:len(candidate.Path)-1], traversePreferences{}), + Rhs: createTraversalTree(candidate.Path[0:len(candidate.Path)-1], traversePreferences{}, false), } _, err := d.GetMatchingNodes(contextToUse, deleteImmediateChildOpNode) diff --git a/pkg/yqlib/operator_multiply.go b/pkg/yqlib/operator_multiply.go index ec86c015..f09c689d 100644 --- a/pkg/yqlib/operator_multiply.go +++ b/pkg/yqlib/operator_multiply.go @@ -66,7 +66,7 @@ func mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs // shouldn't recurse arrays if appending prefs := recursiveDescentPreferences{RecurseArray: !shouldAppendArrays, - TraversePreferences: traversePreferences{DontFollowAlias: true}} + TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: true}} err := recursiveDecent(d, results, context.SingleChildContext(rhs), prefs) if err != nil { return nil, err @@ -78,7 +78,12 @@ func mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs } for el := results.Front(); el != nil; el = el.Next() { - err := applyAssignment(d, context, pathIndexToStartFrom, lhs, el.Value.(*CandidateNode), preferences) + candidate := el.Value.(*CandidateNode) + if candidate.Node.Tag == "!!merge" { + continue + } + + err := applyAssignment(d, context, pathIndexToStartFrom, lhs, candidate, preferences) if err != nil { return nil, err } @@ -88,7 +93,7 @@ func mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode, preferences multiplyPreferences) error { shouldAppendArrays := preferences.AppendArrays - log.Debugf("merge - applyAssignment lhs %v, rhs: %v", NodeToString(lhs), NodeToString(rhs)) + log.Debugf("merge - applyAssignment lhs %v, rhs: %v", lhs.GetKey(), rhs.GetKey()) lhsPath := rhs.Path[pathIndexToStartFrom:] @@ -101,7 +106,7 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom } rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs} - assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath, preferences.TraversePrefs), Rhs: &ExpressionNode{Operation: rhsOp}} + assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath, preferences.TraversePrefs, rhs.IsMapKey), Rhs: &ExpressionNode{Operation: rhsOp}} _, err := d.GetMatchingNodes(context.SingleChildContext(lhs), assignmentOpNode) diff --git a/pkg/yqlib/operator_multiply_test.go b/pkg/yqlib/operator_multiply_test.go index d1aefa0f..76f0e1d9 100644 --- a/pkg/yqlib/operator_multiply_test.go +++ b/pkg/yqlib/operator_multiply_test.go @@ -13,6 +13,22 @@ var multiplyOperatorScenarios = []expressionScenario{ "D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\n", }, }, + { + skipDoc: true, + document: "# b\nb:\n # a\n a: cat", + expression: "{} * .", + expected: []string{ + "D0, P[], (!!map)::# b\nb:\n # a\n a: cat\n", + }, + }, + { + skipDoc: true, + document: "# b\nb:\n # a\n a: cat", + expression: ". * {}", + expected: []string{ + "D0, P[], (!!map)::# b\nb:\n # a\n a: cat\n", + }, + }, { skipDoc: true, document: `{a: &a { b: &b { c: &c cat } } }`, @@ -100,7 +116,7 @@ var multiplyOperatorScenarios = []expressionScenario{ { skipDoc: true, document: `{a: {things: great}, b: {also: me}}`, - expression: `. * {"a":.b}`, + expression: `. * {"a": .b}`, expected: []string{ "D0, P[], (!!map)::{a: {things: great, also: me}, b: {also: me}}\n", }, @@ -108,16 +124,10 @@ var multiplyOperatorScenarios = []expressionScenario{ { description: "Merge keeps style of LHS", dontFormatInputForDoc: true, - document: `a: {things: great} -b: - also: "me" -`, - expression: `. * {"a":.b}`, + document: "a: {things: great}\nb:\n also: \"me\"", + expression: `. * {"a":.b}`, expected: []string{ - `D0, P[], (!!map)::a: {things: great, also: "me"} -b: - also: "me" -`, + "D0, P[], (!!map)::a: {things: great, also: \"me\"}\nb:\n also: \"me\"\n", }, }, { diff --git a/pkg/yqlib/operator_traverse_path.go b/pkg/yqlib/operator_traverse_path.go index 99de7e4c..b64bcb94 100644 --- a/pkg/yqlib/operator_traverse_path.go +++ b/pkg/yqlib/operator_traverse_path.go @@ -10,9 +10,10 @@ import ( ) type traversePreferences struct { - DontFollowAlias bool - IncludeMapKeys bool - DontAutoCreate bool // by default, we automatically create entries on the fly. + DontFollowAlias bool + IncludeMapKeys bool + DontAutoCreate bool // by default, we automatically create entries on the fly. + DontIncludeMapValues bool } func splat(d *dataTreeNavigator, context Context, prefs traversePreferences) (Context, error) { @@ -214,10 +215,21 @@ func traverseMap(context Context, matchingNode *CandidateNode, key string, prefs if !prefs.DontAutoCreate && !context.DontAutoCreate && newMatches.Len() == 0 { //no matches, create one automagically valueNode := &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"} + keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key} node := matchingNode.Node - node.Content = append(node.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: key}, valueNode) - candidateNode := matchingNode.CreateChild(key, valueNode) - newMatches.Set(candidateNode.GetKey(), candidateNode) + node.Content = append(node.Content, keyNode, valueNode) + + if prefs.IncludeMapKeys { + log.Debug("including key") + candidateNode := matchingNode.CreateChild(key, keyNode) + candidateNode.IsMapKey = true + newMatches.Set(fmt.Sprintf("keyOf-%v", candidateNode.GetKey()), candidateNode) + } + if !prefs.DontIncludeMapValues { + log.Debug("including value") + candidateNode := matchingNode.CreateChild(key, valueNode) + newMatches.Set(candidateNode.GetKey(), candidateNode) + } } results := list.New() @@ -253,11 +265,16 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode, } else if splat || keyMatches(key, wantedKey) { log.Debug("MATCHED") if prefs.IncludeMapKeys { + log.Debug("including key") candidateNode := candidate.CreateChild(key.Value, key) + candidateNode.IsMapKey = true newMatches.Set(fmt.Sprintf("keyOf-%v", candidateNode.GetKey()), candidateNode) } - candidateNode := candidate.CreateChild(key.Value, value) - newMatches.Set(candidateNode.GetKey(), candidateNode) + if !prefs.DontIncludeMapValues { + log.Debug("including value") + candidateNode := candidate.CreateChild(key.Value, value) + newMatches.Set(candidateNode.GetKey(), candidateNode) + } } } diff --git a/pkg/yqlib/operators.go b/pkg/yqlib/operators.go index 08e8ad45..798ebe70 100644 --- a/pkg/yqlib/operators.go +++ b/pkg/yqlib/operators.go @@ -4,6 +4,7 @@ import ( "container/list" "fmt" + "github.com/jinzhu/copier" "gopkg.in/yaml.v3" ) @@ -88,16 +89,25 @@ func createBooleanCandidate(owner *CandidateNode, value bool) *CandidateNode { return owner.CreateChild(nil, node) } -func createTraversalTree(path []interface{}, traversePrefs traversePreferences) *ExpressionNode { +func createTraversalTree(path []interface{}, traversePrefs traversePreferences, targetKey bool) *ExpressionNode { if len(path) == 0 { return &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}} } else if len(path) == 1 { - return &ExpressionNode{Operation: &Operation{OperationType: traversePathOpType, Preferences: traversePrefs, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}} + lastPrefs := traversePrefs + if targetKey { + err := copier.Copy(&lastPrefs, traversePrefs) + if err != nil { + panic(err) + } + lastPrefs.IncludeMapKeys = true + lastPrefs.DontIncludeMapValues = true + } + return &ExpressionNode{Operation: &Operation{OperationType: traversePathOpType, Preferences: lastPrefs, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}} } return &ExpressionNode{ Operation: &Operation{OperationType: shortPipeOpType}, - Lhs: createTraversalTree(path[0:1], traversePrefs), - Rhs: createTraversalTree(path[1:], traversePrefs), + Lhs: createTraversalTree(path[0:1], traversePrefs, false), + Rhs: createTraversalTree(path[1:], traversePrefs, targetKey), } }