suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2024-11-13 22:38:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Preserve comments on map keys

Browse Source
This commit is contained in:
Mike Farah 2021-02-08 13:58:46 +11:00
parent e02ad4d7e8
commit c4c8e5e7b0
10 changed files with 83 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -1,7 +1,7 @@
module github.com/mikefarah/yq/v4
require (
github.com/elliotchance/orderedmap v1.3.0
github.com/elliotchance/orderedmap v1.4.0
github.com/fatih/color v1.10.0
github.com/goccy/go-yaml v1.8.8
github.com/jinzhu/copier v0.2.3

7
go.sum
View File

@ -36,8 +36,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
github.com/elliotchance/orderedmap v1.3.0 h1:k6m77/d0zCXTjsk12nX40TkEBkSICq8T4s6R6bpCqU0=
github.com/elliotchance/orderedmap v1.3.0/go.mod h1:8hdSl6jmveQw8ScByd3AaNHNk51RhbTazdqtTty+NFw=
github.com/elliotchance/orderedmap v1.4.0 h1:wZtfeEONCbx6in1CZyE6bELEt/vFayMvsxqI5SgsR+A=
github.com/elliotchance/orderedmap v1.4.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg=
github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
@ -182,6 +182,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/timtadh/data-structures v0.5.3 h1:F2tEjoG9qWIyUjbvXVgJqEOGJPMIiYn7U5W5mE+i/vQ=
github.com/timtadh/data-structures v0.5.3/go.mod h1:9R4XODhJ8JdWFEI8P/HJKqxuJctfBQw6fDibMQny2oU=
@ -324,6 +326,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

53
pkg/yqlib/candidate_node.go
View File

@ -2,8 +2,6 @@ package yqlib
import (
"fmt"
"strconv"
"strings"
"github.com/jinzhu/copier"
yaml "gopkg.in/yaml.v3"
@ -18,10 +16,15 @@ type CandidateNode struct {
// when performing op against all nodes given, this will treat all the nodes as one
// (e.g. top level cross document merge). This property does not propegate to child nodes.
EvaluateTogether bool
IsMapKey bool
}
func (n *CandidateNode) GetKey() string {
return fmt.Sprintf("%v - %v", n.Document, n.Path)
keyPrefix := ""
if n.IsMapKey {
keyPrefix = "key-"
}
return fmt.Sprintf("%v%v - %v", keyPrefix, n.Document, n.Path)
}
func (n *CandidateNode) CreateChild(path interface{}, node *yaml.Node) *CandidateNode {
@ -66,6 +69,7 @@ func (n *CandidateNode) UpdateFrom(other *CandidateNode) {
}
func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode) {
log.Debug("UpdateAttributesFrom: n: %v other: %v", n.GetKey(), other.GetKey())
if n.Node.Kind != other.Node.Kind {
// clear out the contents when switching to a different type
// e.g. map to array
@ -86,46 +90,3 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode) {
n.Node.HeadComment = n.Node.HeadComment + other.Node.HeadComment
n.Node.LineComment = n.Node.LineComment + other.Node.LineComment
}
func (n *CandidateNode) PathStackToString() string {
return mergePathStackToString(n.Path)
}
func mergePathStackToString(pathStack []interface{}) string {
var sb strings.Builder
for index, path := range pathStack {
switch path.(type) {
case int, int64:
// if arrayMergeStrategy == AppendArrayMergeStrategy {
// sb.WriteString("[+]")
// } else {
sb.WriteString(fmt.Sprintf("[%v]", path))
// }
default:
s := fmt.Sprintf("%v", path)
var _, errParsingInt = strconv.ParseInt(s, 10, 64) // nolint
hasSpecial := strings.Contains(s, ".") || strings.Contains(s, "[") || strings.Contains(s, "]") || strings.Contains(s, "\"")
hasDoubleQuotes := strings.Contains(s, "\"")
wrappingCharacterStart := "\""
wrappingCharacterEnd := "\""
if hasDoubleQuotes {
wrappingCharacterStart = "("
wrappingCharacterEnd = ")"
}
if hasSpecial || errParsingInt == nil {
sb.WriteString(wrappingCharacterStart)
}
sb.WriteString(s)
if hasSpecial || errParsingInt == nil {
sb.WriteString(wrappingCharacterEnd)
}
}
if index < len(pathStack)-1 {
sb.WriteString(".")
}
}
return sb.String()
}

1
pkg/yqlib/doc/Multiply.md
View File

@ -71,7 +71,6 @@ Given a sample.yml file of:
a: {things: great}
b:
also: "me"
```
then
```bash

1
pkg/yqlib/operator_assign.go
View File

@ -36,6 +36,7 @@ func assignUpdateOperator(d *dataTreeNavigator, context Context, expressionNode
// does not update content or values
func assignAttributesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
log.Debug("getting lhs matching nodes for update")
lhs, err := d.GetMatchingNodes(context, expressionNode.Lhs)
if err != nil {
return Context{}, err

2
pkg/yqlib/operator_delete.go
View File

@ -26,7 +26,7 @@ func deleteChildOperator(d *dataTreeNavigator, context Context, expressionNode *
deleteImmediateChildOpNode := &ExpressionNode{
Operation: deleteImmediateChildOp,
Rhs: createTraversalTree(candidate.Path[0:len(candidate.Path)-1], traversePreferences{}),
Rhs: createTraversalTree(candidate.Path[0:len(candidate.Path)-1], traversePreferences{}, false),
}
_, err := d.GetMatchingNodes(contextToUse, deleteImmediateChildOpNode)

13
pkg/yqlib/operator_multiply.go
View File

@ -66,7 +66,7 @@ func mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs
// shouldn't recurse arrays if appending
prefs := recursiveDescentPreferences{RecurseArray: !shouldAppendArrays,
TraversePreferences: traversePreferences{DontFollowAlias: true}}
TraversePreferences: traversePreferences{DontFollowAlias: true, IncludeMapKeys: true}}
err := recursiveDecent(d, results, context.SingleChildContext(rhs), prefs)
if err != nil {
return nil, err
@ -78,7 +78,12 @@ func mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs
}
for el := results.Front(); el != nil; el = el.Next() {
err := applyAssignment(d, context, pathIndexToStartFrom, lhs, el.Value.(*CandidateNode), preferences)
candidate := el.Value.(*CandidateNode)
if candidate.Node.Tag == "!!merge" {
continue
}
err := applyAssignment(d, context, pathIndexToStartFrom, lhs, candidate, preferences)
if err != nil {
return nil, err
}
@ -88,7 +93,7 @@ func mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs
func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom int, lhs *CandidateNode, rhs *CandidateNode, preferences multiplyPreferences) error {
shouldAppendArrays := preferences.AppendArrays
log.Debugf("merge - applyAssignment lhs %v, rhs: %v", NodeToString(lhs), NodeToString(rhs))
log.Debugf("merge - applyAssignment lhs %v, rhs: %v", lhs.GetKey(), rhs.GetKey())
lhsPath := rhs.Path[pathIndexToStartFrom:]
@ -101,7 +106,7 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom
}
rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs}
assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath, preferences.TraversePrefs), Rhs: &ExpressionNode{Operation: rhsOp}}
assignmentOpNode := &ExpressionNode{Operation: assignmentOp, Lhs: createTraversalTree(lhsPath, preferences.TraversePrefs, rhs.IsMapKey), Rhs: &ExpressionNode{Operation: rhsOp}}
_, err := d.GetMatchingNodes(context.SingleChildContext(lhs), assignmentOpNode)

28
pkg/yqlib/operator_multiply_test.go
View File

@ -13,6 +13,22 @@ var multiplyOperatorScenarios = []expressionScenario{
"D0, P[], (!!map)::{a: {also: me}, b: {also: me}}\n",
},
},
{
skipDoc: true,
document: "# b\nb:\n # a\n a: cat",
expression: "{} * .",
expected: []string{
"D0, P[], (!!map)::# b\nb:\n # a\n a: cat\n",
},
},
{
skipDoc: true,
document: "# b\nb:\n # a\n a: cat",
expression: ". * {}",
expected: []string{
"D0, P[], (!!map)::# b\nb:\n # a\n a: cat\n",
},
},
{
skipDoc: true,
document: `{a: &a { b: &b { c: &c cat } } }`,
@ -100,7 +116,7 @@ var multiplyOperatorScenarios = []expressionScenario{
{
skipDoc: true,
document: `{a: {things: great}, b: {also: me}}`,
expression: `. * {"a":.b}`,
expression: `. * {"a": .b}`,
expected: []string{
"D0, P[], (!!map)::{a: {things: great, also: me}, b: {also: me}}\n",
},
@ -108,16 +124,10 @@ var multiplyOperatorScenarios = []expressionScenario{
{
description: "Merge keeps style of LHS",
dontFormatInputForDoc: true,
document: `a: {things: great}
b:
also: "me"
`,
document: "a: {things: great}\nb:\n also: \"me\"",
expression: `. * {"a":.b}`,
expected: []string{
`D0, P[], (!!map)::a: {things: great, also: "me"}
b:
also: "me"
`,
"D0, P[], (!!map)::a: {things: great, also: \"me\"}\nb:\n also: \"me\"\n",
},
},
{

19
pkg/yqlib/operator_traverse_path.go
View File

@ -13,6 +13,7 @@ type traversePreferences struct {
DontFollowAlias bool
IncludeMapKeys bool
DontAutoCreate bool // by default, we automatically create entries on the fly.
DontIncludeMapValues bool
}
func splat(d *dataTreeNavigator, context Context, prefs traversePreferences) (Context, error) {
@ -214,11 +215,22 @@ func traverseMap(context Context, matchingNode *CandidateNode, key string, prefs
if !prefs.DontAutoCreate && !context.DontAutoCreate && newMatches.Len() == 0 {
//no matches, create one automagically
valueNode := &yaml.Node{Tag: "!!null", Kind: yaml.ScalarNode, Value: "null"}
keyNode := &yaml.Node{Kind: yaml.ScalarNode, Value: key}
node := matchingNode.Node
node.Content = append(node.Content, &yaml.Node{Kind: yaml.ScalarNode, Value: key}, valueNode)
node.Content = append(node.Content, keyNode, valueNode)
if prefs.IncludeMapKeys {
log.Debug("including key")
candidateNode := matchingNode.CreateChild(key, keyNode)
candidateNode.IsMapKey = true
newMatches.Set(fmt.Sprintf("keyOf-%v", candidateNode.GetKey()), candidateNode)
}
if !prefs.DontIncludeMapValues {
log.Debug("including value")
candidateNode := matchingNode.CreateChild(key, valueNode)
newMatches.Set(candidateNode.GetKey(), candidateNode)
}
}
results := list.New()
i := 0
@ -253,13 +265,18 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode,
} else if splat || keyMatches(key, wantedKey) {
log.Debug("MATCHED")
if prefs.IncludeMapKeys {
log.Debug("including key")
candidateNode := candidate.CreateChild(key.Value, key)
candidateNode.IsMapKey = true
newMatches.Set(fmt.Sprintf("keyOf-%v", candidateNode.GetKey()), candidateNode)
}
if !prefs.DontIncludeMapValues {
log.Debug("including value")
candidateNode := candidate.CreateChild(key.Value, value)
newMatches.Set(candidateNode.GetKey(), candidateNode)
}
}
}
return nil
}

18
pkg/yqlib/operators.go
View File

@ -4,6 +4,7 @@ import (
"container/list"
"fmt"
"github.com/jinzhu/copier"
"gopkg.in/yaml.v3"
)
@ -88,16 +89,25 @@ func createBooleanCandidate(owner *CandidateNode, value bool) *CandidateNode {
return owner.CreateChild(nil, node)
}
func createTraversalTree(path []interface{}, traversePrefs traversePreferences) *ExpressionNode {
func createTraversalTree(path []interface{}, traversePrefs traversePreferences, targetKey bool) *ExpressionNode {
if len(path) == 0 {
return &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
} else if len(path) == 1 {
return &ExpressionNode{Operation: &Operation{OperationType: traversePathOpType, Preferences: traversePrefs, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}}
lastPrefs := traversePrefs
if targetKey {
err := copier.Copy(&lastPrefs, traversePrefs)
if err != nil {
panic(err)
}
lastPrefs.IncludeMapKeys = true
lastPrefs.DontIncludeMapValues = true
}
return &ExpressionNode{Operation: &Operation{OperationType: traversePathOpType, Preferences: lastPrefs, Value: path[0], StringValue: fmt.Sprintf("%v", path[0])}}
}
return &ExpressionNode{
Operation: &Operation{OperationType: shortPipeOpType},
Lhs: createTraversalTree(path[0:1], traversePrefs),
Rhs: createTraversalTree(path[1:], traversePrefs),
Lhs: createTraversalTree(path[0:1], traversePrefs, false),
Rhs: createTraversalTree(path[1:], traversePrefs, targetKey),
}
}