suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2024-12-19 20:19:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Refactor wip

Browse Source
This commit is contained in:
Mike Farah 2019-12-25 12:11:04 +11:00
parent 9925b26b9d
commit ff5b23251b
9 changed files with 331 additions and 258 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
commands_test.go
View File

@ -148,6 +148,18 @@ func TestReadMergeAnchorsOverrideCmd(t *testing.T) {
test.AssertResult(t, "ice", result.Output) test.AssertResult(t, "ice", result.Output)
} }
func TestReadMergeAnchorsPrefixMatchCmd(t *testing.T) {
cmd := getRootCommand()
result := test.RunCmd(cmd, "r -p kv examples/merge-anchor.yaml foobar.th*")
if result.Error != nil {
t.Error(result.Error)
}
expectedOutput := `foobar.thing: ice
foobar.thirty: well beyond
foobar.thirsty: yep`
test.AssertResult(t, expectedOutput, result.Output)
}
func TestReadMergeAnchorsListOriginalCmd(t *testing.T) { func TestReadMergeAnchorsListOriginalCmd(t *testing.T) {
cmd := getRootCommand() cmd := getRootCommand()
result := test.RunCmd(cmd, "read examples/merge-anchor.yaml foobarList.a") result := test.RunCmd(cmd, "read examples/merge-anchor.yaml foobarList.a")
@ -365,8 +377,11 @@ true`
func TestReadCmd_ArrayYaml_ErrorBadPath(t *testing.T) { func TestReadCmd_ArrayYaml_ErrorBadPath(t *testing.T) {
cmd := getRootCommand() cmd := getRootCommand()
result := test.RunCmd(cmd, "read examples/array.yaml [x].gather_facts") result := test.RunCmd(cmd, "read examples/array.yaml [x].gather_facts")
expectedOutput := `` if result.Error == nil {
test.AssertResult(t, expectedOutput, result.Output) t.Error("Expected command to fail due to missing arg")
}
expectedOutput := `Error reading path in document index 0: strconv.ParseInt: parsing "x": invalid syntax`
test.AssertResult(t, expectedOutput, result.Error.Error())
} }
func TestReadCmd_ArrayYaml_Splat_ErrorBadPath(t *testing.T) { func TestReadCmd_ArrayYaml_Splat_ErrorBadPath(t *testing.T) {

2
examples/merge-anchor.yaml
View File

@ -1,6 +1,7 @@
foo: &foo foo: &foo
a: original a: original
thing: coolasdf thing: coolasdf
thirsty: yep
bar: &bar bar: &bar
b: 2 b: 2
@ -14,4 +15,5 @@ foobarList:
foobar: foobar:
<<: *foo <<: *foo
thing: ice thing: ice
thirty: well beyond
c: 3 c: 3

245
pkg/yqlib/data_navigator.go
View File

@ -1,208 +1,73 @@
package yqlib package yqlib
import ( import (
"bytes" "fmt"
"strconv" "strconv"
logging "gopkg.in/op/go-logging.v1"
yaml "gopkg.in/yaml.v3" yaml "gopkg.in/yaml.v3"
) )
type DataNavigator interface { type DataNavigator interface {
DebugNode(node *yaml.Node) Traverse(value *yaml.Node, path []string) error
Get(rootNode *yaml.Node, path []string) ([]MatchingNode, error)
Update(rootNode *yaml.Node, path []string, changesToApply *yaml.Node) error
Delete(rootNode *yaml.Node, path []string) error
GuessKind(tail []string, guess yaml.Kind) yaml.Kind
} }
type navigator struct { type navigator struct {
log *logging.Logger
navigationSettings NavigationSettings navigationSettings NavigationSettings
} }
type VisitorFn func(matchingNode *yaml.Node, pathStack []interface{}) error func NewDataNavigator(navigationSettings NavigationSettings) DataNavigator {
func NewDataNavigator(l *logging.Logger, navigationSettings NavigationSettings) DataNavigator {
return &navigator{ return &navigator{
log: l,
navigationSettings: navigationSettings, navigationSettings: navigationSettings,
} }
} }
type MatchingNode struct { func (n *navigator) Traverse(value *yaml.Node, path []string) error {
Node *yaml.Node
PathStack []interface{}
}
func (n *navigator) Get(value *yaml.Node, path []string) ([]MatchingNode, error) {
matchingNodes := make([]MatchingNode, 0)
n.Visit(value, path, func(matchedNode *yaml.Node, pathStack []interface{}) error {
matchingNodes = append(matchingNodes, MatchingNode{matchedNode, pathStack})
n.log.Debug("Matched")
for _, pathElement := range pathStack {
n.log.Debug("%v", pathElement)
}
n.DebugNode(matchedNode)
return nil
})
return matchingNodes, nil
}
func (n *navigator) Update(rootNode *yaml.Node, path []string, changesToApply *yaml.Node) error {
errorVisiting := n.Visit(rootNode, path, func(nodeToUpdate *yaml.Node, pathStack []interface{}) error {
n.log.Debug("going to update")
n.DebugNode(nodeToUpdate)
n.log.Debug("with")
n.DebugNode(changesToApply)
nodeToUpdate.Value = changesToApply.Value
nodeToUpdate.Tag = changesToApply.Tag
nodeToUpdate.Kind = changesToApply.Kind
nodeToUpdate.Style = changesToApply.Style
nodeToUpdate.Content = changesToApply.Content
nodeToUpdate.HeadComment = changesToApply.HeadComment
nodeToUpdate.LineComment = changesToApply.LineComment
nodeToUpdate.FootComment = changesToApply.FootComment
return nil
})
return errorVisiting
}
// TODO: refactor delete..
func (n *navigator) Delete(rootNode *yaml.Node, path []string) error {
lastBit, newTail := path[len(path)-1], path[:len(path)-1]
n.log.Debug("splitting path, %v", lastBit)
n.log.Debug("new tail, %v", newTail)
errorVisiting := n.Visit(rootNode, newTail, func(nodeToUpdate *yaml.Node, pathStack []interface{}) error {
n.log.Debug("need to find %v in here", lastBit)
n.DebugNode(nodeToUpdate)
original := nodeToUpdate.Content
if nodeToUpdate.Kind == yaml.SequenceNode {
var index, err = strconv.ParseInt(lastBit, 10, 64) // nolint
if err != nil {
return err
}
if index >= int64(len(nodeToUpdate.Content)) {
n.log.Debug("index %v is greater than content length %v", index, len(nodeToUpdate.Content))
return nil
}
nodeToUpdate.Content = append(original[:index], original[index+1:]...)
} else if nodeToUpdate.Kind == yaml.MappingNode {
// need to delete in reverse - otherwise the matching indexes
// become incorrect.
matchingIndices := make([]int, 0)
_, errorVisiting := n.visitMatchingEntries(nodeToUpdate, lastBit, []string{}, pathStack, func(matchingNode []*yaml.Node, indexInMap int) error {
matchingIndices = append(matchingIndices, indexInMap)
n.log.Debug("matchingIndices %v", indexInMap)
return nil
})
n.log.Debug("delete matching indices now")
n.log.Debug("%v", matchingIndices)
if errorVisiting != nil {
return errorVisiting
}
for i := len(matchingIndices) - 1; i >= 0; i-- {
indexToDelete := matchingIndices[i]
n.log.Debug("deleting index %v, %v", indexToDelete, nodeToUpdate.Content[indexToDelete].Value)
nodeToUpdate.Content = append(nodeToUpdate.Content[:indexToDelete], nodeToUpdate.Content[indexToDelete+2:]...)
}
}
return nil
})
return errorVisiting
}
func (n *navigator) Visit(value *yaml.Node, path []string, visitor VisitorFn) error {
realValue := value realValue := value
emptyArray := make([]interface{}, 0) emptyArray := make([]interface{}, 0)
if realValue.Kind == yaml.DocumentNode { if realValue.Kind == yaml.DocumentNode {
n.log.Debugf("its a document! returning the first child") log.Debugf("its a document! returning the first child")
return n.doVisit(value.Content[0], path, visitor, emptyArray) return n.doTraverse(value.Content[0], "", path, emptyArray)
} }
return n.doVisit(value, path, visitor, emptyArray) return n.doTraverse(value, "", path, emptyArray)
} }
func (n *navigator) doVisit(value *yaml.Node, path []string, visitor VisitorFn, pathStack []interface{}) error { func (n *navigator) doTraverse(value *yaml.Node, head string, path []string, pathStack []interface{}) error {
if len(path) > 0 { if len(path) > 0 {
n.log.Debugf("diving into %v", path[0]) log.Debugf("diving into %v", path[0])
n.DebugNode(value) DebugNode(value)
return n.recurse(value, path[0], path[1:], visitor, pathStack) return n.recurse(value, path[0], path[1:], pathStack)
} }
return visitor(value, pathStack) return n.navigationSettings.Visit(value, head, path, pathStack)
}
func (n *navigator) GuessKind(tail []string, guess yaml.Kind) yaml.Kind {
n.log.Debug("tail %v", tail)
if len(tail) == 0 && guess == 0 {
n.log.Debug("end of path, must be a scalar")
return yaml.ScalarNode
} else if len(tail) == 0 {
return guess
}
var _, errorParsingInt = strconv.ParseInt(tail[0], 10, 64)
if tail[0] == "+" || errorParsingInt == nil {
return yaml.SequenceNode
}
if tail[0] == "*" && (guess == yaml.SequenceNode || guess == yaml.MappingNode) {
return guess
}
if guess == yaml.AliasNode {
n.log.Debug("guess was an alias, okey doke.")
return guess
}
n.log.Debug("forcing a mapping node")
n.log.Debug("yaml.SequenceNode ?", guess == yaml.SequenceNode)
n.log.Debug("yaml.ScalarNode ?", guess == yaml.ScalarNode)
return yaml.MappingNode
} }
func (n *navigator) getOrReplace(original *yaml.Node, expectedKind yaml.Kind) *yaml.Node { func (n *navigator) getOrReplace(original *yaml.Node, expectedKind yaml.Kind) *yaml.Node {
if original.Kind != expectedKind { if original.Kind != expectedKind {
n.log.Debug("wanted %v but it was %v, overriding", expectedKind, original.Kind) log.Debug("wanted %v but it was %v, overriding", expectedKind, original.Kind)
return &yaml.Node{Kind: expectedKind} return &yaml.Node{Kind: expectedKind}
} }
return original return original
} }
func (n *navigator) DebugNode(value *yaml.Node) { func (n *navigator) recurse(value *yaml.Node, head string, tail []string, pathStack []interface{}) error {
if value == nil { log.Debug("recursing, processing %v", head)
n.log.Debug("-- node is nil --")
} else if n.log.IsEnabledFor(logging.DEBUG) {
buf := new(bytes.Buffer)
encoder := yaml.NewEncoder(buf)
encoder.Encode(value)
encoder.Close()
n.log.Debug("Tag: %v", value.Tag)
n.log.Debug("%v", buf.String())
}
}
func (n *navigator) recurse(value *yaml.Node, head string, tail []string, visitor VisitorFn, pathStack []interface{}) error {
n.log.Debug("recursing, processing %v", head)
switch value.Kind { switch value.Kind {
case yaml.MappingNode: case yaml.MappingNode:
n.log.Debug("its a map with %v entries", len(value.Content)/2) log.Debug("its a map with %v entries", len(value.Content)/2)
return n.recurseMap(value, head, tail, visitor, pathStack) return n.recurseMap(value, head, tail, pathStack)
case yaml.SequenceNode: case yaml.SequenceNode:
n.log.Debug("its a sequence of %v things!, %v", len(value.Content)) log.Debug("its a sequence of %v things!, %v", len(value.Content))
if head == "*" { if head == "*" {
return n.splatArray(value, tail, visitor, pathStack) return n.splatArray(value, tail, pathStack)
} else if head == "+" { } else if head == "+" {
return n.appendArray(value, tail, visitor, pathStack) return n.appendArray(value, tail, pathStack)
} }
return n.recurseArray(value, head, tail, visitor, pathStack) return n.recurseArray(value, head, tail, pathStack)
case yaml.AliasNode: case yaml.AliasNode:
n.log.Debug("its an alias!") log.Debug("its an alias!")
n.DebugNode(value.Alias) DebugNode(value.Alias)
if n.navigationSettings.FollowAlias(value, head, tail, pathStack) == true { if n.navigationSettings.FollowAlias(value, head, tail, pathStack) == true {
n.log.Debug("following the alias") log.Debug("following the alias")
return n.recurse(value.Alias, head, tail, visitor, pathStack) return n.recurse(value.Alias, head, tail, pathStack)
} }
return nil return nil
default: default:
@ -210,10 +75,10 @@ func (n *navigator) recurse(value *yaml.Node, head string, tail []string, visito
} }
} }
func (n *navigator) recurseMap(value *yaml.Node, head string, tail []string, visitor VisitorFn, pathStack []interface{}) error { func (n *navigator) recurseMap(value *yaml.Node, head string, tail []string, pathStack []interface{}) error {
visited, errorVisiting := n.visitMatchingEntries(value, head, tail, pathStack, func(contents []*yaml.Node, indexInMap int) error { visited, errorVisiting := n.visitMatchingEntries(value, head, tail, pathStack, func(contents []*yaml.Node, indexInMap int) error {
contents[indexInMap+1] = n.getOrReplace(contents[indexInMap+1], n.GuessKind(tail, contents[indexInMap+1].Kind)) contents[indexInMap+1] = n.getOrReplace(contents[indexInMap+1], guessKind(tail, contents[indexInMap+1].Kind))
return n.doVisit(contents[indexInMap+1], tail, visitor, append(pathStack, contents[indexInMap].Value)) return n.doTraverse(contents[indexInMap+1], head, tail, append(pathStack, contents[indexInMap].Value))
}) })
if errorVisiting != nil { if errorVisiting != nil {
@ -226,10 +91,10 @@ func (n *navigator) recurseMap(value *yaml.Node, head string, tail []string, vis
mapEntryKey := yaml.Node{Value: head, Kind: yaml.ScalarNode} mapEntryKey := yaml.Node{Value: head, Kind: yaml.ScalarNode}
value.Content = append(value.Content, &mapEntryKey) value.Content = append(value.Content, &mapEntryKey)
mapEntryValue := yaml.Node{Kind: n.GuessKind(tail, 0)} mapEntryValue := yaml.Node{Kind: guessKind(tail, 0)}
value.Content = append(value.Content, &mapEntryValue) value.Content = append(value.Content, &mapEntryValue)
n.log.Debug("adding new node %v", value.Content) log.Debug("adding new node %v", value.Content)
return n.doVisit(&mapEntryValue, tail, visitor, append(pathStack, head)) return n.doTraverse(&mapEntryValue, head, tail, append(pathStack, head))
} }
// need to pass the node in, as it may be aliased // need to pass the node in, as it may be aliased
@ -240,10 +105,10 @@ func (n *navigator) visitDirectMatchingEntries(node *yaml.Node, head string, tai
visited := false visited := false
for index := 0; index < len(contents); index = index + 2 { for index := 0; index < len(contents); index = index + 2 {
content := contents[index] content := contents[index]
n.log.Debug("index %v, checking %v, %v", index, content.Value, content.Tag) log.Debug("index %v, checking %v, %v", index, content.Value, content.Tag)
if n.navigationSettings.ShouldVisit(content, head, tail, pathStack) == true { if n.navigationSettings.ShouldVisit(content, head, tail, pathStack) == true {
n.log.Debug("found a match! %v", content.Value) log.Debug("found a match! %v", content.Value)
errorVisiting := visit(contents, index) errorVisiting := visit(contents, index)
if errorVisiting != nil { if errorVisiting != nil {
return visited, errorVisiting return visited, errorVisiting
@ -256,8 +121,8 @@ func (n *navigator) visitDirectMatchingEntries(node *yaml.Node, head string, tai
func (n *navigator) visitMatchingEntries(node *yaml.Node, head string, tail []string, pathStack []interface{}, visit mapVisitorFn) (bool, error) { func (n *navigator) visitMatchingEntries(node *yaml.Node, head string, tail []string, pathStack []interface{}, visit mapVisitorFn) (bool, error) {
var contents = node.Content var contents = node.Content
n.log.Debug("visitMatchingEntries %v", head) log.Debug("visitMatchingEntries %v", head)
n.DebugNode(node) DebugNode(node)
// value.Content is a concatenated array of key, value, // value.Content is a concatenated array of key, value,
// so keys are in the even indexes, values in odd. // so keys are in the even indexes, values in odd.
// merge aliases are defined first, but we only want to traverse them // merge aliases are defined first, but we only want to traverse them
@ -282,14 +147,14 @@ func (n *navigator) visitAliases(contents []*yaml.Node, head string, tail []stri
// a node can either be // a node can either be
// an alias to one other node (e.g. <<: *blah) // an alias to one other node (e.g. <<: *blah)
// or a sequence of aliases (e.g. <<: [*blah, *foo]) // or a sequence of aliases (e.g. <<: [*blah, *foo])
n.log.Debug("checking for aliases") log.Debug("checking for aliases")
for index := len(contents) - 2; index >= 0; index = index - 2 { for index := len(contents) - 2; index >= 0; index = index - 2 {
if contents[index+1].Kind == yaml.AliasNode { if contents[index+1].Kind == yaml.AliasNode {
valueNode := contents[index+1] valueNode := contents[index+1]
n.log.Debug("found an alias") log.Debug("found an alias")
n.DebugNode(contents[index]) DebugNode(contents[index])
n.DebugNode(valueNode) DebugNode(valueNode)
visitedAlias, errorInAlias := n.visitMatchingEntries(valueNode.Alias, head, tail, pathStack, visit) visitedAlias, errorInAlias := n.visitMatchingEntries(valueNode.Alias, head, tail, pathStack, visit)
if visitedAlias == true || errorInAlias != nil { if visitedAlias == true || errorInAlias != nil {
@ -303,7 +168,7 @@ func (n *navigator) visitAliases(contents []*yaml.Node, head string, tail []stri
} }
} }
} }
n.log.Debug("nope no matching aliases found") log.Debug("nope no matching aliases found")
return false, nil return false, nil
} }
@ -312,8 +177,8 @@ func (n *navigator) visitAliasSequence(possibleAliasArray []*yaml.Node, head str
for aliasIndex := len(possibleAliasArray) - 1; aliasIndex >= 0; aliasIndex = aliasIndex - 1 { for aliasIndex := len(possibleAliasArray) - 1; aliasIndex >= 0; aliasIndex = aliasIndex - 1 {
child := possibleAliasArray[aliasIndex] child := possibleAliasArray[aliasIndex]
if child.Kind == yaml.AliasNode { if child.Kind == yaml.AliasNode {
n.log.Debug("found an alias") log.Debug("found an alias")
n.DebugNode(child) DebugNode(child)
visitedAlias, errorInAlias := n.visitMatchingEntries(child.Alias, head, tail, pathStack, visit) visitedAlias, errorInAlias := n.visitMatchingEntries(child.Alias, head, tail, pathStack, visit)
if visitedAlias == true || errorInAlias != nil { if visitedAlias == true || errorInAlias != nil {
return visitedAlias, errorInAlias return visitedAlias, errorInAlias
@ -323,12 +188,13 @@ func (n *navigator) visitAliasSequence(possibleAliasArray []*yaml.Node, head str
return false, nil return false, nil
} }
func (n *navigator) splatArray(value *yaml.Node, tail []string, visitor VisitorFn, pathStack []interface{}) error { func (n *navigator) splatArray(value *yaml.Node, tail []string, pathStack []interface{}) error {
for index, childValue := range value.Content { for index, childValue := range value.Content {
n.log.Debug("processing") log.Debug("processing")
n.DebugNode(childValue) DebugNode(childValue)
childValue = n.getOrReplace(childValue, n.GuessKind(tail, childValue.Kind)) head := fmt.Sprintf("%v", index)
var err = n.doVisit(childValue, tail, visitor, append(pathStack, index)) childValue = n.getOrReplace(childValue, guessKind(tail, childValue.Kind))
var err = n.doTraverse(childValue, head, tail, append(pathStack, index))
if err != nil { if err != nil {
return err return err
} }
@ -336,14 +202,15 @@ func (n *navigator) splatArray(value *yaml.Node, tail []string, visitor VisitorF
return nil return nil
} }
func (n *navigator) appendArray(value *yaml.Node, tail []string, visitor VisitorFn, pathStack []interface{}) error { func (n *navigator) appendArray(value *yaml.Node, tail []string, pathStack []interface{}) error {
var newNode = yaml.Node{Kind: n.GuessKind(tail, 0)} var newNode = yaml.Node{Kind: guessKind(tail, 0)}
value.Content = append(value.Content, &newNode) value.Content = append(value.Content, &newNode)
n.log.Debug("appending a new node, %v", value.Content) log.Debug("appending a new node, %v", value.Content)
return n.doVisit(&newNode, tail, visitor, append(pathStack, len(value.Content)-1)) head := fmt.Sprintf("%v", len(value.Content)-1)
return n.doTraverse(&newNode, head, tail, append(pathStack, len(value.Content)-1))
} }
func (n *navigator) recurseArray(value *yaml.Node, head string, tail []string, visitor VisitorFn, pathStack []interface{}) error { func (n *navigator) recurseArray(value *yaml.Node, head string, tail []string, pathStack []interface{}) error {
var index, err = strconv.ParseInt(head, 10, 64) // nolint var index, err = strconv.ParseInt(head, 10, 64) // nolint
if err != nil { if err != nil {
return err return err
@ -351,6 +218,6 @@ func (n *navigator) recurseArray(value *yaml.Node, head string, tail []string, v
if index >= int64(len(value.Content)) { if index >= int64(len(value.Content)) {
return nil return nil
} }
value.Content[index] = n.getOrReplace(value.Content[index], n.GuessKind(tail, value.Content[index].Kind)) value.Content[index] = n.getOrReplace(value.Content[index], guessKind(tail, value.Content[index].Kind))
return n.doVisit(value.Content[index], tail, visitor, append(pathStack, index)) return n.doTraverse(value.Content[index], head, tail, append(pathStack, index))
} }

72
pkg/yqlib/delete_navigation_settings.go Normal file
View File

@ -0,0 +1,72 @@
package yqlib
import (
"strconv"
"strings"
yaml "gopkg.in/yaml.v3"
)
func DeleteNavigationSettings(lastBit string) NavigationSettings {
return &NavigationSettingsImpl{
visitedNodes: []*VisitedNode{},
followAlias: func(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool {
return false
},
autoCreateMap: func(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool {
return true
},
shouldVisit: func(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool {
var prefixMatch = strings.TrimSuffix(head, "*")
if prefixMatch != head {
log.Debug("prefix match, %v", strings.HasPrefix(node.Value, prefixMatch))
return strings.HasPrefix(node.Value, prefixMatch)
}
log.Debug("equals match, %v", node.Value == head)
return node.Value == head
},
visit: func(node *yaml.Node, head string, tail []string, pathStack []interface{}) error {
log.Debug("need to find %v in here", lastBit)
DebugNode(node)
if node.Kind == yaml.SequenceNode {
newContent, errorDeleting := deleteFromArray(node.Content, lastBit)
if errorDeleting != nil {
return errorDeleting
}
node.Content = newContent
} else if node.Kind == yaml.MappingNode {
// need to delete in reverse - otherwise the matching indexes
// become incorrect.
// matchingIndices := make([]int, 0)
// _, errorVisiting := n.visitMatchingEntries(node, lastBit, []string{}, pathStack, func(matchingNode []*yaml.Node, indexInMap int) error {
// matchingIndices = append(matchingIndices, indexInMap)
// log.Debug("matchingIndices %v", indexInMap)
// return nil
// })
// log.Debug("delete matching indices now")
// log.Debug("%v", matchingIndices)
// if errorVisiting != nil {
// return errorVisiting
// }
// for i := len(matchingIndices) - 1; i >= 0; i-- {
// indexToDelete := matchingIndices[i]
// log.Debug("deleting index %v, %v", indexToDelete, node.Content[indexToDelete].Value)
// node.Content = append(node.Content[:indexToDelete], node.Content[indexToDelete+2:]...)
// }
}
return nil
},
}
}
func deleteFromArray(content []*yaml.Node, lastBit string) ([]*yaml.Node, error) {
var index, err = strconv.ParseInt(lastBit, 10, 64) // nolint
if err != nil {
return content, err
}
if index >= int64(len(content)) {
log.Debug("index %v is greater than content length %v", index, len(content))
return content, nil
}
return append(content[:index], content[index+1:]...), nil
}

75
pkg/yqlib/lib.go
View File

@ -1,21 +1,63 @@
package yqlib package yqlib
import ( import (
"bytes"
"fmt" "fmt"
"strconv"
logging "gopkg.in/op/go-logging.v1" logging "gopkg.in/op/go-logging.v1"
yaml "gopkg.in/yaml.v3" yaml "gopkg.in/yaml.v3"
) )
var log = logging.MustGetLogger("yq")
type UpdateCommand struct { type UpdateCommand struct {
Command string Command string
Path string Path string
Value *yaml.Node Value *yaml.Node
} }
func DebugNode(value *yaml.Node) {
if value == nil {
log.Debug("-- node is nil --")
} else if log.IsEnabledFor(logging.DEBUG) {
buf := new(bytes.Buffer)
encoder := yaml.NewEncoder(buf)
encoder.Encode(value)
encoder.Close()
log.Debug("Tag: %v", value.Tag)
log.Debug("%v", buf.String())
}
}
func guessKind(tail []string, guess yaml.Kind) yaml.Kind {
log.Debug("tail %v", tail)
if len(tail) == 0 && guess == 0 {
log.Debug("end of path, must be a scalar")
return yaml.ScalarNode
} else if len(tail) == 0 {
return guess
}
var _, errorParsingInt = strconv.ParseInt(tail[0], 10, 64)
if tail[0] == "+" || errorParsingInt == nil {
return yaml.SequenceNode
}
if tail[0] == "*" && (guess == yaml.SequenceNode || guess == yaml.MappingNode) {
return guess
}
if guess == yaml.AliasNode {
log.Debug("guess was an alias, okey doke.")
return guess
}
log.Debug("forcing a mapping node")
log.Debug("yaml.SequenceNode ?", guess == yaml.SequenceNode)
log.Debug("yaml.ScalarNode ?", guess == yaml.ScalarNode)
return yaml.MappingNode
}
type YqLib interface { type YqLib interface {
DebugNode(node *yaml.Node) Get(rootNode *yaml.Node, path string) ([]*VisitedNode, error)
Get(rootNode *yaml.Node, path string) ([]MatchingNode, error)
Update(rootNode *yaml.Node, updateCommand UpdateCommand) error Update(rootNode *yaml.Node, updateCommand UpdateCommand) error
New(path string) yaml.Node New(path string) yaml.Node
} }
@ -23,44 +65,41 @@ type YqLib interface {
type lib struct { type lib struct {
navigator DataNavigator navigator DataNavigator
parser PathParser parser PathParser
log *logging.Logger
} }
func NewYqLib(l *logging.Logger) YqLib { func NewYqLib(l *logging.Logger) YqLib {
return &lib{ return &lib{
parser: NewPathParser(), parser: NewPathParser(),
log: l,
} }
} }
func (l *lib) DebugNode(node *yaml.Node) { func (l *lib) Get(rootNode *yaml.Node, path string) ([]*VisitedNode, error) {
navigator := NewDataNavigator(l.log, ReadNavigationSettings(l.log))
navigator.DebugNode(node)
}
func (l *lib) Get(rootNode *yaml.Node, path string) ([]MatchingNode, error) {
var paths = l.parser.ParsePath(path) var paths = l.parser.ParsePath(path)
navigator := NewDataNavigator(l.log, ReadNavigationSettings(l.log)) navigationSettings := ReadNavigationSettings()
return navigator.Get(rootNode, paths) navigator := NewDataNavigator(navigationSettings)
error := navigator.Traverse(rootNode, paths)
return navigationSettings.GetVisitedNodes(), error
} }
func (l *lib) New(path string) yaml.Node { func (l *lib) New(path string) yaml.Node {
var paths = l.parser.ParsePath(path) var paths = l.parser.ParsePath(path)
navigator := NewDataNavigator(l.log, UpdateNavigationSettings(l.log)) newNode := yaml.Node{Kind: guessKind(paths, 0)}
newNode := yaml.Node{Kind: navigator.GuessKind(paths, 0)}
return newNode return newNode
} }
func (l *lib) Update(rootNode *yaml.Node, updateCommand UpdateCommand) error { func (l *lib) Update(rootNode *yaml.Node, updateCommand UpdateCommand) error {
navigator := NewDataNavigator(l.log, UpdateNavigationSettings(l.log)) log.Debugf("%v to %v", updateCommand.Command, updateCommand.Path)
l.log.Debugf("%v to %v", updateCommand.Command, updateCommand.Path)
switch updateCommand.Command { switch updateCommand.Command {
case "update": case "update":
var paths = l.parser.ParsePath(updateCommand.Path) var paths = l.parser.ParsePath(updateCommand.Path)
return navigator.Update(rootNode, paths, updateCommand.Value) navigator := NewDataNavigator(UpdateNavigationSettings(updateCommand.Value))
return navigator.Traverse(rootNode, paths)
case "delete": case "delete":
var paths = l.parser.ParsePath(updateCommand.Path) var paths = l.parser.ParsePath(updateCommand.Path)
return navigator.Delete(rootNode, paths) lastBit, newTail := paths[len(paths)-1], paths[:len(paths)-1]
navigator := NewDataNavigator(DeleteNavigationSettings(lastBit))
return navigator.Traverse(rootNode, newTail)
default: default:
return fmt.Errorf("Unknown command %v", updateCommand.Command) return fmt.Errorf("Unknown command %v", updateCommand.Command)
} }

86
pkg/yqlib/navigation_settings.go
View File

@ -3,76 +3,74 @@ package yqlib
import ( import (
"strings" "strings"
logging "gopkg.in/op/go-logging.v1"
yaml "gopkg.in/yaml.v3" yaml "gopkg.in/yaml.v3"
) )
type VisitedNode struct {
Node *yaml.Node
Head string
Tail []string
PathStack []interface{}
}
type NavigationSettings interface { type NavigationSettings interface {
FollowAlias(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool FollowAlias(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool
AutoCreateMap(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool AutoCreateMap(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool
ShouldVisit(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool ShouldVisit(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool
Visit(node *yaml.Node, head string, tail []string, pathStack []interface{}) error
GetVisitedNodes() []*VisitedNode
} }
type NavigationSettingsImpl struct { type NavigationSettingsImpl struct {
followAlias func(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool followAlias func(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool
autoCreateMap func(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool autoCreateMap func(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool
shouldVisit func(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool shouldVisit func(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool
visit func(node *yaml.Node, head string, tail []string, pathStack []interface{}) error
visitedNodes []*VisitedNode
} }
func (ns NavigationSettingsImpl) FollowAlias(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool { func matches(node *yaml.Node, head string) bool {
var prefixMatch = strings.TrimSuffix(head, "*")
if prefixMatch != head {
log.Debug("prefix match, %v", strings.HasPrefix(node.Value, prefixMatch))
return strings.HasPrefix(node.Value, prefixMatch)
}
log.Debug("equals match, %v", node.Value == head)
return node.Value == head
}
func (ns *NavigationSettingsImpl) GetVisitedNodes() []*VisitedNode {
return ns.visitedNodes
}
func (ns *NavigationSettingsImpl) FollowAlias(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool {
return ns.followAlias(node, head, tail, pathStack) return ns.followAlias(node, head, tail, pathStack)
} }
func (ns NavigationSettingsImpl) AutoCreateMap(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool { func (ns *NavigationSettingsImpl) AutoCreateMap(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool {
return ns.autoCreateMap(node, head, tail, pathStack) return ns.autoCreateMap(node, head, tail, pathStack)
} }
func (ns NavigationSettingsImpl) ShouldVisit(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool { func (ns *NavigationSettingsImpl) Visit(node *yaml.Node, head string, tail []string, pathStack []interface{}) error {
ns.visitedNodes = append(ns.visitedNodes, &VisitedNode{node, head, tail, pathStack})
log.Debug("adding to visited nodes")
return ns.visit(node, head, tail, pathStack)
}
func (ns *NavigationSettingsImpl) ShouldVisit(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool {
if !ns.alreadyVisited(node) {
return ns.shouldVisit(node, head, tail, pathStack) return ns.shouldVisit(node, head, tail, pathStack)
} else {
log.Debug("Skipping over %v as we have seen it already", node.Value)
}
return false
} }
func UpdateNavigationSettings(l *logging.Logger) NavigationSettings { func (ns *NavigationSettingsImpl) alreadyVisited(node *yaml.Node) bool {
return NavigationSettingsImpl{ for _, candidate := range ns.visitedNodes {
followAlias: func(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool { if candidate.Node.Value == node.Value {
return false
},
autoCreateMap: func(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool {
return true return true
},
shouldVisit: func(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool {
var prefixMatch = strings.TrimSuffix(head, "*")
if prefixMatch != head {
l.Debug("prefix match, %v", strings.HasPrefix(node.Value, prefixMatch))
return strings.HasPrefix(node.Value, prefixMatch)
}
l.Debug("equals match, %v", node.Value == head)
return node.Value == head
},
} }
} }
func ReadNavigationSettings(l *logging.Logger) NavigationSettings {
return NavigationSettingsImpl{
followAlias: func(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool {
return true
},
autoCreateMap: func(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool {
return false
},
shouldVisit: func(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool {
l.Debug("shouldVisit h: %v, actual: %v", head, node.Value)
if node.Value == "<<" {
l.Debug("its an alias, skip it")
// dont match alias keys, as we'll follow them instead
return false return false
} }
var prefixMatch = strings.TrimSuffix(head, "*")
if prefixMatch != head {
l.Debug("prefix match, %v", strings.HasPrefix(node.Value, prefixMatch))
return strings.HasPrefix(node.Value, prefixMatch)
}
l.Debug("equals match, %v", node.Value == head)
return node.Value == head
},
}
}

37
pkg/yqlib/read_navigation_strategy.go Normal file
View File

@ -0,0 +1,37 @@
package yqlib
import (
"strings"
yaml "gopkg.in/yaml.v3"
)
func ReadNavigationSettings() NavigationSettings {
return &NavigationSettingsImpl{
visitedNodes: []*VisitedNode{},
followAlias: func(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool {
return true
},
autoCreateMap: func(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool {
return false
},
shouldVisit: func(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool {
log.Debug("shouldVisit h: %v, actual: %v", head, node.Value)
if node.Value == "<<" {
log.Debug("its an alias, skip it")
// dont match alias keys, as we'll follow them instead
return false
}
var prefixMatch = strings.TrimSuffix(head, "*")
if prefixMatch != head {
log.Debug("prefix match, %v", strings.HasPrefix(node.Value, prefixMatch))
return strings.HasPrefix(node.Value, prefixMatch)
}
log.Debug("equals match, %v", node.Value == head)
return node.Value == head
},
visit: func(node *yaml.Node, head string, tail []string, pathStack []interface{}) error {
return nil
},
}
}

43
pkg/yqlib/update_navigation_strategy.go Normal file
View File

@ -0,0 +1,43 @@
package yqlib
import (
"strings"
yaml "gopkg.in/yaml.v3"
)
func UpdateNavigationSettings(changesToApply *yaml.Node) NavigationSettings {
return &NavigationSettingsImpl{
visitedNodes: []*VisitedNode{},
followAlias: func(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool {
return false
},
autoCreateMap: func(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool {
return true
},
shouldVisit: func(node *yaml.Node, head string, tail []string, pathStack []interface{}) bool {
var prefixMatch = strings.TrimSuffix(head, "*")
if prefixMatch != head {
log.Debug("prefix match, %v", strings.HasPrefix(node.Value, prefixMatch))
return strings.HasPrefix(node.Value, prefixMatch)
}
log.Debug("equals match, %v", node.Value == head)
return node.Value == head
},
visit: func(node *yaml.Node, head string, tail []string, pathStack []interface{}) error {
log.Debug("going to update")
DebugNode(node)
log.Debug("with")
DebugNode(changesToApply)
node.Value = changesToApply.Value
node.Tag = changesToApply.Tag
node.Kind = changesToApply.Kind
node.Style = changesToApply.Style
node.Content = changesToApply.Content
node.HeadComment = changesToApply.HeadComment
node.LineComment = changesToApply.LineComment
node.FootComment = changesToApply.FootComment
return nil
},
}
}

10
yq.go
View File

@ -257,7 +257,7 @@ func readProperty(cmd *cobra.Command, args []string) error {
return errorParsingDocIndex return errorParsingDocIndex
} }
var matchingNodes []yqlib.MatchingNode var matchingNodes []*yqlib.VisitedNode
var currentIndex = 0 var currentIndex = 0
var errorReadingStream = readStream(args[0], func(decoder *yaml.Decoder) error { var errorReadingStream = readStream(args[0], func(decoder *yaml.Decoder) error {
@ -292,9 +292,9 @@ func handleEOF(updateAll bool, docIndexInt int, currentIndex int) error {
return nil return nil
} }
func appendDocument(originalMatchingNodes []yqlib.MatchingNode, dataBucket yaml.Node, path string, updateAll bool, docIndexInt int, currentIndex int) ([]yqlib.MatchingNode, error) { func appendDocument(originalMatchingNodes []*yqlib.VisitedNode, dataBucket yaml.Node, path string, updateAll bool, docIndexInt int, currentIndex int) ([]*yqlib.VisitedNode, error) {
log.Debugf("processing document %v - requested index %v", currentIndex, docIndexInt) log.Debugf("processing document %v - requested index %v", currentIndex, docIndexInt)
lib.DebugNode(&dataBucket) yqlib.DebugNode(&dataBucket)
if !updateAll && currentIndex != docIndexInt { if !updateAll && currentIndex != docIndexInt {
return originalMatchingNodes, nil return originalMatchingNodes, nil
} }
@ -337,7 +337,7 @@ func printValue(node *yaml.Node, cmd *cobra.Command) error {
return nil return nil
} }
func printResults(matchingNodes []yqlib.MatchingNode, cmd *cobra.Command) error { func printResults(matchingNodes []*yqlib.VisitedNode, cmd *cobra.Command) error {
if len(matchingNodes) == 0 { if len(matchingNodes) == 0 {
log.Debug("no matching results, nothing to print") log.Debug("no matching results, nothing to print")
return nil return nil
@ -485,7 +485,7 @@ func prefixProperty(cmd *cobra.Command, args []string) error {
func prefixDocument(updateAll bool, docIndexInt int, currentIndex int, dataBucket *yaml.Node, updateCommand yqlib.UpdateCommand) error { func prefixDocument(updateAll bool, docIndexInt int, currentIndex int, dataBucket *yaml.Node, updateCommand yqlib.UpdateCommand) error {
if updateAll || currentIndex == docIndexInt { if updateAll || currentIndex == docIndexInt {
log.Debugf("Prefixing document %v", currentIndex) log.Debugf("Prefixing document %v", currentIndex)
lib.DebugNode(dataBucket) yqlib.DebugNode(dataBucket)
updateCommand.Value = dataBucket.Content[0] updateCommand.Value = dataBucket.Content[0]
dataBucket.Content = make([]*yaml.Node, 1) dataBucket.Content = make([]*yaml.Node, 1)