suyiiyii/yq
1
0
Fork 0
mirror of https://github.com/mikefarah/yq.git synced 2024-12-19 20:19:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

better v3

Browse Source
This commit is contained in:
Mike Farah 2019-12-06 15:57:46 +11:00
parent 5fc13bdccd
commit aad15ccc6e
6 changed files with 753 additions and 621 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
examples/instruction_sample.yaml
View File

@ -1,2 +1,7 @@
b.c: cat - command: update
path: b.c
value:
#great
things: frog # wow!
b.e[+].name: Mike Farah b.e[+].name: Mike Farah

3
go.mod
View File

@ -1,4 +1,4 @@
module github.com/mikefarah/yq/v2 module github.com/mikefarah/yq/v3
require ( require (
github.com/mikefarah/yaml/v2 v2.4.0 github.com/mikefarah/yaml/v2 v2.4.0
@ -7,6 +7,7 @@ require (
golang.org/x/tools v0.0.0-20191030203535-5e247c9ad0a0 // indirect golang.org/x/tools v0.0.0-20191030203535-5e247c9ad0a0 // indirect
gopkg.in/imdario/mergo.v0 v0.3.7 gopkg.in/imdario/mergo.v0 v0.3.7
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2
) )
go 1.13 go 1.13

5
go.sum
View File

@ -10,8 +10,11 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T
github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mikefarah/yaml v2.1.0+incompatible h1:nu2cqmzk4WlWJNgnevY88faMcdrDzYGcsUjYFxEpB7Y=
github.com/mikefarah/yaml/v2 v2.4.0 h1:eYqfooY0BnvKTJxr7+ABJs13n3dg9n347GScDaU2Lww= github.com/mikefarah/yaml/v2 v2.4.0 h1:eYqfooY0BnvKTJxr7+ABJs13n3dg9n347GScDaU2Lww=
github.com/mikefarah/yaml/v2 v2.4.0/go.mod h1:ahVqZF4n1W4NqwvVnZzC4es67xsW9uR/RRf2RRxieJU= github.com/mikefarah/yaml/v2 v2.4.0/go.mod h1:ahVqZF4n1W4NqwvVnZzC4es67xsW9uR/RRf2RRxieJU=
github.com/mikefarah/yq v2.4.0+incompatible h1:oBxbWy8R9hI3BIUUxEf0CzikWa2AgnGrGhvGQt5jgjk=
github.com/mikefarah/yq/v2 v2.4.1 h1:tajDonaFK6WqitSZExB6fKlWQy/yCkptqxh2AXEe3N4=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
@ -48,3 +51,5 @@ gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW
gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog= gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2 h1:XZx7nhd5GMaZpmDaEHFVafUZC7ya0fuo7cSJ3UCKYmM=
gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

751
pkg/yqlib/data_navigator.go
View File

@ -1,19 +1,20 @@
package yqlib package yqlib
import ( import (
"fmt"
"reflect"
"strconv" "strconv"
"strings"
yaml "github.com/mikefarah/yaml/v2"
logging "gopkg.in/op/go-logging.v1" logging "gopkg.in/op/go-logging.v1"
yaml "gopkg.in/yaml.v3"
) )
type WriteCommand struct {
// Command string TODO
Value yaml.Node
}
type DataNavigator interface { type DataNavigator interface {
ReadChildValue(child interface{}, remainingPaths []string) (interface{}, error) Get(rootNode *yaml.Node, remainingPath []string) (*yaml.Node, error)
UpdatedChildValue(child interface{}, remainingPaths []string, value interface{}) interface{} Update(rootNode *yaml.Node, remainingPath []string, writeCommand WriteCommand) error
DeleteChildValue(child interface{}, remainingPaths []string) (interface{}, error)
} }
type navigator struct { type navigator struct {
@ -26,351 +27,451 @@ func NewDataNavigator(l *logging.Logger) DataNavigator {
} }
} }
func (n *navigator) ReadChildValue(child interface{}, remainingPaths []string) (interface{}, error) { func (n *navigator) Get(value *yaml.Node, path []string) (*yaml.Node, error) {
if len(remainingPaths) == 0 { realValue := value
return child, nil if realValue.Kind == yaml.DocumentNode {
realValue = value.Content[0]
} }
return n.recurse(child, remainingPaths[0], remainingPaths[1:]) if len(path) > 0 {
n.log.Debug("diving into %v", path[0])
return n.recurse(realValue, path[0], path[1:])
}
return realValue, nil
} }
func (n *navigator) UpdatedChildValue(child interface{}, remainingPaths []string, value interface{}) interface{} { func (n *navigator) guessKind(tail []string) yaml.Kind {
if len(remainingPaths) == 0 { n.log.Debug("tail %v", tail)
return value if len(tail) == 0 {
n.log.Debug("scalar")
return yaml.ScalarNode
} }
n.log.Debugf("UpdatedChildValue for child %v with path %v to set value %v", child, remainingPaths, value) var _, errorParsingInt = strconv.ParseInt(tail[0], 10, 64)
n.log.Debugf("type of child is %v", reflect.TypeOf(child)) if tail[0] == "*" || tail[0] == "+" || errorParsingInt == nil {
return yaml.SequenceNode
switch child := child.(type) {
case nil:
if remainingPaths[0] == "+" || remainingPaths[0] == "*" {
return n.writeArray(child, remainingPaths, value)
}
case []interface{}:
_, nextIndexErr := strconv.ParseInt(remainingPaths[0], 10, 64)
arrayCommand := nextIndexErr == nil || remainingPaths[0] == "+" || remainingPaths[0] == "*"
if arrayCommand {
return n.writeArray(child, remainingPaths, value)
}
} }
return n.writeMap(child, remainingPaths, value) return yaml.MappingNode
} }
func (n *navigator) DeleteChildValue(child interface{}, remainingPaths []string) (interface{}, error) { func (n *navigator) getOrReplace(original *yaml.Node, expectedKind yaml.Kind) *yaml.Node {
n.log.Debugf("DeleteChildValue for %v for %v\n", remainingPaths, child) // expected is a scalar when we reach the end of the path
if len(remainingPaths) == 0 { // no need to clobber the original because:
return child, nil // when reading, it should deal with the original kind
// when writing, it will clobber the kind anyway
if original.Kind != expectedKind && (expectedKind != yaml.ScalarNode) {
return &yaml.Node{Kind: expectedKind}
} }
var head = remainingPaths[0] return original
var tail = remainingPaths[1:]
switch child := child.(type) {
case yaml.MapSlice:
return n.deleteMap(child, remainingPaths)
case []interface{}:
if head == "*" {
return n.deleteArraySplat(child, tail)
}
index, err := strconv.ParseInt(head, 10, 64)
if err != nil {
return nil, fmt.Errorf("error accessing array: %v", err)
}
return n.deleteArray(child, remainingPaths, index)
}
return child, nil
} }
func (n *navigator) recurse(value interface{}, head string, tail []string) (interface{}, error) { func (n *navigator) recurse(value *yaml.Node, head string, tail []string) (*yaml.Node, error) {
switch value := value.(type) { switch value.Kind {
case []interface{}: case yaml.MappingNode:
if head == "*" { n.log.Debug("its a map with %v entries", len(value.Content)/2)
return n.readArraySplat(value, tail) for index, content := range value.Content {
} // value.Content is a concatenated array of key, value,
index, err := strconv.ParseInt(head, 10, 64) // so keys are in the even indexes, values in odd.
if err != nil { if index%2 == 1 || content.Value != head {
return nil, fmt.Errorf("error accessing array: %v", err) continue
}
return n.readArray(value, index, tail)
case yaml.MapSlice:
return n.readMap(value, head, tail)
default:
return nil, nil
}
}
func (n *navigator) matchesKey(key string, actual interface{}) bool {
var actualString = fmt.Sprintf("%v", actual)
var prefixMatch = strings.TrimSuffix(key, "*")
if prefixMatch != key {
return strings.HasPrefix(actualString, prefixMatch)
}
return actualString == key
}
func (n *navigator) entriesInSlice(context yaml.MapSlice, key string) []*yaml.MapItem {
var matches = make([]*yaml.MapItem, 0)
for idx := range context {
var entry = &context[idx]
if n.matchesKey(key, entry.Key) {
matches = append(matches, entry)
}
}
return matches
}
func (n *navigator) getMapSlice(context interface{}) yaml.MapSlice {
var mapSlice yaml.MapSlice
switch context := context.(type) {
case yaml.MapSlice:
mapSlice = context
default:
mapSlice = make(yaml.MapSlice, 0)
}
return mapSlice
}
func (n *navigator) getArray(context interface{}) (array []interface{}, ok bool) {
switch context := context.(type) {
case []interface{}:
array = context
ok = true
default:
array = make([]interface{}, 0)
ok = false
}
return
}
func (n *navigator) writeMap(context interface{}, paths []string, value interface{}) interface{} {
n.log.Debugf("writeMap with path %v for %v to set value %v\n", paths, context, value)
mapSlice := n.getMapSlice(context)
if len(paths) == 0 {
return context
}
children := n.entriesInSlice(mapSlice, paths[0])
if len(children) == 0 && paths[0] == "*" {
n.log.Debugf("\tNo matches, return map as is")
return context
}
if len(children) == 0 {
newChild := yaml.MapItem{Key: paths[0]}
mapSlice = append(mapSlice, newChild)
children = n.entriesInSlice(mapSlice, paths[0])
n.log.Debugf("\tAppended child at %v for mapSlice %v\n", paths[0], mapSlice)
}
remainingPaths := paths[1:]
for _, child := range children {
child.Value = n.UpdatedChildValue(child.Value, remainingPaths, value)
}
n.log.Debugf("\tReturning mapSlice %v\n", mapSlice)
return mapSlice
}
func (n *navigator) writeArray(context interface{}, paths []string, value interface{}) []interface{} {
n.log.Debugf("writeArray with path %v for %v to set value %v\n", paths, context, value)
array, _ := n.getArray(context)
if len(paths) == 0 {
return array
}
n.log.Debugf("\tarray %v\n", array)
rawIndex := paths[0]
remainingPaths := paths[1:]
var index int64
// the append array indicator
if rawIndex == "+" {
index = int64(len(array))
} else if rawIndex == "*" {
for index, oldChild := range array {
array[index] = n.UpdatedChildValue(oldChild, remainingPaths, value)
}
return array
} else {
index, _ = strconv.ParseInt(rawIndex, 10, 64) // nolint
// writeArray is only called by UpdatedChildValue which handles parsing the
// index, as such this renders this dead code.
}
for index >= int64(len(array)) {
array = append(array, nil)
}
currentChild := array[index]
n.log.Debugf("\tcurrentChild %v\n", currentChild)
array[index] = n.UpdatedChildValue(currentChild, remainingPaths, value)
n.log.Debugf("\tReturning array %v\n", array)
return array
}
func (n *navigator) readMap(context yaml.MapSlice, head string, tail []string) (interface{}, error) {
n.log.Debugf("readingMap %v with key %v\n", context, head)
if head == "*" {
return n.readMapSplat(context, tail)
}
entries := n.entriesInSlice(context, head)
if len(entries) == 1 {
return n.calculateValue(entries[0].Value, tail)
} else if len(entries) == 0 {
return nil, nil
}
var errInIdx error
values := make([]interface{}, len(entries))
for idx, entry := range entries {
values[idx], errInIdx = n.calculateValue(entry.Value, tail)
if errInIdx != nil {
n.log.Errorf("Error updating index %v in %v", idx, context)
return nil, errInIdx
}
}
return values, nil
}
func (n *navigator) readMapSplat(context yaml.MapSlice, tail []string) (interface{}, error) {
var newArray = make([]interface{}, len(context))
var i = 0
for _, entry := range context {
if len(tail) > 0 {
val, err := n.recurse(entry.Value, tail[0], tail[1:])
if err != nil {
return nil, err
} }
newArray[i] = val value.Content[index+1] = n.getOrReplace(value.Content[index+1], n.guessKind(tail))
} else { return n.Get(value.Content[index+1], tail)
newArray[i] = entry.Value
} }
i++ value.Content = append(value.Content, &yaml.Node{Value: head, Kind: yaml.ScalarNode})
} mapEntryValue := yaml.Node{Kind: n.guessKind(tail)}
return newArray, nil value.Content = append(value.Content, &mapEntryValue)
} n.log.Debug("adding new node %v", value.Content)
return n.Get(&mapEntryValue, tail)
case yaml.SequenceNode:
n.log.Debug("its a sequence of %v things!", len(value.Content))
if head == "*" {
var newNode = yaml.Node{Kind: yaml.SequenceNode, Style: value.Style}
newNode.Content = make([]*yaml.Node, len(value.Content))
func (n *navigator) readArray(array []interface{}, head int64, tail []string) (interface{}, error) { for index, value := range value.Content {
if head >= int64(len(array)) { value.Content[index] = n.getOrReplace(value.Content[index], n.guessKind(tail))
return nil, nil var nestedValue, err = n.Get(value.Content[index], tail)
} if err != nil {
return nil, err
}
newNode.Content[index] = nestedValue
}
return &newNode, nil
} else if head == "+" {
value := array[head] var newNode = yaml.Node{Kind: n.guessKind(tail)}
return n.calculateValue(value, tail) value.Content = append(value.Content, &newNode)
} n.log.Debug("appending a new node, %v", value.Content)
return n.Get(&newNode, tail)
func (n *navigator) readArraySplat(array []interface{}, tail []string) (interface{}, error) { }
var newArray = make([]interface{}, len(array)) var index, err = strconv.ParseInt(head, 10, 64) // nolint
for index, value := range array {
val, err := n.calculateValue(value, tail)
if err != nil { if err != nil {
return nil, err return nil, err
} }
newArray[index] = val if index >= int64(len(value.Content)) {
} return nil, nil
return newArray, nil
}
func (n *navigator) calculateValue(value interface{}, tail []string) (interface{}, error) {
if len(tail) > 0 {
return n.recurse(value, tail[0], tail[1:])
}
return value, nil
}
func (n *navigator) deleteMap(context interface{}, paths []string) (yaml.MapSlice, error) {
n.log.Debugf("deleteMap for %v for %v\n", paths, context)
mapSlice := n.getMapSlice(context)
if len(paths) == 0 {
return mapSlice, nil
}
var index int
var child yaml.MapItem
for index, child = range mapSlice {
if n.matchesKey(paths[0], child.Key) {
n.log.Debugf("\tMatched [%v] with [%v] at index %v", paths[0], child.Key, index)
var badDelete error
mapSlice, badDelete = n.deleteEntryInMap(mapSlice, child, index, paths)
if badDelete != nil {
return nil, badDelete
}
} }
value.Content[index] = n.getOrReplace(value.Content[index], n.guessKind(tail))
return n.Get(value.Content[index], tail)
default:
return nil, nil
} }
return mapSlice, nil
} }
func (n *navigator) deleteEntryInMap(original yaml.MapSlice, child yaml.MapItem, index int, paths []string) (yaml.MapSlice, error) { func (n *navigator) Update(dataBucket *yaml.Node, remainingPath []string, writeCommand WriteCommand) error {
remainingPaths := paths[1:] nodeToUpdate, errorRecursing := n.Get(dataBucket, remainingPath)
if errorRecursing != nil {
var newSlice yaml.MapSlice return errorRecursing
if len(remainingPaths) > 0 {
newChild := yaml.MapItem{Key: child.Key}
var errorDeleting error
newChild.Value, errorDeleting = n.DeleteChildValue(child.Value, remainingPaths)
if errorDeleting != nil {
return nil, errorDeleting
}
newSlice = make(yaml.MapSlice, len(original))
for i := range original {
item := original[i]
if i == index {
item = newChild
}
newSlice[i] = item
}
} else {
// Delete item from slice at index
newSlice = append(original[:index], original[index+1:]...)
n.log.Debugf("\tDeleted item index %d from original", index)
} }
// later, support ability to execute other commands
n.log.Debugf("\tReturning original %v\n", original) changesToApply := writeCommand.Value
return newSlice, nil
nodeToUpdate.Value = changesToApply.Value
nodeToUpdate.Tag = changesToApply.Tag
nodeToUpdate.Kind = changesToApply.Kind
nodeToUpdate.Style = changesToApply.Style
nodeToUpdate.Content = changesToApply.Content
nodeToUpdate.HeadComment = changesToApply.HeadComment
nodeToUpdate.LineComment = changesToApply.LineComment
nodeToUpdate.FootComment = changesToApply.FootComment
return nil
} }
func (n *navigator) deleteArraySplat(array []interface{}, tail []string) (interface{}, error) { // func matchesKey(key string, actual interface{}) bool {
n.log.Debugf("deleteArraySplat for %v for %v\n", tail, array) // var actualString = fmt.Sprintf("%v", actual)
var newArray = make([]interface{}, len(array)) // var prefixMatch = strings.TrimSuffix(key, "*")
for index, value := range array { // if prefixMatch != key {
val, err := n.DeleteChildValue(value, tail) // return strings.HasPrefix(actualString, prefixMatch)
if err != nil { // }
return nil, err // return actualString == key
} // }
newArray[index] = val
}
return newArray, nil
}
func (n *navigator) deleteArray(array []interface{}, paths []string, index int64) (interface{}, error) { // func entriesInSlice(context yaml.MapSlice, key string) []*yaml.MapItem {
n.log.Debugf("deleteArray for %v for %v\n", paths, array) // var matches = make([]*yaml.MapItem, 0)
// for idx := range context {
// var entry = &context[idx]
// if matchesKey(key, entry.Key) {
// matches = append(matches, entry)
// }
// }
// return matches
// }
if index >= int64(len(array)) { // func getMapSlice(context interface{}) yaml.MapSlice {
return array, nil // var mapSlice yaml.MapSlice
} // switch context := context.(type) {
// case yaml.MapSlice:
// mapSlice = context
// default:
// mapSlice = make(yaml.MapSlice, 0)
// }
// return mapSlice
// }
remainingPaths := paths[1:] // func getArray(context interface{}) (array []interface{}, ok bool) {
if len(remainingPaths) > 0 { // switch context := context.(type) {
// recurse into the array element at index // case []interface{}:
var errorDeleting error // array = context
array[index], errorDeleting = n.deleteMap(array[index], remainingPaths) // ok = true
if errorDeleting != nil { // default:
return nil, errorDeleting // array = make([]interface{}, 0)
} // ok = false
// }
// return
// }
} else { // func writeMap(context interface{}, paths []string, value interface{}) interface{} {
// Delete the array element at index // log.Debugf("writeMap with path %v for %v to set value %v\n", paths, context, value)
array = append(array[:index], array[index+1:]...)
n.log.Debugf("\tDeleted item index %d from array, leaving %v", index, array)
}
n.log.Debugf("\tReturning array: %v\n", array) // mapSlice := getMapSlice(context)
return array, nil
} // if len(paths) == 0 {
// return context
// }
// children := entriesInSlice(mapSlice, paths[0])
// if len(children) == 0 && paths[0] == "*" {
// log.Debugf("\tNo matches, return map as is")
// return context
// }
// if len(children) == 0 {
// newChild := yaml.MapItem{Key: paths[0]}
// mapSlice = append(mapSlice, newChild)
// children = entriesInSlice(mapSlice, paths[0])
// log.Debugf("\tAppended child at %v for mapSlice %v\n", paths[0], mapSlice)
// }
// remainingPaths := paths[1:]
// for _, child := range children {
// child.Value = updatedChildValue(child.Value, remainingPaths, value)
// }
// log.Debugf("\tReturning mapSlice %v\n", mapSlice)
// return mapSlice
// }
// func updatedChildValue(child interface{}, remainingPaths []string, value interface{}) interface{} {
// if len(remainingPaths) == 0 {
// return value
// }
// log.Debugf("updatedChildValue for child %v with path %v to set value %v", child, remainingPaths, value)
// log.Debugf("type of child is %v", reflect.TypeOf(child))
// switch child := child.(type) {
// case nil:
// if remainingPaths[0] == "+" || remainingPaths[0] == "*" {
// return writeArray(child, remainingPaths, value)
// }
// case []interface{}:
// _, nextIndexErr := strconv.ParseInt(remainingPaths[0], 10, 64)
// arrayCommand := nextIndexErr == nil || remainingPaths[0] == "+" || remainingPaths[0] == "*"
// if arrayCommand {
// return writeArray(child, remainingPaths, value)
// }
// }
// return writeMap(child, remainingPaths, value)
// }
// func writeArray(context interface{}, paths []string, value interface{}) []interface{} {
// log.Debugf("writeArray with path %v for %v to set value %v\n", paths, context, value)
// array, _ := getArray(context)
// if len(paths) == 0 {
// return array
// }
// log.Debugf("\tarray %v\n", array)
// rawIndex := paths[0]
// remainingPaths := paths[1:]
// var index int64
// // the append array indicator
// if rawIndex == "+" {
// index = int64(len(array))
// } else if rawIndex == "*" {
// for index, oldChild := range array {
// array[index] = updatedChildValue(oldChild, remainingPaths, value)
// }
// return array
// } else {
// index, _ = strconv.ParseInt(rawIndex, 10, 64) // nolint
// // writeArray is only called by updatedChildValue which handles parsing the
// // index, as such this renders this dead code.
// }
// for index >= int64(len(array)) {
// array = append(array, nil)
// }
// currentChild := array[index]
// log.Debugf("\tcurrentChild %v\n", currentChild)
// array[index] = updatedChildValue(currentChild, remainingPaths, value)
// log.Debugf("\tReturning array %v\n", array)
// return array
// }
// func readMap(context yaml.MapSlice, head string, tail []string) (interface{}, error) {
// log.Debugf("readingMap %v with key %v\n", context, head)
// if head == "*" {
// return readMapSplat(context, tail)
// }
// entries := entriesInSlice(context, head)
// if len(entries) == 1 {
// return calculateValue(entries[0].Value, tail)
// } else if len(entries) == 0 {
// return nil, nil
// }
// var errInIdx error
// values := make([]interface{}, len(entries))
// for idx, entry := range entries {
// values[idx], errInIdx = calculateValue(entry.Value, tail)
// if errInIdx != nil {
// log.Errorf("Error updating index %v in %v", idx, context)
// return nil, errInIdx
// }
// }
// return values, nil
// }
// func readMapSplat(context yaml.MapSlice, tail []string) (interface{}, error) {
// var newArray = make([]interface{}, len(context))
// var i = 0
// for _, entry := range context {
// if len(tail) > 0 {
// val, err := recurse(entry.Value, tail[0], tail[1:])
// if err != nil {
// return nil, err
// }
// newArray[i] = val
// } else {
// newArray[i] = entry.Value
// }
// i++
// }
// return newArray, nil
// }
// func recurse(value interface{}, head string, tail []string) (interface{}, error) {
// switch value := value.(type) {
// case []interface{}:
// if head == "*" {
// return readArraySplat(value, tail)
// }
// index, err := strconv.ParseInt(head, 10, 64)
// if err != nil {
// return nil, fmt.Errorf("error accessing array: %v", err)
// }
// return readArray(value, index, tail)
// case yaml.MapSlice:
// return readMap(value, head, tail)
// default:
// return nil, nil
// }
// }
// func readArray(array []interface{}, head int64, tail []string) (interface{}, error) {
// if head >= int64(len(array)) {
// return nil, nil
// }
// value := array[head]
// return calculateValue(value, tail)
// }
// func readArraySplat(array []interface{}, tail []string) (interface{}, error) {
// var newArray = make([]interface{}, len(array))
// for index, value := range array {
// val, err := calculateValue(value, tail)
// if err != nil {
// return nil, err
// }
// newArray[index] = val
// }
// return newArray, nil
// }
// func calculateValue(value interface{}, tail []string) (interface{}, error) {
// if len(tail) > 0 {
// return recurse(value, tail[0], tail[1:])
// }
// return value, nil
// }
// func deleteMap(context interface{}, paths []string) (yaml.MapSlice, error) {
// log.Debugf("deleteMap for %v for %v\n", paths, context)
// mapSlice := getMapSlice(context)
// if len(paths) == 0 {
// return mapSlice, nil
// }
// var index int
// var child yaml.MapItem
// for index, child = range mapSlice {
// if matchesKey(paths[0], child.Key) {
// log.Debugf("\tMatched [%v] with [%v] at index %v", paths[0], child.Key, index)
// var badDelete error
// mapSlice, badDelete = deleteEntryInMap(mapSlice, child, index, paths)
// if badDelete != nil {
// return nil, badDelete
// }
// }
// }
// return mapSlice, nil
// }
// func deleteEntryInMap(original yaml.MapSlice, child yaml.MapItem, index int, paths []string) (yaml.MapSlice, error) {
// remainingPaths := paths[1:]
// var newSlice yaml.MapSlice
// if len(remainingPaths) > 0 {
// newChild := yaml.MapItem{Key: child.Key}
// var errorDeleting error
// newChild.Value, errorDeleting = deleteChildValue(child.Value, remainingPaths)
// if errorDeleting != nil {
// return nil, errorDeleting
// }
// newSlice = make(yaml.MapSlice, len(original))
// for i := range original {
// item := original[i]
// if i == index {
// item = newChild
// }
// newSlice[i] = item
// }
// } else {
// // Delete item from slice at index
// newSlice = append(original[:index], original[index+1:]...)
// log.Debugf("\tDeleted item index %d from original", index)
// }
// log.Debugf("\tReturning original %v\n", original)
// return newSlice, nil
// }
// func deleteArraySplat(array []interface{}, tail []string) (interface{}, error) {
// log.Debugf("deleteArraySplat for %v for %v\n", tail, array)
// var newArray = make([]interface{}, len(array))
// for index, value := range array {
// val, err := deleteChildValue(value, tail)
// if err != nil {
// return nil, err
// }
// newArray[index] = val
// }
// return newArray, nil
// }
// func deleteArray(array []interface{}, paths []string, index int64) (interface{}, error) {
// log.Debugf("deleteArray for %v for %v\n", paths, array)
// if index >= int64(len(array)) {
// return array, nil
// }
// remainingPaths := paths[1:]
// if len(remainingPaths) > 0 {
// // Recurse into the array element at index
// var errorDeleting error
// array[index], errorDeleting = deleteMap(array[index], remainingPaths)
// if errorDeleting != nil {
// return nil, errorDeleting
// }
// } else {
// // Delete the array element at index
// array = append(array[:index], array[index+1:]...)
// log.Debugf("\tDeleted item index %d from array, leaving %v", index, array)
// }
// log.Debugf("\tReturning array: %v\n", array)
// return array, nil
// }
// func deleteChildValue(child interface{}, remainingPaths []string) (interface{}, error) {
// log.Debugf("deleteChildValue for %v for %v\n", remainingPaths, child)
// var head = remainingPaths[0]
// var tail = remainingPaths[1:]
// switch child := child.(type) {
// case yaml.MapSlice:
// return deleteMap(child, remainingPaths)
// case []interface{}:
// if head == "*" {
// return deleteArraySplat(child, tail)
// }
// index, err := strconv.ParseInt(head, 10, 64)
// if err != nil {
// return nil, fmt.Errorf("error accessing array: %v", err)
// }
// return deleteArray(child, remainingPaths, index)
// }
// return child, nil
// }

52
pkg/yqlib/lib.go
View File

@ -1,16 +1,13 @@
package yqlib package yqlib
import ( import (
mergo "gopkg.in/imdario/mergo.v0"
logging "gopkg.in/op/go-logging.v1" logging "gopkg.in/op/go-logging.v1"
yaml "gopkg.in/yaml.v3"
) )
type YqLib interface { type YqLib interface {
ReadPath(dataBucket interface{}, path string) (interface{}, error) Get(rootNode *yaml.Node, path string) (*yaml.Node, error)
WritePath(dataBucket interface{}, path string, value interface{}) interface{} Update(rootNode *yaml.Node, path string, writeCommand WriteCommand) error
PrefixPath(dataBucket interface{}, prefix string) interface{}
DeletePath(dataBucket interface{}, path string) (interface{}, error)
Merge(dst interface{}, src interface{}, overwrite bool, append bool) error
} }
type lib struct { type lib struct {
@ -25,44 +22,15 @@ func NewYqLib(l *logging.Logger) YqLib {
} }
} }
func (l *lib) ReadPath(dataBucket interface{}, path string) (interface{}, error) { func (l *lib) Get(rootNode *yaml.Node, path string) (*yaml.Node, error) {
var paths = l.parser.ParsePath(path) if path == "" {
return l.navigator.ReadChildValue(dataBucket, paths) return rootNode, nil
}
func (l *lib) WritePath(dataBucket interface{}, path string, value interface{}) interface{} {
var paths = l.parser.ParsePath(path)
return l.navigator.UpdatedChildValue(dataBucket, paths, value)
}
func (l *lib) PrefixPath(dataBucket interface{}, prefix string) interface{} {
var paths = l.parser.ParsePath(prefix)
// Inverse order
for i := len(paths)/2 - 1; i >= 0; i-- {
opp := len(paths) - 1 - i
paths[i], paths[opp] = paths[opp], paths[i]
} }
var mapDataBucket = dataBucket
for _, key := range paths {
singlePath := []string{key}
mapDataBucket = l.navigator.UpdatedChildValue(nil, singlePath, mapDataBucket)
}
return mapDataBucket
}
func (l *lib) DeletePath(dataBucket interface{}, path string) (interface{}, error) {
var paths = l.parser.ParsePath(path) var paths = l.parser.ParsePath(path)
return l.navigator.DeleteChildValue(dataBucket, paths) return l.navigator.Get(rootNode, paths)
} }
func (l *lib) Merge(dst interface{}, src interface{}, overwrite bool, append bool) error { func (l *lib) Update(rootNode *yaml.Node, path string, writeCommand WriteCommand) error {
if overwrite { var paths = l.parser.ParsePath(path)
return mergo.Merge(dst, src, mergo.WithOverride) return l.navigator.Update(rootNode, paths, writeCommand)
} else if append {
return mergo.Merge(dst, src, mergo.WithAppendSlice)
}
return mergo.Merge(dst, src)
} }

556
yq.go
View File

@ -6,20 +6,19 @@ import (
"io" "io"
"io/ioutil" "io/ioutil"
"os" "os"
"reflect"
"strconv" "strconv"
"strings"
"github.com/mikefarah/yq/v2/pkg/marshal" "github.com/mikefarah/yq/v3/pkg/marshal"
"github.com/mikefarah/yq/v2/pkg/yqlib" "github.com/mikefarah/yq/v3/pkg/yqlib"
errors "github.com/pkg/errors" errors "github.com/pkg/errors"
yaml "github.com/mikefarah/yaml/v2"
"github.com/spf13/cobra" "github.com/spf13/cobra"
logging "gopkg.in/op/go-logging.v1" logging "gopkg.in/op/go-logging.v1"
yaml "gopkg.in/yaml.v3"
) )
var rawOutput = false
var trimOutput = true var trimOutput = true
var writeInplace = false var writeInplace = false
var writeScript = "" var writeScript = ""
@ -45,7 +44,6 @@ func main() {
} }
func newCommandCLI() *cobra.Command { func newCommandCLI() *cobra.Command {
yaml.DefaultMapType = reflect.TypeOf(yaml.MapSlice{})
var rootCmd = &cobra.Command{ var rootCmd = &cobra.Command{
Use: "yq", Use: "yq",
Short: "yq is a lightweight and portable command-line YAML processor.", Short: "yq is a lightweight and portable command-line YAML processor.",
@ -83,10 +81,10 @@ func newCommandCLI() *cobra.Command {
rootCmd.AddCommand( rootCmd.AddCommand(
createReadCmd(), createReadCmd(),
createWriteCmd(), createWriteCmd(),
createPrefixCmd(), // createPrefixCmd(),
createDeleteCmd(), // createDeleteCmd(),
createNewCmd(), // createNewCmd(),
createMergeCmd(), // createMergeCmd(),
) )
rootCmd.SetOutput(os.Stdout) rootCmd.SetOutput(os.Stdout)
@ -110,6 +108,7 @@ yq r -- things.yaml --key-starting-with-dashes
RunE: readProperty, RunE: readProperty,
} }
cmdRead.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)") cmdRead.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)")
cmdRead.PersistentFlags().BoolVarP(&rawOutput, "raw", "r", false, "raw yaml output - prints out values instead of yaml")
cmdRead.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "output as json") cmdRead.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "output as json")
return cmdRead return cmdRead
} }
@ -149,104 +148,104 @@ a.b.e:
return cmdWrite return cmdWrite
} }
func createPrefixCmd() *cobra.Command { // func createPrefixCmd() *cobra.Command {
var cmdWrite = &cobra.Command{ // var cmdWrite = &cobra.Command{
Use: "prefix [yaml_file] [path]", // Use: "prefix [yaml_file] [path]",
Aliases: []string{"p"}, // Aliases: []string{"p"},
Short: "yq p [--inplace/-i] [--doc/-d index] sample.yaml a.b.c", // Short: "yq p [--inplace/-i] [--doc/-d index] sample.yaml a.b.c",
Example: ` // Example: `
yq prefix things.yaml a.b.c // yq prefix things.yaml a.b.c
yq prefix --inplace things.yaml a.b.c // yq prefix --inplace things.yaml a.b.c
yq prefix --inplace -- things.yaml --key-starting-with-dash // yq prefix --inplace -- things.yaml --key-starting-with-dash
yq p -i things.yaml a.b.c // yq p -i things.yaml a.b.c
yq p --doc 2 things.yaml a.b.d // yq p --doc 2 things.yaml a.b.d
yq p -d2 things.yaml a.b.d // yq p -d2 things.yaml a.b.d
`, // `,
Long: `Prefixes w.r.t to the yaml file at the given path. // Long: `Prefixes w.r.t to the yaml file at the given path.
Outputs to STDOUT unless the inplace flag is used, in which case the file is updated instead. // Outputs to STDOUT unless the inplace flag is used, in which case the file is updated instead.
`, // `,
RunE: prefixProperty, // RunE: prefixProperty,
} // }
cmdWrite.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace") // cmdWrite.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace")
cmdWrite.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)") // cmdWrite.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)")
return cmdWrite // return cmdWrite
} // }
func createDeleteCmd() *cobra.Command { // func createDeleteCmd() *cobra.Command {
var cmdDelete = &cobra.Command{ // var cmdDelete = &cobra.Command{
Use: "delete [yaml_file] [path]", // Use: "delete [yaml_file] [path]",
Aliases: []string{"d"}, // Aliases: []string{"d"},
Short: "yq d [--inplace/-i] [--doc/-d index] sample.yaml a.b.c", // Short: "yq d [--inplace/-i] [--doc/-d index] sample.yaml a.b.c",
Example: ` // Example: `
yq delete things.yaml a.b.c // yq delete things.yaml a.b.c
yq delete --inplace things.yaml a.b.c // yq delete --inplace things.yaml a.b.c
yq delete --inplace -- things.yaml --key-starting-with-dash // yq delete --inplace -- things.yaml --key-starting-with-dash
yq d -i things.yaml a.b.c // yq d -i things.yaml a.b.c
yq d things.yaml a.b.c // yq d things.yaml a.b.c
`, // `,
Long: `Deletes the given path from the YAML file. // Long: `Deletes the given path from the YAML file.
Outputs to STDOUT unless the inplace flag is used, in which case the file is updated instead. // Outputs to STDOUT unless the inplace flag is used, in which case the file is updated instead.
`, // `,
RunE: deleteProperty, // RunE: deleteProperty,
} // }
cmdDelete.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace") // cmdDelete.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace")
cmdDelete.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)") // cmdDelete.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)")
return cmdDelete // return cmdDelete
} // }
func createNewCmd() *cobra.Command { // func createNewCmd() *cobra.Command {
var cmdNew = &cobra.Command{ // var cmdNew = &cobra.Command{
Use: "new [path] [value]", // Use: "new [path] [value]",
Aliases: []string{"n"}, // Aliases: []string{"n"},
Short: "yq n [--script/-s script_file] a.b.c newValue", // Short: "yq n [--script/-s script_file] a.b.c newValue",
Example: ` // Example: `
yq new a.b.c cat // yq new a.b.c cat
yq n a.b.c cat // yq n a.b.c cat
yq n -- --key-starting-with-dash cat // yq n -- --key-starting-with-dash cat
yq n --script create_script.yaml // yq n --script create_script.yaml
`, // `,
Long: `Creates a new yaml w.r.t the given path and value. // Long: `Creates a new yaml w.r.t the given path and value.
Outputs to STDOUT // Outputs to STDOUT
Create Scripts: // Create Scripts:
Note that you can give a create script to perform more sophisticated yaml. This follows the same format as the update script. // Note that you can give a create script to perform more sophisticated yaml. This follows the same format as the update script.
`, // `,
RunE: newProperty, // RunE: newProperty,
} // }
cmdNew.PersistentFlags().StringVarP(&writeScript, "script", "s", "", "yaml script for updating yaml") // cmdNew.PersistentFlags().StringVarP(&writeScript, "script", "s", "", "yaml script for updating yaml")
return cmdNew // return cmdNew
} // }
func createMergeCmd() *cobra.Command { // func createMergeCmd() *cobra.Command {
var cmdMerge = &cobra.Command{ // var cmdMerge = &cobra.Command{
Use: "merge [initial_yaml_file] [additional_yaml_file]...", // Use: "merge [initial_yaml_file] [additional_yaml_file]...",
Aliases: []string{"m"}, // Aliases: []string{"m"},
Short: "yq m [--inplace/-i] [--doc/-d index] [--overwrite/-x] [--append/-a] sample.yaml sample2.yaml", // Short: "yq m [--inplace/-i] [--doc/-d index] [--overwrite/-x] [--append/-a] sample.yaml sample2.yaml",
Example: ` // Example: `
yq merge things.yaml other.yaml // yq merge things.yaml other.yaml
yq merge --inplace things.yaml other.yaml // yq merge --inplace things.yaml other.yaml
yq m -i things.yaml other.yaml // yq m -i things.yaml other.yaml
yq m --overwrite things.yaml other.yaml // yq m --overwrite things.yaml other.yaml
yq m -i -x things.yaml other.yaml // yq m -i -x things.yaml other.yaml
yq m -i -a things.yaml other.yaml // yq m -i -a things.yaml other.yaml
`, // `,
Long: `Updates the yaml file by adding/updating the path(s) and value(s) from additional yaml file(s). // Long: `Updates the yaml file by adding/updating the path(s) and value(s) from additional yaml file(s).
Outputs to STDOUT unless the inplace flag is used, in which case the file is updated instead. // Outputs to STDOUT unless the inplace flag is used, in which case the file is updated instead.
If overwrite flag is set then existing values will be overwritten using the values from each additional yaml file. // If overwrite flag is set then existing values will be overwritten using the values from each additional yaml file.
If append flag is set then existing arrays will be merged with the arrays from each additional yaml file. // If append flag is set then existing arrays will be merged with the arrays from each additional yaml file.
Note that if you set both flags only overwrite will take effect. // Note that if you set both flags only overwrite will take effect.
`, // `,
RunE: mergeProperties, // RunE: mergeProperties,
} // }
cmdMerge.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace") // cmdMerge.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace")
cmdMerge.PersistentFlags().BoolVarP(&overwriteFlag, "overwrite", "x", false, "update the yaml file by overwriting existing values") // cmdMerge.PersistentFlags().BoolVarP(&overwriteFlag, "overwrite", "x", false, "update the yaml file by overwriting existing values")
cmdMerge.PersistentFlags().BoolVarP(&appendFlag, "append", "a", false, "update the yaml file by appending array values") // cmdMerge.PersistentFlags().BoolVarP(&appendFlag, "append", "a", false, "update the yaml file by appending array values")
cmdMerge.PersistentFlags().BoolVarP(&allowEmptyFlag, "allow-empty", "e", false, "allow empty yaml files") // cmdMerge.PersistentFlags().BoolVarP(&allowEmptyFlag, "allow-empty", "e", false, "allow empty yaml files")
cmdMerge.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)") // cmdMerge.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)")
return cmdMerge // return cmdMerge
} // }
func readProperty(cmd *cobra.Command, args []string) error { func readProperty(cmd *cobra.Command, args []string) error {
var path = "" var path = ""
@ -261,8 +260,9 @@ func readProperty(cmd *cobra.Command, args []string) error {
if errorParsingDocIndex != nil { if errorParsingDocIndex != nil {
return errorParsingDocIndex return errorParsingDocIndex
} }
var mappedDocs []interface{}
var dataBucket interface{} var mappedDocs []*yaml.Node
var dataBucket yaml.Node
var currentIndex = 0 var currentIndex = 0
var errorReadingStream = readStream(args[0], func(decoder *yaml.Decoder) error { var errorReadingStream = readStream(args[0], func(decoder *yaml.Decoder) error {
for { for {
@ -274,21 +274,14 @@ func readProperty(cmd *cobra.Command, args []string) error {
} }
return nil return nil
} }
log.Debugf("processing %v - requested index %v", currentIndex, docIndexInt) log.Debugf("processing document %v - requested index %v", currentIndex, docIndexInt)
if updateAll || currentIndex == docIndexInt { if updateAll || currentIndex == docIndexInt {
log.Debugf("reading %v in index %v", path, currentIndex) log.Debugf("reading %v in document %v", path, currentIndex)
if path == "" { mappedDoc, errorParsing := lib.Get(&dataBucket, path)
log.Debug("no path") if errorParsing != nil {
log.Debugf("%v", dataBucket) return errors.Wrapf(errorParsing, "Error reading path in document index %v", currentIndex)
mappedDocs = append(mappedDocs, dataBucket)
} else {
mappedDoc, errorParsing := lib.ReadPath(dataBucket, path)
log.Debugf("%v", mappedDoc)
if errorParsing != nil {
return errors.Wrapf(errorParsing, "Error reading path in document index %v", currentIndex)
}
mappedDocs = append(mappedDocs, mappedDoc)
} }
mappedDocs = append(mappedDocs, mappedDoc)
} }
currentIndex = currentIndex + 1 currentIndex = currentIndex + 1
} }
@ -298,56 +291,64 @@ func readProperty(cmd *cobra.Command, args []string) error {
return errorReadingStream return errorReadingStream
} }
if !updateAll { var encoder = yaml.NewEncoder(cmd.OutOrStdout())
dataBucket = mappedDocs[0] encoder.SetIndent(2)
} else { var err error
dataBucket = mappedDocs
}
dataStr, err := toString(dataBucket) if rawOutput {
for _, mappedDoc := range mappedDocs {
if mappedDoc != nil {
cmd.Println(mappedDoc.Value)
}
}
} else if !updateAll {
err = encoder.Encode(mappedDocs[0])
} else {
err = encoder.Encode(&yaml.Node{Kind: yaml.SequenceNode, Content: mappedDocs})
}
if err != nil { if err != nil {
return err return err
} }
cmd.Println(dataStr) encoder.Close()
return nil return nil
} }
func newProperty(cmd *cobra.Command, args []string) error { // func newProperty(cmd *cobra.Command, args []string) error {
updatedData, err := newYaml(args) // updatedData, err := newYaml(args)
if err != nil { // if err != nil {
return err // return err
} // }
dataStr, err := toString(updatedData) // dataStr, err := toString(updatedData)
if err != nil { // if err != nil {
return err // return err
} // }
cmd.Println(dataStr) // cmd.Println(dataStr)
return nil // return nil
} // }
func newYaml(args []string) (interface{}, error) { // func newYaml(args []string) (interface{}, error) {
var writeCommands, writeCommandsError = readWriteCommands(args, 2, "Must provide <path_to_update> <value>") // var writeCommands, writeCommandsError = readWriteCommands(args, 2, "Must provide <path_to_update> <value>")
if writeCommandsError != nil { // if writeCommandsError != nil {
return nil, writeCommandsError // return nil, writeCommandsError
} // }
var dataBucket interface{} // var dataBucket interface{}
var isArray = strings.HasPrefix(writeCommands[0].Key.(string), "[") // var isArray = strings.HasPrefix(writeCommands[0].Key.(string), "[")
if isArray { // if isArray {
dataBucket = make([]interface{}, 0) // dataBucket = make([]interface{}, 0)
} else { // } else {
dataBucket = make(yaml.MapSlice, 0) // dataBucket = make(yaml.MapSlice, 0)
} // }
for _, entry := range writeCommands { // for _, entry := range writeCommands {
path := entry.Key.(string) // path := entry.Key.(string)
value := entry.Value // value := entry.Value
log.Debugf("setting %v to %v", path, value) // log.Debugf("setting %v to %v", path, value)
dataBucket = lib.WritePath(dataBucket, path, value) // dataBucket = lib.WritePath(dataBucket, path, value)
} // }
return dataBucket, nil // return dataBucket, nil
} // }
func parseDocumentIndex() (bool, int, error) { func parseDocumentIndex() (bool, int, error) {
if docIndex == "*" { if docIndex == "*" {
@ -360,11 +361,11 @@ func parseDocumentIndex() (bool, int, error) {
return false, int(docIndexInt64), nil return false, int(docIndexInt64), nil
} }
type updateDataFn func(dataBucket interface{}, currentIndex int) (interface{}, error) type updateDataFn func(dataBucket *yaml.Node, currentIndex int) error
func mapYamlDecoder(updateData updateDataFn, encoder *yaml.Encoder) yamlDecoderFn { func mapYamlDecoder(updateData updateDataFn, encoder *yaml.Encoder) yamlDecoderFn {
return func(decoder *yaml.Decoder) error { return func(decoder *yaml.Decoder) error {
var dataBucket interface{} var dataBucket yaml.Node
var errorReading error var errorReading error
var errorWriting error var errorWriting error
var errorUpdating error var errorUpdating error
@ -387,12 +388,12 @@ func mapYamlDecoder(updateData updateDataFn, encoder *yaml.Encoder) yamlDecoderF
} else if errorReading != nil { } else if errorReading != nil {
return errors.Wrapf(errorReading, "Error reading document at index %v, %v", currentIndex, errorReading) return errors.Wrapf(errorReading, "Error reading document at index %v, %v", currentIndex, errorReading)
} }
dataBucket, errorUpdating = updateData(dataBucket, currentIndex) errorUpdating = updateData(&dataBucket, currentIndex)
if errorUpdating != nil { if errorUpdating != nil {
return errors.Wrapf(errorUpdating, "Error updating document at index %v", currentIndex) return errors.Wrapf(errorUpdating, "Error updating document at index %v", currentIndex)
} }
errorWriting = encoder.Encode(dataBucket) errorWriting = encoder.Encode(&dataBucket)
if errorWriting != nil { if errorWriting != nil {
return errors.Wrapf(errorWriting, "Error writing document at index %v, %v", currentIndex, errorWriting) return errors.Wrapf(errorWriting, "Error writing document at index %v, %v", currentIndex, errorWriting)
@ -412,43 +413,47 @@ func writeProperty(cmd *cobra.Command, args []string) error {
return errorParsingDocIndex return errorParsingDocIndex
} }
var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) { var updateData = func(dataBucket *yaml.Node, currentIndex int) error {
if updateAll || currentIndex == docIndexInt { if updateAll || currentIndex == docIndexInt {
log.Debugf("Updating doc %v", currentIndex) log.Debugf("Updating doc %v", currentIndex)
for _, entry := range writeCommands { for _, entry := range writeCommands {
path := entry.Key.(string) path := entry.Key
value := entry.Value changesToApply := entry.Value
log.Debugf("setting %v to %v", path, value) var paths = parsePath(path)
dataBucket = lib.WritePath(dataBucket, path, value)
errorUpdating := updateChild(dataBucket, paths, changesToApply)
if errorUpdating != nil {
return errorUpdating
}
} }
} }
return dataBucket, nil return nil
} }
return readAndUpdate(cmd.OutOrStdout(), args[0], updateData) return readAndUpdate(cmd.OutOrStdout(), args[0], updateData)
} }
func prefixProperty(cmd *cobra.Command, args []string) error { // func prefixProperty(cmd *cobra.Command, args []string) error {
if len(args) != 2 { // if len(args) != 2 {
return errors.New("Must provide <filename> <prefixed_path>") // return errors.New("Must provide <filename> <prefixed_path>")
} // }
prefixPath := args[1] // prefixPath := args[1]
var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex() // var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex()
if errorParsingDocIndex != nil { // if errorParsingDocIndex != nil {
return errorParsingDocIndex // return errorParsingDocIndex
} // }
var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) { // var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) {
if updateAll || currentIndex == docIndexInt { // if updateAll || currentIndex == docIndexInt {
log.Debugf("Prefixing %v to doc %v", prefixPath, currentIndex) // log.Debugf("Prefixing %v to doc %v", prefixPath, currentIndex)
var mapDataBucket = lib.PrefixPath(dataBucket, prefixPath) // var mapDataBucket = lib.PrefixPath(dataBucket, prefixPath)
return mapDataBucket, nil // return mapDataBucket, nil
} // }
return dataBucket, nil // return dataBucket, nil
} // }
return readAndUpdate(cmd.OutOrStdout(), args[0], updateData) // return readAndUpdate(cmd.OutOrStdout(), args[0], updateData)
} // }
func readAndUpdate(stdOut io.Writer, inputFile string, updateData updateDataFn) error { func readAndUpdate(stdOut io.Writer, inputFile string, updateData updateDataFn) error {
var destination io.Writer var destination io.Writer
@ -479,90 +484,137 @@ func readAndUpdate(stdOut io.Writer, inputFile string, updateData updateDataFn)
defer safelyFlush(writer) defer safelyFlush(writer)
} }
var encoder = yaml.NewEncoder(destination) var encoder = yaml.NewEncoder(destination)
encoder.SetIndent(2)
log.Debugf("Writing to %v from %v", destinationName, inputFile) log.Debugf("Writing to %v from %v", destinationName, inputFile)
return readStream(inputFile, mapYamlDecoder(updateData, encoder)) return readStream(inputFile, mapYamlDecoder(updateData, encoder))
} }
func deleteProperty(cmd *cobra.Command, args []string) error { // func deleteProperty(cmd *cobra.Command, args []string) error {
if len(args) < 2 { // if len(args) < 2 {
return errors.New("Must provide <filename> <path_to_delete>") // return errors.New("Must provide <filename> <path_to_delete>")
} // }
var deletePath = args[1] // var deletePath = args[1]
var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex() // var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex()
if errorParsingDocIndex != nil { // if errorParsingDocIndex != nil {
return errorParsingDocIndex // return errorParsingDocIndex
} // }
var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) { // var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) {
if updateAll || currentIndex == docIndexInt { // if updateAll || currentIndex == docIndexInt {
log.Debugf("Deleting path in doc %v", currentIndex) // log.Debugf("Deleting path in doc %v", currentIndex)
return lib.DeletePath(dataBucket, deletePath) // return lib.DeletePath(dataBucket, deletePath)
} // }
return dataBucket, nil // return dataBucket, nil
} // }
return readAndUpdate(cmd.OutOrStdout(), args[0], updateData) // return readAndUpdate(cmd.OutOrStdout(), args[0], updateData)
// }
// func mergeProperties(cmd *cobra.Command, args []string) error {
// if len(args) < 2 {
// return errors.New("Must provide at least 2 yaml files")
// }
// var input = args[0]
// var filesToMerge = args[1:]
// var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex()
// if errorParsingDocIndex != nil {
// return errorParsingDocIndex
// }
// var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) {
// if updateAll || currentIndex == docIndexInt {
// log.Debugf("Merging doc %v", currentIndex)
// var mergedData map[interface{}]interface{}
// // merge only works for maps, so put everything in a temporary
// // map
// var mapDataBucket = make(map[interface{}]interface{})
// mapDataBucket["root"] = dataBucket
// if err := lib.Merge(&mergedData, mapDataBucket, overwriteFlag, appendFlag); err != nil {
// return nil, err
// }
// for _, f := range filesToMerge {
// var fileToMerge interface{}
// if err := readData(f, 0, &fileToMerge); err != nil {
// if allowEmptyFlag && err == io.EOF {
// continue
// }
// return nil, err
// }
// mapDataBucket["root"] = fileToMerge
// if err := lib.Merge(&mergedData, mapDataBucket, overwriteFlag, appendFlag); err != nil {
// return nil, err
// }
// }
// return mergedData["root"], nil
// }
// return dataBucket, nil
// }
// return readAndUpdate(cmd.OutOrStdout(), input, updateData)
// }
type rawWriteCommand struct {
// Command string TODO
Key string
Value yaml.Node
} }
func mergeProperties(cmd *cobra.Command, args []string) error { func readWriteCommands(args []string, expectedArgs int, badArgsMessage string) ([]rawWriteCommand, error) {
if len(args) < 2 { var writeCommands []rawWriteCommand
return errors.New("Must provide at least 2 yaml files")
}
var input = args[0]
var filesToMerge = args[1:]
var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex()
if errorParsingDocIndex != nil {
return errorParsingDocIndex
}
var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) {
if updateAll || currentIndex == docIndexInt {
log.Debugf("Merging doc %v", currentIndex)
var mergedData map[interface{}]interface{}
// merge only works for maps, so put everything in a temporary
// map
var mapDataBucket = make(map[interface{}]interface{})
mapDataBucket["root"] = dataBucket
if err := lib.Merge(&mergedData, mapDataBucket, overwriteFlag, appendFlag); err != nil {
return nil, err
}
for _, f := range filesToMerge {
var fileToMerge interface{}
if err := readData(f, 0, &fileToMerge); err != nil {
if allowEmptyFlag && err == io.EOF {
continue
}
return nil, err
}
mapDataBucket["root"] = fileToMerge
if err := lib.Merge(&mergedData, mapDataBucket, overwriteFlag, appendFlag); err != nil {
return nil, err
}
}
return mergedData["root"], nil
}
return dataBucket, nil
}
yaml.DefaultMapType = reflect.TypeOf(map[interface{}]interface{}{})
defer func() { yaml.DefaultMapType = reflect.TypeOf(yaml.MapSlice{}) }()
return readAndUpdate(cmd.OutOrStdout(), input, updateData)
}
func readWriteCommands(args []string, expectedArgs int, badArgsMessage string) (yaml.MapSlice, error) {
var writeCommands yaml.MapSlice
if writeScript != "" { if writeScript != "" {
if err := readData(writeScript, 0, &writeCommands); err != nil { var rawCommands yaml.Node
if err := readData(writeScript, 0, &rawCommands); err != nil {
return nil, err return nil, err
} }
log.Debugf("Read write commands file '%v'", rawCommands)
var key string
for index, content := range rawCommands.Content[0].Content {
if index%2 == 0 { // must be the key
key = content.Value
} else { // its the value
writeCommands = append(writeCommands, rawWriteCommand{Key: key, Value: *content})
}
}
log.Debugf("Read write commands '%v'", writeCommands)
} else if len(args) < expectedArgs { } else if len(args) < expectedArgs {
return nil, errors.New(badArgsMessage) return nil, errors.New(badArgsMessage)
} else { } else {
writeCommands = make(yaml.MapSlice, 1) writeCommands = make([]rawWriteCommand, 1)
writeCommands[0] = yaml.MapItem{Key: args[expectedArgs-2], Value: valueParser.ParseValue(args[expectedArgs-1])} writeCommands[0] = rawWriteCommand{Key: args[expectedArgs-2], Value: parseValue(args[expectedArgs-1])}
} }
return writeCommands, nil return writeCommands, nil
} }
func parseValue(argument string) yaml.Node {
var err interface{}
var tag = customTag
var inQuotes = len(argument) > 0 && argument[0] == '"'
if tag == "" && !inQuotes {
_, err = strconv.ParseBool(argument)
if err == nil {
tag = "!!bool"
}
_, err = strconv.ParseFloat(argument, 64)
if err == nil {
tag = "!!float"
}
_, err = strconv.ParseInt(argument, 10, 64)
if err == nil {
tag = "!!int"
}
if argument == "null" {
tag = "!!null"
}
if argument == "[]" {
return yaml.Node{Tag: "!!seq", Kind: yaml.SequenceNode}
}
}
log.Debugf("Updating node to value '%v', tag: '%v'", argument, tag)
return yaml.Node{Value: argument, Tag: tag, Kind: yaml.ScalarNode}
}
func toString(context interface{}) (string, error) { func toString(context interface{}) (string, error) {
if outputToJSON { if outputToJSON {
return jsonConverter.JsonToString(context) return jsonConverter.JsonToString(context)
@ -572,7 +624,7 @@ func toString(context interface{}) (string, error) {
func safelyRenameFile(from string, to string) { func safelyRenameFile(from string, to string) {
if renameError := os.Rename(from, to); renameError != nil { if renameError := os.Rename(from, to); renameError != nil {
log.Debugf("Error renaming from %v to %v, attemting to copy contents", from, to) log.Debugf("Error renaming from %v to %v, attempting to copy contents", from, to)
log.Debug(renameError.Error()) log.Debug(renameError.Error())
// can't do this rename when running in docker to a file targeted in a mounted volume, // can't do this rename when running in docker to a file targeted in a mounted volume,
// so gracefully degrade to copying the entire contents. // so gracefully degrade to copying the entire contents.