From aad15ccc6e838552994ab5404a11f59fbfcc7519 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 6 Dec 2019 15:57:46 +1100 Subject: [PATCH] better v3 --- examples/instruction_sample.yaml | 7 +- go.mod | 3 +- go.sum | 5 + pkg/yqlib/data_navigator.go | 751 ++++++++++++++++++------------- pkg/yqlib/lib.go | 52 +-- yq.go | 556 ++++++++++++----------- 6 files changed, 753 insertions(+), 621 deletions(-) diff --git a/examples/instruction_sample.yaml b/examples/instruction_sample.yaml index ee2f8e8d..b7864db4 100644 --- a/examples/instruction_sample.yaml +++ b/examples/instruction_sample.yaml @@ -1,2 +1,7 @@ -b.c: cat +- command: update + path: b.c + value: + #great + things: frog # wow! + b.e[+].name: Mike Farah diff --git a/go.mod b/go.mod index 0b008769..efc8ffd2 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/mikefarah/yq/v2 +module github.com/mikefarah/yq/v3 require ( github.com/mikefarah/yaml/v2 v2.4.0 @@ -7,6 +7,7 @@ require ( golang.org/x/tools v0.0.0-20191030203535-5e247c9ad0a0 // indirect gopkg.in/imdario/mergo.v0 v0.3.7 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 + gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2 ) go 1.13 diff --git a/go.sum b/go.sum index 40ff65e5..307ebb49 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,11 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mikefarah/yaml v2.1.0+incompatible h1:nu2cqmzk4WlWJNgnevY88faMcdrDzYGcsUjYFxEpB7Y= github.com/mikefarah/yaml/v2 v2.4.0 h1:eYqfooY0BnvKTJxr7+ABJs13n3dg9n347GScDaU2Lww= github.com/mikefarah/yaml/v2 v2.4.0/go.mod h1:ahVqZF4n1W4NqwvVnZzC4es67xsW9uR/RRf2RRxieJU= +github.com/mikefarah/yq v2.4.0+incompatible h1:oBxbWy8R9hI3BIUUxEf0CzikWa2AgnGrGhvGQt5jgjk= +github.com/mikefarah/yq/v2 v2.4.1 h1:tajDonaFK6WqitSZExB6fKlWQy/yCkptqxh2AXEe3N4= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -48,3 +51,5 @@ gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2 h1:XZx7nhd5GMaZpmDaEHFVafUZC7ya0fuo7cSJ3UCKYmM= +gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/yqlib/data_navigator.go b/pkg/yqlib/data_navigator.go index 203eb6c0..66741598 100644 --- a/pkg/yqlib/data_navigator.go +++ b/pkg/yqlib/data_navigator.go @@ -1,19 +1,20 @@ package yqlib import ( - "fmt" - "reflect" "strconv" - "strings" - yaml "github.com/mikefarah/yaml/v2" logging "gopkg.in/op/go-logging.v1" + yaml "gopkg.in/yaml.v3" ) +type WriteCommand struct { + // Command string TODO + Value yaml.Node +} + type DataNavigator interface { - ReadChildValue(child interface{}, remainingPaths []string) (interface{}, error) - UpdatedChildValue(child interface{}, remainingPaths []string, value interface{}) interface{} - DeleteChildValue(child interface{}, remainingPaths []string) (interface{}, error) + Get(rootNode *yaml.Node, remainingPath []string) (*yaml.Node, error) + Update(rootNode *yaml.Node, remainingPath []string, writeCommand WriteCommand) error } type navigator struct { @@ -26,351 +27,451 @@ func NewDataNavigator(l *logging.Logger) DataNavigator { } } -func (n *navigator) ReadChildValue(child interface{}, remainingPaths []string) (interface{}, error) { - if len(remainingPaths) == 0 { - return child, nil +func (n *navigator) Get(value *yaml.Node, path []string) (*yaml.Node, error) { + realValue := value + if realValue.Kind == yaml.DocumentNode { + realValue = value.Content[0] } - return n.recurse(child, remainingPaths[0], remainingPaths[1:]) + if len(path) > 0 { + n.log.Debug("diving into %v", path[0]) + return n.recurse(realValue, path[0], path[1:]) + } + return realValue, nil } -func (n *navigator) UpdatedChildValue(child interface{}, remainingPaths []string, value interface{}) interface{} { - if len(remainingPaths) == 0 { - return value +func (n *navigator) guessKind(tail []string) yaml.Kind { + n.log.Debug("tail %v", tail) + if len(tail) == 0 { + n.log.Debug("scalar") + return yaml.ScalarNode } - n.log.Debugf("UpdatedChildValue for child %v with path %v to set value %v", child, remainingPaths, value) - n.log.Debugf("type of child is %v", reflect.TypeOf(child)) - - switch child := child.(type) { - case nil: - if remainingPaths[0] == "+" || remainingPaths[0] == "*" { - return n.writeArray(child, remainingPaths, value) - } - case []interface{}: - _, nextIndexErr := strconv.ParseInt(remainingPaths[0], 10, 64) - arrayCommand := nextIndexErr == nil || remainingPaths[0] == "+" || remainingPaths[0] == "*" - if arrayCommand { - return n.writeArray(child, remainingPaths, value) - } + var _, errorParsingInt = strconv.ParseInt(tail[0], 10, 64) + if tail[0] == "*" || tail[0] == "+" || errorParsingInt == nil { + return yaml.SequenceNode } - return n.writeMap(child, remainingPaths, value) + return yaml.MappingNode } -func (n *navigator) DeleteChildValue(child interface{}, remainingPaths []string) (interface{}, error) { - n.log.Debugf("DeleteChildValue for %v for %v\n", remainingPaths, child) - if len(remainingPaths) == 0 { - return child, nil +func (n *navigator) getOrReplace(original *yaml.Node, expectedKind yaml.Kind) *yaml.Node { + // expected is a scalar when we reach the end of the path + // no need to clobber the original because: + // when reading, it should deal with the original kind + // when writing, it will clobber the kind anyway + if original.Kind != expectedKind && (expectedKind != yaml.ScalarNode) { + return &yaml.Node{Kind: expectedKind} } - var head = remainingPaths[0] - var tail = remainingPaths[1:] - switch child := child.(type) { - case yaml.MapSlice: - return n.deleteMap(child, remainingPaths) - case []interface{}: - if head == "*" { - return n.deleteArraySplat(child, tail) - } - index, err := strconv.ParseInt(head, 10, 64) - if err != nil { - return nil, fmt.Errorf("error accessing array: %v", err) - } - return n.deleteArray(child, remainingPaths, index) - } - return child, nil + return original } -func (n *navigator) recurse(value interface{}, head string, tail []string) (interface{}, error) { - switch value := value.(type) { - case []interface{}: - if head == "*" { - return n.readArraySplat(value, tail) - } - index, err := strconv.ParseInt(head, 10, 64) - if err != nil { - return nil, fmt.Errorf("error accessing array: %v", err) - } - return n.readArray(value, index, tail) - case yaml.MapSlice: - return n.readMap(value, head, tail) - default: - return nil, nil - } -} - -func (n *navigator) matchesKey(key string, actual interface{}) bool { - var actualString = fmt.Sprintf("%v", actual) - var prefixMatch = strings.TrimSuffix(key, "*") - if prefixMatch != key { - return strings.HasPrefix(actualString, prefixMatch) - } - return actualString == key -} - -func (n *navigator) entriesInSlice(context yaml.MapSlice, key string) []*yaml.MapItem { - var matches = make([]*yaml.MapItem, 0) - for idx := range context { - var entry = &context[idx] - if n.matchesKey(key, entry.Key) { - matches = append(matches, entry) - } - } - return matches -} - -func (n *navigator) getMapSlice(context interface{}) yaml.MapSlice { - var mapSlice yaml.MapSlice - switch context := context.(type) { - case yaml.MapSlice: - mapSlice = context - default: - mapSlice = make(yaml.MapSlice, 0) - } - return mapSlice -} - -func (n *navigator) getArray(context interface{}) (array []interface{}, ok bool) { - switch context := context.(type) { - case []interface{}: - array = context - ok = true - default: - array = make([]interface{}, 0) - ok = false - } - return -} - -func (n *navigator) writeMap(context interface{}, paths []string, value interface{}) interface{} { - n.log.Debugf("writeMap with path %v for %v to set value %v\n", paths, context, value) - - mapSlice := n.getMapSlice(context) - - if len(paths) == 0 { - return context - } - - children := n.entriesInSlice(mapSlice, paths[0]) - - if len(children) == 0 && paths[0] == "*" { - n.log.Debugf("\tNo matches, return map as is") - return context - } - - if len(children) == 0 { - newChild := yaml.MapItem{Key: paths[0]} - mapSlice = append(mapSlice, newChild) - children = n.entriesInSlice(mapSlice, paths[0]) - n.log.Debugf("\tAppended child at %v for mapSlice %v\n", paths[0], mapSlice) - } - - remainingPaths := paths[1:] - for _, child := range children { - child.Value = n.UpdatedChildValue(child.Value, remainingPaths, value) - } - n.log.Debugf("\tReturning mapSlice %v\n", mapSlice) - return mapSlice -} - -func (n *navigator) writeArray(context interface{}, paths []string, value interface{}) []interface{} { - n.log.Debugf("writeArray with path %v for %v to set value %v\n", paths, context, value) - array, _ := n.getArray(context) - - if len(paths) == 0 { - return array - } - - n.log.Debugf("\tarray %v\n", array) - - rawIndex := paths[0] - remainingPaths := paths[1:] - var index int64 - // the append array indicator - if rawIndex == "+" { - index = int64(len(array)) - } else if rawIndex == "*" { - for index, oldChild := range array { - array[index] = n.UpdatedChildValue(oldChild, remainingPaths, value) - } - return array - } else { - index, _ = strconv.ParseInt(rawIndex, 10, 64) // nolint - // writeArray is only called by UpdatedChildValue which handles parsing the - // index, as such this renders this dead code. - } - - for index >= int64(len(array)) { - array = append(array, nil) - } - currentChild := array[index] - - n.log.Debugf("\tcurrentChild %v\n", currentChild) - - array[index] = n.UpdatedChildValue(currentChild, remainingPaths, value) - n.log.Debugf("\tReturning array %v\n", array) - return array -} - -func (n *navigator) readMap(context yaml.MapSlice, head string, tail []string) (interface{}, error) { - n.log.Debugf("readingMap %v with key %v\n", context, head) - if head == "*" { - return n.readMapSplat(context, tail) - } - - entries := n.entriesInSlice(context, head) - if len(entries) == 1 { - return n.calculateValue(entries[0].Value, tail) - } else if len(entries) == 0 { - return nil, nil - } - var errInIdx error - values := make([]interface{}, len(entries)) - for idx, entry := range entries { - values[idx], errInIdx = n.calculateValue(entry.Value, tail) - if errInIdx != nil { - n.log.Errorf("Error updating index %v in %v", idx, context) - return nil, errInIdx - } - - } - return values, nil -} - -func (n *navigator) readMapSplat(context yaml.MapSlice, tail []string) (interface{}, error) { - var newArray = make([]interface{}, len(context)) - var i = 0 - for _, entry := range context { - if len(tail) > 0 { - val, err := n.recurse(entry.Value, tail[0], tail[1:]) - if err != nil { - return nil, err +func (n *navigator) recurse(value *yaml.Node, head string, tail []string) (*yaml.Node, error) { + switch value.Kind { + case yaml.MappingNode: + n.log.Debug("its a map with %v entries", len(value.Content)/2) + for index, content := range value.Content { + // value.Content is a concatenated array of key, value, + // so keys are in the even indexes, values in odd. + if index%2 == 1 || content.Value != head { + continue } - newArray[i] = val - } else { - newArray[i] = entry.Value + value.Content[index+1] = n.getOrReplace(value.Content[index+1], n.guessKind(tail)) + return n.Get(value.Content[index+1], tail) } - i++ - } - return newArray, nil -} + value.Content = append(value.Content, &yaml.Node{Value: head, Kind: yaml.ScalarNode}) + mapEntryValue := yaml.Node{Kind: n.guessKind(tail)} + value.Content = append(value.Content, &mapEntryValue) + n.log.Debug("adding new node %v", value.Content) + return n.Get(&mapEntryValue, tail) + case yaml.SequenceNode: + n.log.Debug("its a sequence of %v things!", len(value.Content)) + if head == "*" { + var newNode = yaml.Node{Kind: yaml.SequenceNode, Style: value.Style} + newNode.Content = make([]*yaml.Node, len(value.Content)) -func (n *navigator) readArray(array []interface{}, head int64, tail []string) (interface{}, error) { - if head >= int64(len(array)) { - return nil, nil - } + for index, value := range value.Content { + value.Content[index] = n.getOrReplace(value.Content[index], n.guessKind(tail)) + var nestedValue, err = n.Get(value.Content[index], tail) + if err != nil { + return nil, err + } + newNode.Content[index] = nestedValue + } + return &newNode, nil + } else if head == "+" { - value := array[head] - return n.calculateValue(value, tail) -} - -func (n *navigator) readArraySplat(array []interface{}, tail []string) (interface{}, error) { - var newArray = make([]interface{}, len(array)) - for index, value := range array { - val, err := n.calculateValue(value, tail) + var newNode = yaml.Node{Kind: n.guessKind(tail)} + value.Content = append(value.Content, &newNode) + n.log.Debug("appending a new node, %v", value.Content) + return n.Get(&newNode, tail) + } + var index, err = strconv.ParseInt(head, 10, 64) // nolint if err != nil { return nil, err } - newArray[index] = val - } - return newArray, nil -} - -func (n *navigator) calculateValue(value interface{}, tail []string) (interface{}, error) { - if len(tail) > 0 { - return n.recurse(value, tail[0], tail[1:]) - } - return value, nil -} - -func (n *navigator) deleteMap(context interface{}, paths []string) (yaml.MapSlice, error) { - n.log.Debugf("deleteMap for %v for %v\n", paths, context) - - mapSlice := n.getMapSlice(context) - - if len(paths) == 0 { - return mapSlice, nil - } - - var index int - var child yaml.MapItem - for index, child = range mapSlice { - if n.matchesKey(paths[0], child.Key) { - n.log.Debugf("\tMatched [%v] with [%v] at index %v", paths[0], child.Key, index) - var badDelete error - mapSlice, badDelete = n.deleteEntryInMap(mapSlice, child, index, paths) - if badDelete != nil { - return nil, badDelete - } + if index >= int64(len(value.Content)) { + return nil, nil } + value.Content[index] = n.getOrReplace(value.Content[index], n.guessKind(tail)) + return n.Get(value.Content[index], tail) + default: + return nil, nil } - - return mapSlice, nil - } -func (n *navigator) deleteEntryInMap(original yaml.MapSlice, child yaml.MapItem, index int, paths []string) (yaml.MapSlice, error) { - remainingPaths := paths[1:] - - var newSlice yaml.MapSlice - if len(remainingPaths) > 0 { - newChild := yaml.MapItem{Key: child.Key} - var errorDeleting error - newChild.Value, errorDeleting = n.DeleteChildValue(child.Value, remainingPaths) - if errorDeleting != nil { - return nil, errorDeleting - } - - newSlice = make(yaml.MapSlice, len(original)) - for i := range original { - item := original[i] - if i == index { - item = newChild - } - newSlice[i] = item - } - } else { - // Delete item from slice at index - newSlice = append(original[:index], original[index+1:]...) - n.log.Debugf("\tDeleted item index %d from original", index) +func (n *navigator) Update(dataBucket *yaml.Node, remainingPath []string, writeCommand WriteCommand) error { + nodeToUpdate, errorRecursing := n.Get(dataBucket, remainingPath) + if errorRecursing != nil { + return errorRecursing } + // later, support ability to execute other commands - n.log.Debugf("\tReturning original %v\n", original) - return newSlice, nil + changesToApply := writeCommand.Value + + nodeToUpdate.Value = changesToApply.Value + nodeToUpdate.Tag = changesToApply.Tag + nodeToUpdate.Kind = changesToApply.Kind + nodeToUpdate.Style = changesToApply.Style + nodeToUpdate.Content = changesToApply.Content + nodeToUpdate.HeadComment = changesToApply.HeadComment + nodeToUpdate.LineComment = changesToApply.LineComment + nodeToUpdate.FootComment = changesToApply.FootComment + return nil } -func (n *navigator) deleteArraySplat(array []interface{}, tail []string) (interface{}, error) { - n.log.Debugf("deleteArraySplat for %v for %v\n", tail, array) - var newArray = make([]interface{}, len(array)) - for index, value := range array { - val, err := n.DeleteChildValue(value, tail) - if err != nil { - return nil, err - } - newArray[index] = val - } - return newArray, nil -} +// func matchesKey(key string, actual interface{}) bool { +// var actualString = fmt.Sprintf("%v", actual) +// var prefixMatch = strings.TrimSuffix(key, "*") +// if prefixMatch != key { +// return strings.HasPrefix(actualString, prefixMatch) +// } +// return actualString == key +// } -func (n *navigator) deleteArray(array []interface{}, paths []string, index int64) (interface{}, error) { - n.log.Debugf("deleteArray for %v for %v\n", paths, array) +// func entriesInSlice(context yaml.MapSlice, key string) []*yaml.MapItem { +// var matches = make([]*yaml.MapItem, 0) +// for idx := range context { +// var entry = &context[idx] +// if matchesKey(key, entry.Key) { +// matches = append(matches, entry) +// } +// } +// return matches +// } - if index >= int64(len(array)) { - return array, nil - } +// func getMapSlice(context interface{}) yaml.MapSlice { +// var mapSlice yaml.MapSlice +// switch context := context.(type) { +// case yaml.MapSlice: +// mapSlice = context +// default: +// mapSlice = make(yaml.MapSlice, 0) +// } +// return mapSlice +// } - remainingPaths := paths[1:] - if len(remainingPaths) > 0 { - // recurse into the array element at index - var errorDeleting error - array[index], errorDeleting = n.deleteMap(array[index], remainingPaths) - if errorDeleting != nil { - return nil, errorDeleting - } +// func getArray(context interface{}) (array []interface{}, ok bool) { +// switch context := context.(type) { +// case []interface{}: +// array = context +// ok = true +// default: +// array = make([]interface{}, 0) +// ok = false +// } +// return +// } - } else { - // Delete the array element at index - array = append(array[:index], array[index+1:]...) - n.log.Debugf("\tDeleted item index %d from array, leaving %v", index, array) - } +// func writeMap(context interface{}, paths []string, value interface{}) interface{} { +// log.Debugf("writeMap with path %v for %v to set value %v\n", paths, context, value) - n.log.Debugf("\tReturning array: %v\n", array) - return array, nil -} +// mapSlice := getMapSlice(context) + +// if len(paths) == 0 { +// return context +// } + +// children := entriesInSlice(mapSlice, paths[0]) + +// if len(children) == 0 && paths[0] == "*" { +// log.Debugf("\tNo matches, return map as is") +// return context +// } + +// if len(children) == 0 { +// newChild := yaml.MapItem{Key: paths[0]} +// mapSlice = append(mapSlice, newChild) +// children = entriesInSlice(mapSlice, paths[0]) +// log.Debugf("\tAppended child at %v for mapSlice %v\n", paths[0], mapSlice) +// } + +// remainingPaths := paths[1:] +// for _, child := range children { +// child.Value = updatedChildValue(child.Value, remainingPaths, value) +// } +// log.Debugf("\tReturning mapSlice %v\n", mapSlice) +// return mapSlice +// } + +// func updatedChildValue(child interface{}, remainingPaths []string, value interface{}) interface{} { +// if len(remainingPaths) == 0 { +// return value +// } +// log.Debugf("updatedChildValue for child %v with path %v to set value %v", child, remainingPaths, value) +// log.Debugf("type of child is %v", reflect.TypeOf(child)) + +// switch child := child.(type) { +// case nil: +// if remainingPaths[0] == "+" || remainingPaths[0] == "*" { +// return writeArray(child, remainingPaths, value) +// } +// case []interface{}: +// _, nextIndexErr := strconv.ParseInt(remainingPaths[0], 10, 64) +// arrayCommand := nextIndexErr == nil || remainingPaths[0] == "+" || remainingPaths[0] == "*" +// if arrayCommand { +// return writeArray(child, remainingPaths, value) +// } +// } +// return writeMap(child, remainingPaths, value) +// } + +// func writeArray(context interface{}, paths []string, value interface{}) []interface{} { +// log.Debugf("writeArray with path %v for %v to set value %v\n", paths, context, value) +// array, _ := getArray(context) + +// if len(paths) == 0 { +// return array +// } + +// log.Debugf("\tarray %v\n", array) + +// rawIndex := paths[0] +// remainingPaths := paths[1:] +// var index int64 +// // the append array indicator +// if rawIndex == "+" { +// index = int64(len(array)) +// } else if rawIndex == "*" { +// for index, oldChild := range array { +// array[index] = updatedChildValue(oldChild, remainingPaths, value) +// } +// return array +// } else { +// index, _ = strconv.ParseInt(rawIndex, 10, 64) // nolint +// // writeArray is only called by updatedChildValue which handles parsing the +// // index, as such this renders this dead code. +// } + +// for index >= int64(len(array)) { +// array = append(array, nil) +// } +// currentChild := array[index] + +// log.Debugf("\tcurrentChild %v\n", currentChild) + +// array[index] = updatedChildValue(currentChild, remainingPaths, value) +// log.Debugf("\tReturning array %v\n", array) +// return array +// } + +// func readMap(context yaml.MapSlice, head string, tail []string) (interface{}, error) { +// log.Debugf("readingMap %v with key %v\n", context, head) +// if head == "*" { +// return readMapSplat(context, tail) +// } + +// entries := entriesInSlice(context, head) +// if len(entries) == 1 { +// return calculateValue(entries[0].Value, tail) +// } else if len(entries) == 0 { +// return nil, nil +// } +// var errInIdx error +// values := make([]interface{}, len(entries)) +// for idx, entry := range entries { +// values[idx], errInIdx = calculateValue(entry.Value, tail) +// if errInIdx != nil { +// log.Errorf("Error updating index %v in %v", idx, context) +// return nil, errInIdx +// } + +// } +// return values, nil +// } + +// func readMapSplat(context yaml.MapSlice, tail []string) (interface{}, error) { +// var newArray = make([]interface{}, len(context)) +// var i = 0 +// for _, entry := range context { +// if len(tail) > 0 { +// val, err := recurse(entry.Value, tail[0], tail[1:]) +// if err != nil { +// return nil, err +// } +// newArray[i] = val +// } else { +// newArray[i] = entry.Value +// } +// i++ +// } +// return newArray, nil +// } + +// func recurse(value interface{}, head string, tail []string) (interface{}, error) { +// switch value := value.(type) { +// case []interface{}: +// if head == "*" { +// return readArraySplat(value, tail) +// } +// index, err := strconv.ParseInt(head, 10, 64) +// if err != nil { +// return nil, fmt.Errorf("error accessing array: %v", err) +// } +// return readArray(value, index, tail) +// case yaml.MapSlice: +// return readMap(value, head, tail) +// default: +// return nil, nil +// } +// } + +// func readArray(array []interface{}, head int64, tail []string) (interface{}, error) { +// if head >= int64(len(array)) { +// return nil, nil +// } + +// value := array[head] +// return calculateValue(value, tail) +// } + +// func readArraySplat(array []interface{}, tail []string) (interface{}, error) { +// var newArray = make([]interface{}, len(array)) +// for index, value := range array { +// val, err := calculateValue(value, tail) +// if err != nil { +// return nil, err +// } +// newArray[index] = val +// } +// return newArray, nil +// } + +// func calculateValue(value interface{}, tail []string) (interface{}, error) { +// if len(tail) > 0 { +// return recurse(value, tail[0], tail[1:]) +// } +// return value, nil +// } + +// func deleteMap(context interface{}, paths []string) (yaml.MapSlice, error) { +// log.Debugf("deleteMap for %v for %v\n", paths, context) + +// mapSlice := getMapSlice(context) + +// if len(paths) == 0 { +// return mapSlice, nil +// } + +// var index int +// var child yaml.MapItem +// for index, child = range mapSlice { +// if matchesKey(paths[0], child.Key) { +// log.Debugf("\tMatched [%v] with [%v] at index %v", paths[0], child.Key, index) +// var badDelete error +// mapSlice, badDelete = deleteEntryInMap(mapSlice, child, index, paths) +// if badDelete != nil { +// return nil, badDelete +// } +// } +// } + +// return mapSlice, nil + +// } + +// func deleteEntryInMap(original yaml.MapSlice, child yaml.MapItem, index int, paths []string) (yaml.MapSlice, error) { +// remainingPaths := paths[1:] + +// var newSlice yaml.MapSlice +// if len(remainingPaths) > 0 { +// newChild := yaml.MapItem{Key: child.Key} +// var errorDeleting error +// newChild.Value, errorDeleting = deleteChildValue(child.Value, remainingPaths) +// if errorDeleting != nil { +// return nil, errorDeleting +// } + +// newSlice = make(yaml.MapSlice, len(original)) +// for i := range original { +// item := original[i] +// if i == index { +// item = newChild +// } +// newSlice[i] = item +// } +// } else { +// // Delete item from slice at index +// newSlice = append(original[:index], original[index+1:]...) +// log.Debugf("\tDeleted item index %d from original", index) +// } + +// log.Debugf("\tReturning original %v\n", original) +// return newSlice, nil +// } + +// func deleteArraySplat(array []interface{}, tail []string) (interface{}, error) { +// log.Debugf("deleteArraySplat for %v for %v\n", tail, array) +// var newArray = make([]interface{}, len(array)) +// for index, value := range array { +// val, err := deleteChildValue(value, tail) +// if err != nil { +// return nil, err +// } +// newArray[index] = val +// } +// return newArray, nil +// } + +// func deleteArray(array []interface{}, paths []string, index int64) (interface{}, error) { +// log.Debugf("deleteArray for %v for %v\n", paths, array) + +// if index >= int64(len(array)) { +// return array, nil +// } + +// remainingPaths := paths[1:] +// if len(remainingPaths) > 0 { +// // Recurse into the array element at index +// var errorDeleting error +// array[index], errorDeleting = deleteMap(array[index], remainingPaths) +// if errorDeleting != nil { +// return nil, errorDeleting +// } + +// } else { +// // Delete the array element at index +// array = append(array[:index], array[index+1:]...) +// log.Debugf("\tDeleted item index %d from array, leaving %v", index, array) +// } + +// log.Debugf("\tReturning array: %v\n", array) +// return array, nil +// } + +// func deleteChildValue(child interface{}, remainingPaths []string) (interface{}, error) { +// log.Debugf("deleteChildValue for %v for %v\n", remainingPaths, child) +// var head = remainingPaths[0] +// var tail = remainingPaths[1:] +// switch child := child.(type) { +// case yaml.MapSlice: +// return deleteMap(child, remainingPaths) +// case []interface{}: +// if head == "*" { +// return deleteArraySplat(child, tail) +// } +// index, err := strconv.ParseInt(head, 10, 64) +// if err != nil { +// return nil, fmt.Errorf("error accessing array: %v", err) +// } +// return deleteArray(child, remainingPaths, index) +// } +// return child, nil +// } diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 8d330063..a477b8dc 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -1,16 +1,13 @@ package yqlib import ( - mergo "gopkg.in/imdario/mergo.v0" logging "gopkg.in/op/go-logging.v1" + yaml "gopkg.in/yaml.v3" ) type YqLib interface { - ReadPath(dataBucket interface{}, path string) (interface{}, error) - WritePath(dataBucket interface{}, path string, value interface{}) interface{} - PrefixPath(dataBucket interface{}, prefix string) interface{} - DeletePath(dataBucket interface{}, path string) (interface{}, error) - Merge(dst interface{}, src interface{}, overwrite bool, append bool) error + Get(rootNode *yaml.Node, path string) (*yaml.Node, error) + Update(rootNode *yaml.Node, path string, writeCommand WriteCommand) error } type lib struct { @@ -25,44 +22,15 @@ func NewYqLib(l *logging.Logger) YqLib { } } -func (l *lib) ReadPath(dataBucket interface{}, path string) (interface{}, error) { - var paths = l.parser.ParsePath(path) - return l.navigator.ReadChildValue(dataBucket, paths) -} - -func (l *lib) WritePath(dataBucket interface{}, path string, value interface{}) interface{} { - var paths = l.parser.ParsePath(path) - return l.navigator.UpdatedChildValue(dataBucket, paths, value) -} - -func (l *lib) PrefixPath(dataBucket interface{}, prefix string) interface{} { - var paths = l.parser.ParsePath(prefix) - - // Inverse order - for i := len(paths)/2 - 1; i >= 0; i-- { - opp := len(paths) - 1 - i - paths[i], paths[opp] = paths[opp], paths[i] +func (l *lib) Get(rootNode *yaml.Node, path string) (*yaml.Node, error) { + if path == "" { + return rootNode, nil } - - var mapDataBucket = dataBucket - for _, key := range paths { - singlePath := []string{key} - mapDataBucket = l.navigator.UpdatedChildValue(nil, singlePath, mapDataBucket) - } - - return mapDataBucket -} - -func (l *lib) DeletePath(dataBucket interface{}, path string) (interface{}, error) { var paths = l.parser.ParsePath(path) - return l.navigator.DeleteChildValue(dataBucket, paths) + return l.navigator.Get(rootNode, paths) } -func (l *lib) Merge(dst interface{}, src interface{}, overwrite bool, append bool) error { - if overwrite { - return mergo.Merge(dst, src, mergo.WithOverride) - } else if append { - return mergo.Merge(dst, src, mergo.WithAppendSlice) - } - return mergo.Merge(dst, src) +func (l *lib) Update(rootNode *yaml.Node, path string, writeCommand WriteCommand) error { + var paths = l.parser.ParsePath(path) + return l.navigator.Update(rootNode, paths, writeCommand) } diff --git a/yq.go b/yq.go index 345706fa..f3b0c24d 100644 --- a/yq.go +++ b/yq.go @@ -6,20 +6,19 @@ import ( "io" "io/ioutil" "os" - "reflect" "strconv" - "strings" - "github.com/mikefarah/yq/v2/pkg/marshal" - "github.com/mikefarah/yq/v2/pkg/yqlib" + "github.com/mikefarah/yq/v3/pkg/marshal" + "github.com/mikefarah/yq/v3/pkg/yqlib" errors "github.com/pkg/errors" - yaml "github.com/mikefarah/yaml/v2" "github.com/spf13/cobra" logging "gopkg.in/op/go-logging.v1" + yaml "gopkg.in/yaml.v3" ) +var rawOutput = false var trimOutput = true var writeInplace = false var writeScript = "" @@ -45,7 +44,6 @@ func main() { } func newCommandCLI() *cobra.Command { - yaml.DefaultMapType = reflect.TypeOf(yaml.MapSlice{}) var rootCmd = &cobra.Command{ Use: "yq", Short: "yq is a lightweight and portable command-line YAML processor.", @@ -83,10 +81,10 @@ func newCommandCLI() *cobra.Command { rootCmd.AddCommand( createReadCmd(), createWriteCmd(), - createPrefixCmd(), - createDeleteCmd(), - createNewCmd(), - createMergeCmd(), + // createPrefixCmd(), + // createDeleteCmd(), + // createNewCmd(), + // createMergeCmd(), ) rootCmd.SetOutput(os.Stdout) @@ -110,6 +108,7 @@ yq r -- things.yaml --key-starting-with-dashes RunE: readProperty, } cmdRead.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)") + cmdRead.PersistentFlags().BoolVarP(&rawOutput, "raw", "r", false, "raw yaml output - prints out values instead of yaml") cmdRead.PersistentFlags().BoolVarP(&outputToJSON, "tojson", "j", false, "output as json") return cmdRead } @@ -149,104 +148,104 @@ a.b.e: return cmdWrite } -func createPrefixCmd() *cobra.Command { - var cmdWrite = &cobra.Command{ - Use: "prefix [yaml_file] [path]", - Aliases: []string{"p"}, - Short: "yq p [--inplace/-i] [--doc/-d index] sample.yaml a.b.c", - Example: ` -yq prefix things.yaml a.b.c -yq prefix --inplace things.yaml a.b.c -yq prefix --inplace -- things.yaml --key-starting-with-dash -yq p -i things.yaml a.b.c -yq p --doc 2 things.yaml a.b.d -yq p -d2 things.yaml a.b.d - `, - Long: `Prefixes w.r.t to the yaml file at the given path. -Outputs to STDOUT unless the inplace flag is used, in which case the file is updated instead. -`, - RunE: prefixProperty, - } - cmdWrite.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace") - cmdWrite.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)") - return cmdWrite -} +// func createPrefixCmd() *cobra.Command { +// var cmdWrite = &cobra.Command{ +// Use: "prefix [yaml_file] [path]", +// Aliases: []string{"p"}, +// Short: "yq p [--inplace/-i] [--doc/-d index] sample.yaml a.b.c", +// Example: ` +// yq prefix things.yaml a.b.c +// yq prefix --inplace things.yaml a.b.c +// yq prefix --inplace -- things.yaml --key-starting-with-dash +// yq p -i things.yaml a.b.c +// yq p --doc 2 things.yaml a.b.d +// yq p -d2 things.yaml a.b.d +// `, +// Long: `Prefixes w.r.t to the yaml file at the given path. +// Outputs to STDOUT unless the inplace flag is used, in which case the file is updated instead. +// `, +// RunE: prefixProperty, +// } +// cmdWrite.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace") +// cmdWrite.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)") +// return cmdWrite +// } -func createDeleteCmd() *cobra.Command { - var cmdDelete = &cobra.Command{ - Use: "delete [yaml_file] [path]", - Aliases: []string{"d"}, - Short: "yq d [--inplace/-i] [--doc/-d index] sample.yaml a.b.c", - Example: ` -yq delete things.yaml a.b.c -yq delete --inplace things.yaml a.b.c -yq delete --inplace -- things.yaml --key-starting-with-dash -yq d -i things.yaml a.b.c -yq d things.yaml a.b.c - `, - Long: `Deletes the given path from the YAML file. -Outputs to STDOUT unless the inplace flag is used, in which case the file is updated instead. -`, - RunE: deleteProperty, - } - cmdDelete.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace") - cmdDelete.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)") - return cmdDelete -} +// func createDeleteCmd() *cobra.Command { +// var cmdDelete = &cobra.Command{ +// Use: "delete [yaml_file] [path]", +// Aliases: []string{"d"}, +// Short: "yq d [--inplace/-i] [--doc/-d index] sample.yaml a.b.c", +// Example: ` +// yq delete things.yaml a.b.c +// yq delete --inplace things.yaml a.b.c +// yq delete --inplace -- things.yaml --key-starting-with-dash +// yq d -i things.yaml a.b.c +// yq d things.yaml a.b.c +// `, +// Long: `Deletes the given path from the YAML file. +// Outputs to STDOUT unless the inplace flag is used, in which case the file is updated instead. +// `, +// RunE: deleteProperty, +// } +// cmdDelete.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace") +// cmdDelete.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)") +// return cmdDelete +// } -func createNewCmd() *cobra.Command { - var cmdNew = &cobra.Command{ - Use: "new [path] [value]", - Aliases: []string{"n"}, - Short: "yq n [--script/-s script_file] a.b.c newValue", - Example: ` -yq new a.b.c cat -yq n a.b.c cat -yq n -- --key-starting-with-dash cat -yq n --script create_script.yaml - `, - Long: `Creates a new yaml w.r.t the given path and value. -Outputs to STDOUT +// func createNewCmd() *cobra.Command { +// var cmdNew = &cobra.Command{ +// Use: "new [path] [value]", +// Aliases: []string{"n"}, +// Short: "yq n [--script/-s script_file] a.b.c newValue", +// Example: ` +// yq new a.b.c cat +// yq n a.b.c cat +// yq n -- --key-starting-with-dash cat +// yq n --script create_script.yaml +// `, +// Long: `Creates a new yaml w.r.t the given path and value. +// Outputs to STDOUT -Create Scripts: -Note that you can give a create script to perform more sophisticated yaml. This follows the same format as the update script. -`, - RunE: newProperty, - } - cmdNew.PersistentFlags().StringVarP(&writeScript, "script", "s", "", "yaml script for updating yaml") - return cmdNew -} +// Create Scripts: +// Note that you can give a create script to perform more sophisticated yaml. This follows the same format as the update script. +// `, +// RunE: newProperty, +// } +// cmdNew.PersistentFlags().StringVarP(&writeScript, "script", "s", "", "yaml script for updating yaml") +// return cmdNew +// } -func createMergeCmd() *cobra.Command { - var cmdMerge = &cobra.Command{ - Use: "merge [initial_yaml_file] [additional_yaml_file]...", - Aliases: []string{"m"}, - Short: "yq m [--inplace/-i] [--doc/-d index] [--overwrite/-x] [--append/-a] sample.yaml sample2.yaml", - Example: ` -yq merge things.yaml other.yaml -yq merge --inplace things.yaml other.yaml -yq m -i things.yaml other.yaml -yq m --overwrite things.yaml other.yaml -yq m -i -x things.yaml other.yaml -yq m -i -a things.yaml other.yaml - `, - Long: `Updates the yaml file by adding/updating the path(s) and value(s) from additional yaml file(s). -Outputs to STDOUT unless the inplace flag is used, in which case the file is updated instead. +// func createMergeCmd() *cobra.Command { +// var cmdMerge = &cobra.Command{ +// Use: "merge [initial_yaml_file] [additional_yaml_file]...", +// Aliases: []string{"m"}, +// Short: "yq m [--inplace/-i] [--doc/-d index] [--overwrite/-x] [--append/-a] sample.yaml sample2.yaml", +// Example: ` +// yq merge things.yaml other.yaml +// yq merge --inplace things.yaml other.yaml +// yq m -i things.yaml other.yaml +// yq m --overwrite things.yaml other.yaml +// yq m -i -x things.yaml other.yaml +// yq m -i -a things.yaml other.yaml +// `, +// Long: `Updates the yaml file by adding/updating the path(s) and value(s) from additional yaml file(s). +// Outputs to STDOUT unless the inplace flag is used, in which case the file is updated instead. -If overwrite flag is set then existing values will be overwritten using the values from each additional yaml file. -If append flag is set then existing arrays will be merged with the arrays from each additional yaml file. +// If overwrite flag is set then existing values will be overwritten using the values from each additional yaml file. +// If append flag is set then existing arrays will be merged with the arrays from each additional yaml file. -Note that if you set both flags only overwrite will take effect. -`, - RunE: mergeProperties, - } - cmdMerge.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace") - cmdMerge.PersistentFlags().BoolVarP(&overwriteFlag, "overwrite", "x", false, "update the yaml file by overwriting existing values") - cmdMerge.PersistentFlags().BoolVarP(&appendFlag, "append", "a", false, "update the yaml file by appending array values") - cmdMerge.PersistentFlags().BoolVarP(&allowEmptyFlag, "allow-empty", "e", false, "allow empty yaml files") - cmdMerge.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)") - return cmdMerge -} +// Note that if you set both flags only overwrite will take effect. +// `, +// RunE: mergeProperties, +// } +// cmdMerge.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace") +// cmdMerge.PersistentFlags().BoolVarP(&overwriteFlag, "overwrite", "x", false, "update the yaml file by overwriting existing values") +// cmdMerge.PersistentFlags().BoolVarP(&appendFlag, "append", "a", false, "update the yaml file by appending array values") +// cmdMerge.PersistentFlags().BoolVarP(&allowEmptyFlag, "allow-empty", "e", false, "allow empty yaml files") +// cmdMerge.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)") +// return cmdMerge +// } func readProperty(cmd *cobra.Command, args []string) error { var path = "" @@ -261,8 +260,9 @@ func readProperty(cmd *cobra.Command, args []string) error { if errorParsingDocIndex != nil { return errorParsingDocIndex } - var mappedDocs []interface{} - var dataBucket interface{} + + var mappedDocs []*yaml.Node + var dataBucket yaml.Node var currentIndex = 0 var errorReadingStream = readStream(args[0], func(decoder *yaml.Decoder) error { for { @@ -274,21 +274,14 @@ func readProperty(cmd *cobra.Command, args []string) error { } return nil } - log.Debugf("processing %v - requested index %v", currentIndex, docIndexInt) + log.Debugf("processing document %v - requested index %v", currentIndex, docIndexInt) if updateAll || currentIndex == docIndexInt { - log.Debugf("reading %v in index %v", path, currentIndex) - if path == "" { - log.Debug("no path") - log.Debugf("%v", dataBucket) - mappedDocs = append(mappedDocs, dataBucket) - } else { - mappedDoc, errorParsing := lib.ReadPath(dataBucket, path) - log.Debugf("%v", mappedDoc) - if errorParsing != nil { - return errors.Wrapf(errorParsing, "Error reading path in document index %v", currentIndex) - } - mappedDocs = append(mappedDocs, mappedDoc) + log.Debugf("reading %v in document %v", path, currentIndex) + mappedDoc, errorParsing := lib.Get(&dataBucket, path) + if errorParsing != nil { + return errors.Wrapf(errorParsing, "Error reading path in document index %v", currentIndex) } + mappedDocs = append(mappedDocs, mappedDoc) } currentIndex = currentIndex + 1 } @@ -298,56 +291,64 @@ func readProperty(cmd *cobra.Command, args []string) error { return errorReadingStream } - if !updateAll { - dataBucket = mappedDocs[0] - } else { - dataBucket = mappedDocs - } + var encoder = yaml.NewEncoder(cmd.OutOrStdout()) + encoder.SetIndent(2) + var err error - dataStr, err := toString(dataBucket) + if rawOutput { + for _, mappedDoc := range mappedDocs { + if mappedDoc != nil { + cmd.Println(mappedDoc.Value) + } + } + } else if !updateAll { + err = encoder.Encode(mappedDocs[0]) + } else { + err = encoder.Encode(&yaml.Node{Kind: yaml.SequenceNode, Content: mappedDocs}) + } if err != nil { return err } - cmd.Println(dataStr) + encoder.Close() return nil } -func newProperty(cmd *cobra.Command, args []string) error { - updatedData, err := newYaml(args) - if err != nil { - return err - } - dataStr, err := toString(updatedData) - if err != nil { - return err - } - cmd.Println(dataStr) - return nil -} +// func newProperty(cmd *cobra.Command, args []string) error { +// updatedData, err := newYaml(args) +// if err != nil { +// return err +// } +// dataStr, err := toString(updatedData) +// if err != nil { +// return err +// } +// cmd.Println(dataStr) +// return nil +// } -func newYaml(args []string) (interface{}, error) { - var writeCommands, writeCommandsError = readWriteCommands(args, 2, "Must provide ") - if writeCommandsError != nil { - return nil, writeCommandsError - } +// func newYaml(args []string) (interface{}, error) { +// var writeCommands, writeCommandsError = readWriteCommands(args, 2, "Must provide ") +// if writeCommandsError != nil { +// return nil, writeCommandsError +// } - var dataBucket interface{} - var isArray = strings.HasPrefix(writeCommands[0].Key.(string), "[") - if isArray { - dataBucket = make([]interface{}, 0) - } else { - dataBucket = make(yaml.MapSlice, 0) - } +// var dataBucket interface{} +// var isArray = strings.HasPrefix(writeCommands[0].Key.(string), "[") +// if isArray { +// dataBucket = make([]interface{}, 0) +// } else { +// dataBucket = make(yaml.MapSlice, 0) +// } - for _, entry := range writeCommands { - path := entry.Key.(string) - value := entry.Value - log.Debugf("setting %v to %v", path, value) - dataBucket = lib.WritePath(dataBucket, path, value) - } +// for _, entry := range writeCommands { +// path := entry.Key.(string) +// value := entry.Value +// log.Debugf("setting %v to %v", path, value) +// dataBucket = lib.WritePath(dataBucket, path, value) +// } - return dataBucket, nil -} +// return dataBucket, nil +// } func parseDocumentIndex() (bool, int, error) { if docIndex == "*" { @@ -360,11 +361,11 @@ func parseDocumentIndex() (bool, int, error) { return false, int(docIndexInt64), nil } -type updateDataFn func(dataBucket interface{}, currentIndex int) (interface{}, error) +type updateDataFn func(dataBucket *yaml.Node, currentIndex int) error func mapYamlDecoder(updateData updateDataFn, encoder *yaml.Encoder) yamlDecoderFn { return func(decoder *yaml.Decoder) error { - var dataBucket interface{} + var dataBucket yaml.Node var errorReading error var errorWriting error var errorUpdating error @@ -387,12 +388,12 @@ func mapYamlDecoder(updateData updateDataFn, encoder *yaml.Encoder) yamlDecoderF } else if errorReading != nil { return errors.Wrapf(errorReading, "Error reading document at index %v, %v", currentIndex, errorReading) } - dataBucket, errorUpdating = updateData(dataBucket, currentIndex) + errorUpdating = updateData(&dataBucket, currentIndex) if errorUpdating != nil { return errors.Wrapf(errorUpdating, "Error updating document at index %v", currentIndex) } - errorWriting = encoder.Encode(dataBucket) + errorWriting = encoder.Encode(&dataBucket) if errorWriting != nil { return errors.Wrapf(errorWriting, "Error writing document at index %v, %v", currentIndex, errorWriting) @@ -412,43 +413,47 @@ func writeProperty(cmd *cobra.Command, args []string) error { return errorParsingDocIndex } - var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) { + var updateData = func(dataBucket *yaml.Node, currentIndex int) error { if updateAll || currentIndex == docIndexInt { log.Debugf("Updating doc %v", currentIndex) for _, entry := range writeCommands { - path := entry.Key.(string) - value := entry.Value - log.Debugf("setting %v to %v", path, value) - dataBucket = lib.WritePath(dataBucket, path, value) + path := entry.Key + changesToApply := entry.Value + var paths = parsePath(path) + + errorUpdating := updateChild(dataBucket, paths, changesToApply) + if errorUpdating != nil { + return errorUpdating + } } } - return dataBucket, nil + return nil } return readAndUpdate(cmd.OutOrStdout(), args[0], updateData) } -func prefixProperty(cmd *cobra.Command, args []string) error { - if len(args) != 2 { - return errors.New("Must provide ") - } - prefixPath := args[1] +// func prefixProperty(cmd *cobra.Command, args []string) error { +// if len(args) != 2 { +// return errors.New("Must provide ") +// } +// prefixPath := args[1] - var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex() - if errorParsingDocIndex != nil { - return errorParsingDocIndex - } +// var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex() +// if errorParsingDocIndex != nil { +// return errorParsingDocIndex +// } - var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) { +// var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) { - if updateAll || currentIndex == docIndexInt { - log.Debugf("Prefixing %v to doc %v", prefixPath, currentIndex) - var mapDataBucket = lib.PrefixPath(dataBucket, prefixPath) - return mapDataBucket, nil - } - return dataBucket, nil - } - return readAndUpdate(cmd.OutOrStdout(), args[0], updateData) -} +// if updateAll || currentIndex == docIndexInt { +// log.Debugf("Prefixing %v to doc %v", prefixPath, currentIndex) +// var mapDataBucket = lib.PrefixPath(dataBucket, prefixPath) +// return mapDataBucket, nil +// } +// return dataBucket, nil +// } +// return readAndUpdate(cmd.OutOrStdout(), args[0], updateData) +// } func readAndUpdate(stdOut io.Writer, inputFile string, updateData updateDataFn) error { var destination io.Writer @@ -479,90 +484,137 @@ func readAndUpdate(stdOut io.Writer, inputFile string, updateData updateDataFn) defer safelyFlush(writer) } var encoder = yaml.NewEncoder(destination) + encoder.SetIndent(2) log.Debugf("Writing to %v from %v", destinationName, inputFile) return readStream(inputFile, mapYamlDecoder(updateData, encoder)) } -func deleteProperty(cmd *cobra.Command, args []string) error { - if len(args) < 2 { - return errors.New("Must provide ") - } - var deletePath = args[1] - var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex() - if errorParsingDocIndex != nil { - return errorParsingDocIndex - } +// func deleteProperty(cmd *cobra.Command, args []string) error { +// if len(args) < 2 { +// return errors.New("Must provide ") +// } +// var deletePath = args[1] +// var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex() +// if errorParsingDocIndex != nil { +// return errorParsingDocIndex +// } - var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) { - if updateAll || currentIndex == docIndexInt { - log.Debugf("Deleting path in doc %v", currentIndex) - return lib.DeletePath(dataBucket, deletePath) - } - return dataBucket, nil - } +// var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) { +// if updateAll || currentIndex == docIndexInt { +// log.Debugf("Deleting path in doc %v", currentIndex) +// return lib.DeletePath(dataBucket, deletePath) +// } +// return dataBucket, nil +// } - return readAndUpdate(cmd.OutOrStdout(), args[0], updateData) +// return readAndUpdate(cmd.OutOrStdout(), args[0], updateData) +// } + +// func mergeProperties(cmd *cobra.Command, args []string) error { +// if len(args) < 2 { +// return errors.New("Must provide at least 2 yaml files") +// } +// var input = args[0] +// var filesToMerge = args[1:] +// var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex() +// if errorParsingDocIndex != nil { +// return errorParsingDocIndex +// } + +// var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) { +// if updateAll || currentIndex == docIndexInt { +// log.Debugf("Merging doc %v", currentIndex) +// var mergedData map[interface{}]interface{} +// // merge only works for maps, so put everything in a temporary +// // map +// var mapDataBucket = make(map[interface{}]interface{}) +// mapDataBucket["root"] = dataBucket +// if err := lib.Merge(&mergedData, mapDataBucket, overwriteFlag, appendFlag); err != nil { +// return nil, err +// } +// for _, f := range filesToMerge { +// var fileToMerge interface{} +// if err := readData(f, 0, &fileToMerge); err != nil { +// if allowEmptyFlag && err == io.EOF { +// continue +// } +// return nil, err +// } +// mapDataBucket["root"] = fileToMerge +// if err := lib.Merge(&mergedData, mapDataBucket, overwriteFlag, appendFlag); err != nil { +// return nil, err +// } +// } +// return mergedData["root"], nil +// } +// return dataBucket, nil +// } +// return readAndUpdate(cmd.OutOrStdout(), input, updateData) +// } + +type rawWriteCommand struct { + // Command string TODO + Key string + Value yaml.Node } -func mergeProperties(cmd *cobra.Command, args []string) error { - if len(args) < 2 { - return errors.New("Must provide at least 2 yaml files") - } - var input = args[0] - var filesToMerge = args[1:] - var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex() - if errorParsingDocIndex != nil { - return errorParsingDocIndex - } - - var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) { - if updateAll || currentIndex == docIndexInt { - log.Debugf("Merging doc %v", currentIndex) - var mergedData map[interface{}]interface{} - // merge only works for maps, so put everything in a temporary - // map - var mapDataBucket = make(map[interface{}]interface{}) - mapDataBucket["root"] = dataBucket - if err := lib.Merge(&mergedData, mapDataBucket, overwriteFlag, appendFlag); err != nil { - return nil, err - } - for _, f := range filesToMerge { - var fileToMerge interface{} - if err := readData(f, 0, &fileToMerge); err != nil { - if allowEmptyFlag && err == io.EOF { - continue - } - return nil, err - } - mapDataBucket["root"] = fileToMerge - if err := lib.Merge(&mergedData, mapDataBucket, overwriteFlag, appendFlag); err != nil { - return nil, err - } - } - return mergedData["root"], nil - } - return dataBucket, nil - } - yaml.DefaultMapType = reflect.TypeOf(map[interface{}]interface{}{}) - defer func() { yaml.DefaultMapType = reflect.TypeOf(yaml.MapSlice{}) }() - return readAndUpdate(cmd.OutOrStdout(), input, updateData) -} - -func readWriteCommands(args []string, expectedArgs int, badArgsMessage string) (yaml.MapSlice, error) { - var writeCommands yaml.MapSlice +func readWriteCommands(args []string, expectedArgs int, badArgsMessage string) ([]rawWriteCommand, error) { + var writeCommands []rawWriteCommand if writeScript != "" { - if err := readData(writeScript, 0, &writeCommands); err != nil { + var rawCommands yaml.Node + if err := readData(writeScript, 0, &rawCommands); err != nil { return nil, err } + log.Debugf("Read write commands file '%v'", rawCommands) + var key string + for index, content := range rawCommands.Content[0].Content { + if index%2 == 0 { // must be the key + key = content.Value + } else { // its the value + writeCommands = append(writeCommands, rawWriteCommand{Key: key, Value: *content}) + } + } + log.Debugf("Read write commands '%v'", writeCommands) } else if len(args) < expectedArgs { return nil, errors.New(badArgsMessage) } else { - writeCommands = make(yaml.MapSlice, 1) - writeCommands[0] = yaml.MapItem{Key: args[expectedArgs-2], Value: valueParser.ParseValue(args[expectedArgs-1])} + writeCommands = make([]rawWriteCommand, 1) + writeCommands[0] = rawWriteCommand{Key: args[expectedArgs-2], Value: parseValue(args[expectedArgs-1])} } return writeCommands, nil } +func parseValue(argument string) yaml.Node { + var err interface{} + var tag = customTag + + var inQuotes = len(argument) > 0 && argument[0] == '"' + if tag == "" && !inQuotes { + + _, err = strconv.ParseBool(argument) + if err == nil { + tag = "!!bool" + } + _, err = strconv.ParseFloat(argument, 64) + if err == nil { + tag = "!!float" + } + _, err = strconv.ParseInt(argument, 10, 64) + if err == nil { + tag = "!!int" + } + + if argument == "null" { + tag = "!!null" + } + if argument == "[]" { + return yaml.Node{Tag: "!!seq", Kind: yaml.SequenceNode} + } + } + log.Debugf("Updating node to value '%v', tag: '%v'", argument, tag) + return yaml.Node{Value: argument, Tag: tag, Kind: yaml.ScalarNode} +} + func toString(context interface{}) (string, error) { if outputToJSON { return jsonConverter.JsonToString(context) @@ -572,7 +624,7 @@ func toString(context interface{}) (string, error) { func safelyRenameFile(from string, to string) { if renameError := os.Rename(from, to); renameError != nil { - log.Debugf("Error renaming from %v to %v, attemting to copy contents", from, to) + log.Debugf("Error renaming from %v to %v, attempting to copy contents", from, to) log.Debug(renameError.Error()) // can't do this rename when running in docker to a file targeted in a mounted volume, // so gracefully degrade to copying the entire contents.