Move implementation files to yqlib and test packages to allow for imports:

- Move data_navigator, json_converter, merge, and path_parser to pkg/yqlib
- Extract yamlToString from yq to pkg/yqlib/yaml_converter
- Move utils_test to test/utils
This commit is contained in:
Conor Nosal 2019-11-22 22:52:29 -05:00 committed by Mike Farah
parent ceafed30f9
commit 26a09e6ec0
15 changed files with 641 additions and 452 deletions

File diff suppressed because it is too large Load Diff

View File

@ -1,46 +0,0 @@
package main
import (
"testing"
)
func TestJsonToString(t *testing.T) {
var data = parseData(`
---
b:
c: 2
`)
got, _ := jsonToString(data)
assertResult(t, "{\"b\":{\"c\":2}}", got)
}
func TestJsonToString_withIntKey(t *testing.T) {
var data = parseData(`
---
b:
2: c
`)
got, _ := jsonToString(data)
assertResult(t, `{"b":{"2":"c"}}`, got)
}
func TestJsonToString_withBoolKey(t *testing.T) {
var data = parseData(`
---
b:
false: c
`)
got, _ := jsonToString(data)
assertResult(t, `{"b":{"false":"c"}}`, got)
}
func TestJsonToString_withArray(t *testing.T) {
var data = parseData(`
---
b:
- item: one
- item: two
`)
got, _ := jsonToString(data)
assertResult(t, "{\"b\":[{\"item\":\"one\"},{\"item\":\"two\"}]}", got)
}

View File

@ -1,12 +0,0 @@
package main
import mergo "gopkg.in/imdario/mergo.v0"
func merge(dst interface{}, src interface{}, overwrite bool, append bool) error {
if overwrite {
return mergo.Merge(dst, src, mergo.WithOverride)
} else if append {
return mergo.Merge(dst, src, mergo.WithAppendSlice)
}
return mergo.Merge(dst, src)
}

View File

@ -1,4 +1,4 @@
package main
package yqlib
import (
"fmt"
@ -77,17 +77,17 @@ func writeMap(context interface{}, paths []string, value interface{}) interface{
remainingPaths := paths[1:]
for _, child := range children {
child.Value = updatedChildValue(child.Value, remainingPaths, value)
child.Value = UpdatedChildValue(child.Value, remainingPaths, value)
}
log.Debugf("\tReturning mapSlice %v\n", mapSlice)
return mapSlice
}
func updatedChildValue(child interface{}, remainingPaths []string, value interface{}) interface{} {
func UpdatedChildValue(child interface{}, remainingPaths []string, value interface{}) interface{} {
if len(remainingPaths) == 0 {
return value
}
log.Debugf("updatedChildValue for child %v with path %v to set value %v", child, remainingPaths, value)
log.Debugf("UpdatedChildValue for child %v with path %v to set value %v", child, remainingPaths, value)
log.Debugf("type of child is %v", reflect.TypeOf(child))
switch child := child.(type) {
@ -123,12 +123,12 @@ func writeArray(context interface{}, paths []string, value interface{}) []interf
index = int64(len(array))
} else if rawIndex == "*" {
for index, oldChild := range array {
array[index] = updatedChildValue(oldChild, remainingPaths, value)
array[index] = UpdatedChildValue(oldChild, remainingPaths, value)
}
return array
} else {
index, _ = strconv.ParseInt(rawIndex, 10, 64) // nolint
// writeArray is only called by updatedChildValue which handles parsing the
// writeArray is only called by UpdatedChildValue which handles parsing the
// index, as such this renders this dead code.
}
@ -139,7 +139,7 @@ func writeArray(context interface{}, paths []string, value interface{}) []interf
log.Debugf("\tcurrentChild %v\n", currentChild)
array[index] = updatedChildValue(currentChild, remainingPaths, value)
array[index] = UpdatedChildValue(currentChild, remainingPaths, value)
log.Debugf("\tReturning array %v\n", array)
return array
}
@ -174,7 +174,7 @@ func readMapSplat(context yaml.MapSlice, tail []string) (interface{}, error) {
var i = 0
for _, entry := range context {
if len(tail) > 0 {
val, err := recurse(entry.Value, tail[0], tail[1:])
val, err := Recurse(entry.Value, tail[0], tail[1:])
if err != nil {
return nil, err
}
@ -187,7 +187,7 @@ func readMapSplat(context yaml.MapSlice, tail []string) (interface{}, error) {
return newArray, nil
}
func recurse(value interface{}, head string, tail []string) (interface{}, error) {
func Recurse(value interface{}, head string, tail []string) (interface{}, error) {
switch value := value.(type) {
case []interface{}:
if head == "*" {
@ -228,7 +228,7 @@ func readArraySplat(array []interface{}, tail []string) (interface{}, error) {
func calculateValue(value interface{}, tail []string) (interface{}, error) {
if len(tail) > 0 {
return recurse(value, tail[0], tail[1:])
return Recurse(value, tail[0], tail[1:])
}
return value, nil
}
@ -266,7 +266,7 @@ func deleteEntryInMap(original yaml.MapSlice, child yaml.MapItem, index int, pat
if len(remainingPaths) > 0 {
newChild := yaml.MapItem{Key: child.Key}
var errorDeleting error
newChild.Value, errorDeleting = deleteChildValue(child.Value, remainingPaths)
newChild.Value, errorDeleting = DeleteChildValue(child.Value, remainingPaths)
if errorDeleting != nil {
return nil, errorDeleting
}
@ -293,7 +293,7 @@ func deleteArraySplat(array []interface{}, tail []string) (interface{}, error) {
log.Debugf("deleteArraySplat for %v for %v\n", tail, array)
var newArray = make([]interface{}, len(array))
for index, value := range array {
val, err := deleteChildValue(value, tail)
val, err := DeleteChildValue(value, tail)
if err != nil {
return nil, err
}
@ -328,8 +328,8 @@ func deleteArray(array []interface{}, paths []string, index int64) (interface{},
return array, nil
}
func deleteChildValue(child interface{}, remainingPaths []string) (interface{}, error) {
log.Debugf("deleteChildValue for %v for %v\n", remainingPaths, child)
func DeleteChildValue(child interface{}, remainingPaths []string) (interface{}, error) {
log.Debugf("DeleteChildValue for %v for %v\n", remainingPaths, child)
var head = remainingPaths[0]
var tail = remainingPaths[1:]
switch child := child.(type) {

View File

@ -1,32 +1,33 @@
package main
package yqlib
import (
"fmt"
"sort"
"testing"
"github.com/mikefarah/yq/test"
)
func TestReadMap_simple(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
---
b:
c: 2
`)
got, _ := readMap(data, "b", []string{"c"})
assertResult(t, 2, got)
test.AssertResult(t, 2, got)
}
func TestReadMap_numberKey(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
---
200: things
`)
got, _ := readMap(data, "200", []string{})
assertResult(t, "things", got)
test.AssertResult(t, "things", got)
}
func TestReadMap_splat(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
---
mapSplat:
item1: things
@ -34,11 +35,11 @@ mapSplat:
otherThing: cat
`)
res, _ := readMap(data, "mapSplat", []string{"*"})
assertResult(t, "[things whatever cat]", fmt.Sprintf("%v", res))
test.AssertResult(t, "[things whatever cat]", fmt.Sprintf("%v", res))
}
func TestReadMap_prefixSplat(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
---
mapSplat:
item1: things
@ -46,11 +47,11 @@ mapSplat:
otherThing: cat
`)
res, _ := readMap(data, "mapSplat", []string{"item*"})
assertResult(t, "[things whatever]", fmt.Sprintf("%v", res))
test.AssertResult(t, "[things whatever]", fmt.Sprintf("%v", res))
}
func TestReadMap_deep_splat(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
---
mapSplatDeep:
item1:
@ -63,30 +64,30 @@ mapSplatDeep:
result := res.([]interface{})
var actual = []string{result[0].(string), result[1].(string)}
sort.Strings(actual)
assertResult(t, "[apples bananas]", fmt.Sprintf("%v", actual))
test.AssertResult(t, "[apples bananas]", fmt.Sprintf("%v", actual))
}
func TestReadMap_key_doesnt_exist(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
---
b:
c: 2
`)
got, _ := readMap(data, "b.x.f", []string{"c"})
assertResult(t, nil, got)
test.AssertResult(t, nil, got)
}
func TestReadMap_recurse_against_string(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
---
a: cat
`)
got, _ := readMap(data, "a", []string{"b"})
assertResult(t, nil, got)
test.AssertResult(t, nil, got)
}
func TestReadMap_with_array(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
---
b:
d:
@ -94,11 +95,11 @@ b:
- 4
`)
got, _ := readMap(data, "b", []string{"d", "1"})
assertResult(t, 4, got)
test.AssertResult(t, 4, got)
}
func TestReadMap_with_array_and_bad_index(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
---
b:
d:
@ -110,11 +111,11 @@ b:
t.Fatal("Expected error due to invalid path")
}
expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
assertResult(t, expectedOutput, err.Error())
test.AssertResult(t, expectedOutput, err.Error())
}
func TestReadMap_with_mapsplat_array_and_bad_index(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
---
b:
d:
@ -130,11 +131,11 @@ b:
t.Fatal("Expected error due to invalid path")
}
expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
assertResult(t, expectedOutput, err.Error())
test.AssertResult(t, expectedOutput, err.Error())
}
func TestReadMap_with_arraysplat_map_array_and_bad_index(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
---
b:
d:
@ -150,11 +151,11 @@ b:
t.Fatal("Expected error due to invalid path")
}
expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
assertResult(t, expectedOutput, err.Error())
test.AssertResult(t, expectedOutput, err.Error())
}
func TestReadMap_with_array_out_of_bounds(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
---
b:
d:
@ -162,11 +163,11 @@ b:
- 4
`)
got, _ := readMap(data, "b", []string{"d", "3"})
assertResult(t, nil, got)
test.AssertResult(t, nil, got)
}
func TestReadMap_with_array_out_of_bounds_by_1(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
---
b:
d:
@ -174,11 +175,11 @@ b:
- 4
`)
got, _ := readMap(data, "b", []string{"d", "2"})
assertResult(t, nil, got)
test.AssertResult(t, nil, got)
}
func TestReadMap_with_array_splat(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
e:
-
name: Fred
@ -188,71 +189,71 @@ e:
thing: dog
`)
got, _ := readMap(data, "e", []string{"*", "name"})
assertResult(t, "[Fred Sam]", fmt.Sprintf("%v", got))
test.AssertResult(t, "[Fred Sam]", fmt.Sprintf("%v", got))
}
func TestWrite_really_simple(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
b: 2
`)
updated := writeMap(data, []string{"b"}, "4")
assertResult(t, "[{b 4}]", fmt.Sprintf("%v", updated))
test.AssertResult(t, "[{b 4}]", fmt.Sprintf("%v", updated))
}
func TestWrite_simple(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
b:
c: 2
`)
updated := writeMap(data, []string{"b", "c"}, "4")
assertResult(t, "[{b [{c 4}]}]", fmt.Sprintf("%v", updated))
test.AssertResult(t, "[{b [{c 4}]}]", fmt.Sprintf("%v", updated))
}
func TestWrite_new(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
b:
c: 2
`)
updated := writeMap(data, []string{"b", "d"}, "4")
assertResult(t, "[{b [{c 2} {d 4}]}]", fmt.Sprintf("%v", updated))
test.AssertResult(t, "[{b [{c 2} {d 4}]}]", fmt.Sprintf("%v", updated))
}
func TestWrite_new_deep(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
b:
c: 2
`)
updated := writeMap(data, []string{"b", "d", "f"}, "4")
assertResult(t, "[{b [{c 2} {d [{f 4}]}]}]", fmt.Sprintf("%v", updated))
test.AssertResult(t, "[{b [{c 2} {d [{f 4}]}]}]", fmt.Sprintf("%v", updated))
}
func TestWrite_array(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
b:
- aa
`)
updated := writeMap(data, []string{"b", "0"}, "bb")
assertResult(t, "[{b [bb]}]", fmt.Sprintf("%v", updated))
test.AssertResult(t, "[{b [bb]}]", fmt.Sprintf("%v", updated))
}
func TestWrite_new_array(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
b:
c: 2
`)
updated := writeMap(data, []string{"b", "0"}, "4")
assertResult(t, "[{b [{c 2} {0 4}]}]", fmt.Sprintf("%v", updated))
test.AssertResult(t, "[{b [{c 2} {0 4}]}]", fmt.Sprintf("%v", updated))
}
func TestWrite_new_array_deep(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
a: apple
`)
@ -261,12 +262,12 @@ b:
- c: "4"`
updated := writeMap(data, []string{"b", "+", "c"}, "4")
got, _ := yamlToString(updated)
assertResult(t, expected, got)
got, _ := YamlToString(updated, true)
test.AssertResult(t, expected, got)
}
func TestWrite_new_map_array_deep(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
b:
c: 2
`)
@ -276,12 +277,12 @@ b:
- "4"`
updated := writeMap(data, []string{"b", "d", "+"}, "4")
got, _ := yamlToString(updated)
assertResult(t, expected, got)
got, _ := YamlToString(updated, true)
test.AssertResult(t, expected, got)
}
func TestWrite_add_to_array(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
b:
- aa
`)
@ -291,109 +292,109 @@ b:
- bb`
updated := writeMap(data, []string{"b", "1"}, "bb")
got, _ := yamlToString(updated)
assertResult(t, expected, got)
got, _ := YamlToString(updated, true)
test.AssertResult(t, expected, got)
}
func TestWrite_with_no_tail(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
b:
c: 2
`)
updated := writeMap(data, []string{"b"}, "4")
assertResult(t, "[{b 4}]", fmt.Sprintf("%v", updated))
test.AssertResult(t, "[{b 4}]", fmt.Sprintf("%v", updated))
}
func TestWriteMap_no_paths(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
b: 5
`)
result := writeMap(data, []string{}, 4)
assertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
}
func TestWriteArray_no_paths(t *testing.T) {
var data = make([]interface{}, 1)
data[0] = "mike"
result := writeArray(data, []string{}, 4)
assertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
}
func TestDelete_MapItem(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
a: 123
b: 456
`)
var expected = parseData(`
var expected = test.ParseData(`
b: 456
`)
result, _ := deleteMap(data, []string{"a"})
assertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result))
test.AssertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result))
}
// Ensure deleting an index into a string does nothing
func TestDelete_index_to_string(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
a: mystring
`)
result, _ := deleteMap(data, []string{"a", "0"})
assertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
}
func TestDelete_list_index(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
a: [3, 4]
`)
var expected = parseData(`
var expected = test.ParseData(`
a: [3]
`)
result, _ := deleteMap(data, []string{"a", "1"})
assertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result))
test.AssertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result))
}
func TestDelete_list_index_beyond_bounds(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
a: [3, 4]
`)
result, _ := deleteMap(data, []string{"a", "5"})
assertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
}
func TestDelete_list_index_out_of_bounds_by_1(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
a: [3, 4]
`)
result, _ := deleteMap(data, []string{"a", "2"})
assertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
}
func TestDelete_no_paths(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
a: [3, 4]
b:
- name: test
`)
result, _ := deleteMap(data, []string{})
assertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
}
func TestDelete_array_map_item(t *testing.T) {
var data = parseData(`
var data = test.ParseData(`
b:
- name: fred
value: blah
- name: john
value: test
`)
var expected = parseData(`
var expected = test.ParseData(`
b:
- value: blah
- name: john
value: test
`)
result, _ := deleteMap(data, []string{"b", "0", "name"})
assertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result))
test.AssertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result))
}

View File

@ -1,4 +1,4 @@
package main
package yqlib
import (
"encoding/json"
@ -8,7 +8,7 @@ import (
yaml "github.com/mikefarah/yaml/v2"
)
func jsonToString(context interface{}) (string, error) {
func JsonToString(context interface{}) (string, error) {
out, err := json.Marshal(toJSON(context))
if err != nil {
return "", fmt.Errorf("error printing yaml as json: %v", err)

View File

@ -0,0 +1,47 @@
package yqlib
import (
"testing"
"github.com/mikefarah/yq/test"
)
func TestJsonToString(t *testing.T) {
var data = test.ParseData(`
---
b:
c: 2
`)
got, _ := JsonToString(data)
test.AssertResult(t, "{\"b\":{\"c\":2}}", got)
}
func TestJsonToString_withIntKey(t *testing.T) {
var data = test.ParseData(`
---
b:
2: c
`)
got, _ := JsonToString(data)
test.AssertResult(t, `{"b":{"2":"c"}}`, got)
}
func TestJsonToString_withBoolKey(t *testing.T) {
var data = test.ParseData(`
---
b:
false: c
`)
got, _ := JsonToString(data)
test.AssertResult(t, `{"b":{"false":"c"}}`, got)
}
func TestJsonToString_withArray(t *testing.T) {
var data = test.ParseData(`
---
b:
- item: one
- item: two
`)
got, _ := JsonToString(data)
test.AssertResult(t, "{\"b\":[{\"item\":\"one\"},{\"item\":\"two\"}]}", got)
}

54
pkg/yqlib/lib.go Normal file
View File

@ -0,0 +1,54 @@
package yqlib
import (
mergo "gopkg.in/imdario/mergo.v0"
logging "gopkg.in/op/go-logging.v1"
)
var log = logging.MustGetLogger("yq")
func SetLogger(l *logging.Logger) {
log = l
}
func ReadPath(dataBucket interface{}, path string) (interface{}, error) {
var paths = ParsePath(path)
return Recurse(dataBucket, paths[0], paths[1:])
}
func WritePath(dataBucket interface{}, path string, value interface{}) (interface{}) {
var paths = ParsePath(path)
return UpdatedChildValue(dataBucket, paths, value)
}
func PrefixPath(dataBucket interface{}, prefix string) (interface{}) {
var paths = ParsePath(prefix)
// Inverse order
for i := len(paths)/2 - 1; i >= 0; i-- {
opp := len(paths) - 1 - i
paths[i], paths[opp] = paths[opp], paths[i]
}
var mapDataBucket = dataBucket
for _, key := range paths {
singlePath := []string{key}
mapDataBucket = UpdatedChildValue(nil, singlePath, mapDataBucket)
}
return mapDataBucket
}
func DeletePath(dataBucket interface{}, path string) (interface{}, error) {
var paths = ParsePath(path)
return DeleteChildValue(dataBucket, paths)
}
func Merge(dst interface{}, src interface{}, overwrite bool, append bool) error {
if overwrite {
return mergo.Merge(dst, src, mergo.WithOverride)
} else if append {
return mergo.Merge(dst, src, mergo.WithAppendSlice)
}
return mergo.Merge(dst, src)
}

148
pkg/yqlib/lib_test.go Normal file
View File

@ -0,0 +1,148 @@
package yqlib
import (
"fmt"
"testing"
"github.com/mikefarah/yq/test"
)
func TestReadPath(t *testing.T) {
var data = test.ParseData(`
---
b:
2: c
`)
got, _ := ReadPath(data, "b.2")
test.AssertResult(t, `c`, got)
}
func TestReadPath_WithError(t *testing.T) {
var data = test.ParseData(`
---
b:
- c
`)
_, err := ReadPath(data, "b.[a]")
if err == nil {
t.Fatal("Expected error due to invalid path")
}
}
func TestWritePath(t *testing.T) {
var data = test.ParseData(`
---
b:
2: c
`)
got := WritePath(data, "b.3", "a")
test.AssertResult(t, `[{b [{2 c} {3 a}]}]`, fmt.Sprintf("%v", got))
}
func TestPrefixPath(t *testing.T) {
var data = test.ParseData(`
---
b:
2: c
`)
got := PrefixPath(data, "d")
test.AssertResult(t, `[{d [{b [{2 c}]}]}]`, fmt.Sprintf("%v", got))
}
func TestDeletePath(t *testing.T) {
var data = test.ParseData(`
---
b:
2: c
3: a
`)
got, _ := DeletePath(data, "b.2")
test.AssertResult(t, `[{b [{3 a}]}]`, fmt.Sprintf("%v", got))
}
func TestDeletePath_WithError(t *testing.T) {
var data = test.ParseData(`
---
b:
- c
`)
_, err := DeletePath(data, "b.[a]")
if err == nil {
t.Fatal("Expected error due to invalid path")
}
}
func TestMerge(t *testing.T) {
var dst = test.ParseData(`
---
a: b
c: d
`)
var src = test.ParseData(`
---
a: 1
b: 2
`)
var mergedData = make(map[interface{}]interface{})
mergedData["root"] = dst
var mapDataBucket = make(map[interface{}]interface{})
mapDataBucket["root"] = src
Merge(&mergedData, mapDataBucket, false, false)
test.AssertResult(t, `[{a b} {c d}]`, fmt.Sprintf("%v", mergedData["root"]))
}
func TestMerge_WithOverwrite(t *testing.T) {
var dst = test.ParseData(`
---
a: b
c: d
`)
var src = test.ParseData(`
---
a: 1
b: 2
`)
var mergedData = make(map[interface{}]interface{})
mergedData["root"] = dst
var mapDataBucket = make(map[interface{}]interface{})
mapDataBucket["root"] = src
Merge(&mergedData, mapDataBucket, true, false)
test.AssertResult(t, `[{a 1} {b 2}]`, fmt.Sprintf("%v", mergedData["root"]))
}
func TestMerge_WithAppend(t *testing.T) {
var dst = test.ParseData(`
---
a: b
c: d
`)
var src = test.ParseData(`
---
a: 1
b: 2
`)
var mergedData = make(map[interface{}]interface{})
mergedData["root"] = dst
var mapDataBucket = make(map[interface{}]interface{})
mapDataBucket["root"] = src
Merge(&mergedData, mapDataBucket, false, true)
test.AssertResult(t, `[{a b} {c d} {a 1} {b 2}]`, fmt.Sprintf("%v", mergedData["root"]))
}
func TestMerge_WithError(t *testing.T) {
err := Merge(nil, nil, false, false)
if err == nil {
t.Fatal("Expected error due to nil")
}
}

View File

@ -1,6 +1,6 @@
package main
package yqlib
func parsePath(path string) []string {
func ParsePath(path string) []string {
return parsePathAccum([]string{}, path)
}

View File

@ -1,7 +1,8 @@
package main
package yqlib
import (
"testing"
"github.com/mikefarah/yq/test"
)
var parsePathsTests = []struct {
@ -15,7 +16,7 @@ var parsePathsTests = []struct {
func TestParsePath(t *testing.T) {
for _, tt := range parsePathsTests {
assertResultComplex(t, tt.expectedPaths, parsePath(tt.path))
test.AssertResultComplex(t, tt.expectedPaths, ParsePath(tt.path))
}
}
@ -37,7 +38,7 @@ var nextYamlPathTests = []struct {
func TestNextYamlPath(t *testing.T) {
for _, tt := range nextYamlPathTests {
var element, remaining = nextYamlPath(tt.path)
assertResultWithContext(t, tt.expectedElement, element, tt)
assertResultWithContext(t, tt.expectedRemaining, remaining, tt)
test.AssertResultWithContext(t, tt.expectedElement, element, tt)
test.AssertResultWithContext(t, tt.expectedRemaining, remaining, tt)
}
}

View File

@ -0,0 +1,32 @@
package yqlib
import (
yaml "github.com/mikefarah/yaml/v2"
errors "github.com/pkg/errors"
"strings"
)
func YamlToString(context interface{}, trimOutput bool) (string, error) {
switch context := context.(type) {
case string:
return context, nil
default:
return marshalContext(context, trimOutput)
}
}
func marshalContext(context interface{}, trimOutput bool) (string, error) {
out, err := yaml.Marshal(context)
if err != nil {
return "", errors.Wrap(err, "error printing yaml")
}
outStr := string(out)
// trim the trailing new line as it's easier for a script to add
// it in if required than to remove it
if trimOutput {
return strings.Trim(outStr, "\n "), nil
}
return outStr, nil
}

View File

@ -1,4 +1,4 @@
package main
package test
import (
"bytes"
@ -19,7 +19,7 @@ type resulter struct {
Command *cobra.Command
}
func runCmd(c *cobra.Command, input string) resulter {
func RunCmd(c *cobra.Command, input string) resulter {
buf := new(bytes.Buffer)
c.SetOutput(buf)
c.SetArgs(strings.Split(input, " "))
@ -30,7 +30,7 @@ func runCmd(c *cobra.Command, input string) resulter {
return resulter{err, output, c}
}
func parseData(rawData string) yaml.MapSlice {
func ParseData(rawData string) yaml.MapSlice {
var parsedData yaml.MapSlice
err := yaml.Unmarshal([]byte(rawData), &parsedData)
if err != nil {
@ -40,21 +40,21 @@ func parseData(rawData string) yaml.MapSlice {
return parsedData
}
func assertResult(t *testing.T, expectedValue interface{}, actualValue interface{}) {
func AssertResult(t *testing.T, expectedValue interface{}, actualValue interface{}) {
t.Helper()
if expectedValue != actualValue {
t.Error("Expected <", expectedValue, "> but got <", actualValue, ">", fmt.Sprintf("%T", actualValue))
}
}
func assertResultComplex(t *testing.T, expectedValue interface{}, actualValue interface{}) {
func AssertResultComplex(t *testing.T, expectedValue interface{}, actualValue interface{}) {
t.Helper()
if !reflect.DeepEqual(expectedValue, actualValue) {
t.Error("Expected <", expectedValue, "> but got <", actualValue, ">", fmt.Sprintf("%T", actualValue))
}
}
func assertResultWithContext(t *testing.T, expectedValue interface{}, actualValue interface{}, context interface{}) {
func AssertResultWithContext(t *testing.T, expectedValue interface{}, actualValue interface{}, context interface{}) {
t.Helper()
if expectedValue != actualValue {
t.Error(context)
@ -62,7 +62,7 @@ func assertResultWithContext(t *testing.T, expectedValue interface{}, actualValu
}
}
func writeTempYamlFile(content string) string {
func WriteTempYamlFile(content string) string {
tmpfile, _ := ioutil.TempFile("", "testyaml")
defer func() {
_ = tmpfile.Close()
@ -72,11 +72,11 @@ func writeTempYamlFile(content string) string {
return tmpfile.Name()
}
func readTempYamlFile(name string) string {
func ReadTempYamlFile(name string) string {
content, _ := ioutil.ReadFile(name)
return string(content)
}
func removeTempYamlFile(name string) {
func RemoveTempYamlFile(name string) {
_ = os.Remove(name)
}

79
yq.go
View File

@ -9,6 +9,7 @@ import (
"reflect"
"strconv"
"strings"
"github.com/mikefarah/yq/pkg/yqlib"
errors "github.com/pkg/errors"
@ -38,6 +39,7 @@ func main() {
}
func newCommandCLI() *cobra.Command {
yqlib.SetLogger(log)
yaml.DefaultMapType = reflect.TypeOf(yaml.MapSlice{})
var rootCmd = &cobra.Command{
Use: "yq",
@ -270,13 +272,19 @@ func readProperty(cmd *cobra.Command, args []string) error {
log.Debugf("processing %v - requested index %v", currentIndex, docIndexInt)
if updateAll || currentIndex == docIndexInt {
log.Debugf("reading %v in index %v", path, currentIndex)
mappedDoc, errorParsing := readPath(dataBucket, path)
if path == "" {
log.Debug("no path")
log.Debugf("%v", dataBucket)
mappedDocs = append(mappedDocs, dataBucket)
} else {
mappedDoc, errorParsing := yqlib.ReadPath(dataBucket, path)
log.Debugf("%v", mappedDoc)
if errorParsing != nil {
return errors.Wrapf(errorParsing, "Error reading path in document index %v", currentIndex)
}
mappedDocs = append(mappedDocs, mappedDoc)
}
}
currentIndex = currentIndex + 1
}
})
@ -299,15 +307,6 @@ func readProperty(cmd *cobra.Command, args []string) error {
return nil
}
func readPath(dataBucket interface{}, path string) (interface{}, error) {
if path == "" {
log.Debug("no path")
return dataBucket, nil
}
var paths = parsePath(path)
return recurse(dataBucket, paths[0], paths[1:])
}
func newProperty(cmd *cobra.Command, args []string) error {
updatedData, err := newYaml(args)
if err != nil {
@ -339,8 +338,7 @@ func newYaml(args []string) (interface{}, error) {
path := entry.Key.(string)
value := entry.Value
log.Debugf("setting %v to %v", path, value)
var paths = parsePath(path)
dataBucket = updatedChildValue(dataBucket, paths, value)
dataBucket = yqlib.WritePath(dataBucket, path, value)
}
return dataBucket, nil
@ -416,8 +414,7 @@ func writeProperty(cmd *cobra.Command, args []string) error {
path := entry.Key.(string)
value := entry.Value
log.Debugf("setting %v to %v", path, value)
var paths = parsePath(path)
dataBucket = updatedChildValue(dataBucket, paths, value)
dataBucket = yqlib.WritePath(dataBucket, path, value)
}
}
return dataBucket, nil
@ -429,28 +426,18 @@ func prefixProperty(cmd *cobra.Command, args []string) error {
if len(args) != 2 {
return errors.New("Must provide <filename> <prefixed_path>")
}
prefixPath := args[1]
var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex()
if errorParsingDocIndex != nil {
return errorParsingDocIndex
}
var paths = parsePath(args[1])
// Inverse order
for i := len(paths)/2 - 1; i >= 0; i-- {
opp := len(paths) - 1 - i
paths[i], paths[opp] = paths[opp], paths[i]
}
var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) {
if updateAll || currentIndex == docIndexInt {
log.Debugf("Prefixing %v to doc %v", paths, currentIndex)
var mapDataBucket = dataBucket
for _, key := range paths {
singlePath := []string{key}
mapDataBucket = updatedChildValue(nil, singlePath, mapDataBucket)
}
log.Debugf("Prefixing %v to doc %v", prefixPath, currentIndex)
var mapDataBucket = yqlib.PrefixPath(dataBucket, prefixPath)
return mapDataBucket, nil
}
return dataBucket, nil
@ -496,7 +483,6 @@ func deleteProperty(cmd *cobra.Command, args []string) error {
return errors.New("Must provide <filename> <path_to_delete>")
}
var deletePath = args[1]
var paths = parsePath(deletePath)
var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex()
if errorParsingDocIndex != nil {
return errorParsingDocIndex
@ -505,7 +491,7 @@ func deleteProperty(cmd *cobra.Command, args []string) error {
var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) {
if updateAll || currentIndex == docIndexInt {
log.Debugf("Deleting path in doc %v", currentIndex)
return deleteChildValue(dataBucket, paths)
return yqlib.DeletePath(dataBucket, deletePath)
}
return dataBucket, nil
}
@ -532,7 +518,7 @@ func mergeProperties(cmd *cobra.Command, args []string) error {
// map
var mapDataBucket = make(map[interface{}]interface{})
mapDataBucket["root"] = dataBucket
if err := merge(&mergedData, mapDataBucket, overwriteFlag, appendFlag); err != nil {
if err := yqlib.Merge(&mergedData, mapDataBucket, overwriteFlag, appendFlag); err != nil {
return nil, err
}
for _, f := range filesToMerge {
@ -544,7 +530,7 @@ func mergeProperties(cmd *cobra.Command, args []string) error {
return nil, err
}
mapDataBucket["root"] = fileToMerge
if err := merge(&mergedData, mapDataBucket, overwriteFlag, appendFlag); err != nil {
if err := yqlib.Merge(&mergedData, mapDataBucket, overwriteFlag, appendFlag); err != nil {
return nil, err
}
}
@ -594,34 +580,9 @@ func parseValue(argument string) interface{} {
func toString(context interface{}) (string, error) {
if outputToJSON {
return jsonToString(context)
return yqlib.JsonToString(context)
}
return yamlToString(context)
}
func yamlToString(context interface{}) (string, error) {
switch context := context.(type) {
case string:
return context, nil
default:
return marshalContext(context)
}
}
func marshalContext(context interface{}) (string, error) {
out, err := yaml.Marshal(context)
if err != nil {
return "", errors.Wrap(err, "error printing yaml")
}
outStr := string(out)
// trim the trailing new line as it's easier for a script to add
// it in if required than to remove it
if trimOutput {
return strings.Trim(outStr, "\n "), nil
}
return outStr, nil
return yqlib.YamlToString(context, trimOutput)
}
func safelyRenameFile(from string, to string) {

View File

@ -4,6 +4,8 @@ import (
"fmt"
"runtime"
"testing"
"github.com/mikefarah/yq/test"
"github.com/mikefarah/yq/pkg/yqlib"
)
var parseValueTests = []struct {
@ -20,7 +22,7 @@ var parseValueTests = []struct {
func TestParseValue(t *testing.T) {
for _, tt := range parseValueTests {
assertResultWithContext(t, tt.expectedResult, parseValue(tt.argument), tt.testDescription)
test.AssertResultWithContext(t, tt.expectedResult, parseValue(tt.argument), tt.testDescription)
}
}
@ -28,14 +30,14 @@ func TestMultilineString(t *testing.T) {
testString := `
abcd
efg`
formattedResult, _ := yamlToString(testString)
assertResult(t, testString, formattedResult)
formattedResult, _ := yqlib.YamlToString(testString, false)
test.AssertResult(t, testString, formattedResult)
}
func TestNewYaml(t *testing.T) {
result, _ := newYaml([]string{"b.c", "3"})
formattedResult := fmt.Sprintf("%v", result)
assertResult(t,
test.AssertResult(t,
"[{b [{c 3}]}]",
formattedResult)
}
@ -43,7 +45,7 @@ func TestNewYaml(t *testing.T) {
func TestNewYamlArray(t *testing.T) {
result, _ := newYaml([]string{"[0].cat", "meow"})
formattedResult := fmt.Sprintf("%v", result)
assertResult(t,
test.AssertResult(t,
"[[{cat meow}]]",
formattedResult)
}
@ -55,8 +57,8 @@ func TestNewYaml_WithScript(t *testing.T) {
e:
- name: Mike Farah`
result, _ := newYaml([]string{""})
actualResult, _ := yamlToString(result)
assertResult(t, expectedResult, actualResult)
actualResult, _ := yqlib.YamlToString(result, true)
test.AssertResult(t, expectedResult, actualResult)
}
func TestNewYaml_WithUnknownScript(t *testing.T) {
@ -71,5 +73,5 @@ func TestNewYaml_WithUnknownScript(t *testing.T) {
} else {
expectedOutput = `open fake-unknown: no such file or directory`
}
assertResult(t, expectedOutput, err.Error())
test.AssertResult(t, expectedOutput, err.Error())
}