Move implementation files to yqlib and test packages to allow for imports:

- Move data_navigator, json_converter, merge, and path_parser to pkg/yqlib
- Extract yamlToString from yq to pkg/yqlib/yaml_converter
- Move utils_test to test/utils
This commit is contained in:
Conor Nosal 2019-11-22 22:52:29 -05:00 committed by Mike Farah
parent ceafed30f9
commit 26a09e6ec0
15 changed files with 641 additions and 452 deletions

File diff suppressed because it is too large Load Diff

View File

@ -1,46 +0,0 @@
package main
import (
"testing"
)
func TestJsonToString(t *testing.T) {
var data = parseData(`
---
b:
c: 2
`)
got, _ := jsonToString(data)
assertResult(t, "{\"b\":{\"c\":2}}", got)
}
func TestJsonToString_withIntKey(t *testing.T) {
var data = parseData(`
---
b:
2: c
`)
got, _ := jsonToString(data)
assertResult(t, `{"b":{"2":"c"}}`, got)
}
func TestJsonToString_withBoolKey(t *testing.T) {
var data = parseData(`
---
b:
false: c
`)
got, _ := jsonToString(data)
assertResult(t, `{"b":{"false":"c"}}`, got)
}
func TestJsonToString_withArray(t *testing.T) {
var data = parseData(`
---
b:
- item: one
- item: two
`)
got, _ := jsonToString(data)
assertResult(t, "{\"b\":[{\"item\":\"one\"},{\"item\":\"two\"}]}", got)
}

View File

@ -1,12 +0,0 @@
package main
import mergo "gopkg.in/imdario/mergo.v0"
func merge(dst interface{}, src interface{}, overwrite bool, append bool) error {
if overwrite {
return mergo.Merge(dst, src, mergo.WithOverride)
} else if append {
return mergo.Merge(dst, src, mergo.WithAppendSlice)
}
return mergo.Merge(dst, src)
}

View File

@ -1,4 +1,4 @@
package main package yqlib
import ( import (
"fmt" "fmt"
@ -77,17 +77,17 @@ func writeMap(context interface{}, paths []string, value interface{}) interface{
remainingPaths := paths[1:] remainingPaths := paths[1:]
for _, child := range children { for _, child := range children {
child.Value = updatedChildValue(child.Value, remainingPaths, value) child.Value = UpdatedChildValue(child.Value, remainingPaths, value)
} }
log.Debugf("\tReturning mapSlice %v\n", mapSlice) log.Debugf("\tReturning mapSlice %v\n", mapSlice)
return mapSlice return mapSlice
} }
func updatedChildValue(child interface{}, remainingPaths []string, value interface{}) interface{} { func UpdatedChildValue(child interface{}, remainingPaths []string, value interface{}) interface{} {
if len(remainingPaths) == 0 { if len(remainingPaths) == 0 {
return value return value
} }
log.Debugf("updatedChildValue for child %v with path %v to set value %v", child, remainingPaths, value) log.Debugf("UpdatedChildValue for child %v with path %v to set value %v", child, remainingPaths, value)
log.Debugf("type of child is %v", reflect.TypeOf(child)) log.Debugf("type of child is %v", reflect.TypeOf(child))
switch child := child.(type) { switch child := child.(type) {
@ -123,12 +123,12 @@ func writeArray(context interface{}, paths []string, value interface{}) []interf
index = int64(len(array)) index = int64(len(array))
} else if rawIndex == "*" { } else if rawIndex == "*" {
for index, oldChild := range array { for index, oldChild := range array {
array[index] = updatedChildValue(oldChild, remainingPaths, value) array[index] = UpdatedChildValue(oldChild, remainingPaths, value)
} }
return array return array
} else { } else {
index, _ = strconv.ParseInt(rawIndex, 10, 64) // nolint index, _ = strconv.ParseInt(rawIndex, 10, 64) // nolint
// writeArray is only called by updatedChildValue which handles parsing the // writeArray is only called by UpdatedChildValue which handles parsing the
// index, as such this renders this dead code. // index, as such this renders this dead code.
} }
@ -139,7 +139,7 @@ func writeArray(context interface{}, paths []string, value interface{}) []interf
log.Debugf("\tcurrentChild %v\n", currentChild) log.Debugf("\tcurrentChild %v\n", currentChild)
array[index] = updatedChildValue(currentChild, remainingPaths, value) array[index] = UpdatedChildValue(currentChild, remainingPaths, value)
log.Debugf("\tReturning array %v\n", array) log.Debugf("\tReturning array %v\n", array)
return array return array
} }
@ -174,7 +174,7 @@ func readMapSplat(context yaml.MapSlice, tail []string) (interface{}, error) {
var i = 0 var i = 0
for _, entry := range context { for _, entry := range context {
if len(tail) > 0 { if len(tail) > 0 {
val, err := recurse(entry.Value, tail[0], tail[1:]) val, err := Recurse(entry.Value, tail[0], tail[1:])
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -187,7 +187,7 @@ func readMapSplat(context yaml.MapSlice, tail []string) (interface{}, error) {
return newArray, nil return newArray, nil
} }
func recurse(value interface{}, head string, tail []string) (interface{}, error) { func Recurse(value interface{}, head string, tail []string) (interface{}, error) {
switch value := value.(type) { switch value := value.(type) {
case []interface{}: case []interface{}:
if head == "*" { if head == "*" {
@ -228,7 +228,7 @@ func readArraySplat(array []interface{}, tail []string) (interface{}, error) {
func calculateValue(value interface{}, tail []string) (interface{}, error) { func calculateValue(value interface{}, tail []string) (interface{}, error) {
if len(tail) > 0 { if len(tail) > 0 {
return recurse(value, tail[0], tail[1:]) return Recurse(value, tail[0], tail[1:])
} }
return value, nil return value, nil
} }
@ -266,7 +266,7 @@ func deleteEntryInMap(original yaml.MapSlice, child yaml.MapItem, index int, pat
if len(remainingPaths) > 0 { if len(remainingPaths) > 0 {
newChild := yaml.MapItem{Key: child.Key} newChild := yaml.MapItem{Key: child.Key}
var errorDeleting error var errorDeleting error
newChild.Value, errorDeleting = deleteChildValue(child.Value, remainingPaths) newChild.Value, errorDeleting = DeleteChildValue(child.Value, remainingPaths)
if errorDeleting != nil { if errorDeleting != nil {
return nil, errorDeleting return nil, errorDeleting
} }
@ -293,7 +293,7 @@ func deleteArraySplat(array []interface{}, tail []string) (interface{}, error) {
log.Debugf("deleteArraySplat for %v for %v\n", tail, array) log.Debugf("deleteArraySplat for %v for %v\n", tail, array)
var newArray = make([]interface{}, len(array)) var newArray = make([]interface{}, len(array))
for index, value := range array { for index, value := range array {
val, err := deleteChildValue(value, tail) val, err := DeleteChildValue(value, tail)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -328,8 +328,8 @@ func deleteArray(array []interface{}, paths []string, index int64) (interface{},
return array, nil return array, nil
} }
func deleteChildValue(child interface{}, remainingPaths []string) (interface{}, error) { func DeleteChildValue(child interface{}, remainingPaths []string) (interface{}, error) {
log.Debugf("deleteChildValue for %v for %v\n", remainingPaths, child) log.Debugf("DeleteChildValue for %v for %v\n", remainingPaths, child)
var head = remainingPaths[0] var head = remainingPaths[0]
var tail = remainingPaths[1:] var tail = remainingPaths[1:]
switch child := child.(type) { switch child := child.(type) {

View File

@ -1,32 +1,33 @@
package main package yqlib
import ( import (
"fmt" "fmt"
"sort" "sort"
"testing" "testing"
"github.com/mikefarah/yq/test"
) )
func TestReadMap_simple(t *testing.T) { func TestReadMap_simple(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
--- ---
b: b:
c: 2 c: 2
`) `)
got, _ := readMap(data, "b", []string{"c"}) got, _ := readMap(data, "b", []string{"c"})
assertResult(t, 2, got) test.AssertResult(t, 2, got)
} }
func TestReadMap_numberKey(t *testing.T) { func TestReadMap_numberKey(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
--- ---
200: things 200: things
`) `)
got, _ := readMap(data, "200", []string{}) got, _ := readMap(data, "200", []string{})
assertResult(t, "things", got) test.AssertResult(t, "things", got)
} }
func TestReadMap_splat(t *testing.T) { func TestReadMap_splat(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
--- ---
mapSplat: mapSplat:
item1: things item1: things
@ -34,11 +35,11 @@ mapSplat:
otherThing: cat otherThing: cat
`) `)
res, _ := readMap(data, "mapSplat", []string{"*"}) res, _ := readMap(data, "mapSplat", []string{"*"})
assertResult(t, "[things whatever cat]", fmt.Sprintf("%v", res)) test.AssertResult(t, "[things whatever cat]", fmt.Sprintf("%v", res))
} }
func TestReadMap_prefixSplat(t *testing.T) { func TestReadMap_prefixSplat(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
--- ---
mapSplat: mapSplat:
item1: things item1: things
@ -46,11 +47,11 @@ mapSplat:
otherThing: cat otherThing: cat
`) `)
res, _ := readMap(data, "mapSplat", []string{"item*"}) res, _ := readMap(data, "mapSplat", []string{"item*"})
assertResult(t, "[things whatever]", fmt.Sprintf("%v", res)) test.AssertResult(t, "[things whatever]", fmt.Sprintf("%v", res))
} }
func TestReadMap_deep_splat(t *testing.T) { func TestReadMap_deep_splat(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
--- ---
mapSplatDeep: mapSplatDeep:
item1: item1:
@ -63,30 +64,30 @@ mapSplatDeep:
result := res.([]interface{}) result := res.([]interface{})
var actual = []string{result[0].(string), result[1].(string)} var actual = []string{result[0].(string), result[1].(string)}
sort.Strings(actual) sort.Strings(actual)
assertResult(t, "[apples bananas]", fmt.Sprintf("%v", actual)) test.AssertResult(t, "[apples bananas]", fmt.Sprintf("%v", actual))
} }
func TestReadMap_key_doesnt_exist(t *testing.T) { func TestReadMap_key_doesnt_exist(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
--- ---
b: b:
c: 2 c: 2
`) `)
got, _ := readMap(data, "b.x.f", []string{"c"}) got, _ := readMap(data, "b.x.f", []string{"c"})
assertResult(t, nil, got) test.AssertResult(t, nil, got)
} }
func TestReadMap_recurse_against_string(t *testing.T) { func TestReadMap_recurse_against_string(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
--- ---
a: cat a: cat
`) `)
got, _ := readMap(data, "a", []string{"b"}) got, _ := readMap(data, "a", []string{"b"})
assertResult(t, nil, got) test.AssertResult(t, nil, got)
} }
func TestReadMap_with_array(t *testing.T) { func TestReadMap_with_array(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
--- ---
b: b:
d: d:
@ -94,11 +95,11 @@ b:
- 4 - 4
`) `)
got, _ := readMap(data, "b", []string{"d", "1"}) got, _ := readMap(data, "b", []string{"d", "1"})
assertResult(t, 4, got) test.AssertResult(t, 4, got)
} }
func TestReadMap_with_array_and_bad_index(t *testing.T) { func TestReadMap_with_array_and_bad_index(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
--- ---
b: b:
d: d:
@ -110,11 +111,11 @@ b:
t.Fatal("Expected error due to invalid path") t.Fatal("Expected error due to invalid path")
} }
expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax` expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
assertResult(t, expectedOutput, err.Error()) test.AssertResult(t, expectedOutput, err.Error())
} }
func TestReadMap_with_mapsplat_array_and_bad_index(t *testing.T) { func TestReadMap_with_mapsplat_array_and_bad_index(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
--- ---
b: b:
d: d:
@ -130,11 +131,11 @@ b:
t.Fatal("Expected error due to invalid path") t.Fatal("Expected error due to invalid path")
} }
expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax` expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
assertResult(t, expectedOutput, err.Error()) test.AssertResult(t, expectedOutput, err.Error())
} }
func TestReadMap_with_arraysplat_map_array_and_bad_index(t *testing.T) { func TestReadMap_with_arraysplat_map_array_and_bad_index(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
--- ---
b: b:
d: d:
@ -150,11 +151,11 @@ b:
t.Fatal("Expected error due to invalid path") t.Fatal("Expected error due to invalid path")
} }
expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax` expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
assertResult(t, expectedOutput, err.Error()) test.AssertResult(t, expectedOutput, err.Error())
} }
func TestReadMap_with_array_out_of_bounds(t *testing.T) { func TestReadMap_with_array_out_of_bounds(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
--- ---
b: b:
d: d:
@ -162,11 +163,11 @@ b:
- 4 - 4
`) `)
got, _ := readMap(data, "b", []string{"d", "3"}) got, _ := readMap(data, "b", []string{"d", "3"})
assertResult(t, nil, got) test.AssertResult(t, nil, got)
} }
func TestReadMap_with_array_out_of_bounds_by_1(t *testing.T) { func TestReadMap_with_array_out_of_bounds_by_1(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
--- ---
b: b:
d: d:
@ -174,11 +175,11 @@ b:
- 4 - 4
`) `)
got, _ := readMap(data, "b", []string{"d", "2"}) got, _ := readMap(data, "b", []string{"d", "2"})
assertResult(t, nil, got) test.AssertResult(t, nil, got)
} }
func TestReadMap_with_array_splat(t *testing.T) { func TestReadMap_with_array_splat(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
e: e:
- -
name: Fred name: Fred
@ -188,71 +189,71 @@ e:
thing: dog thing: dog
`) `)
got, _ := readMap(data, "e", []string{"*", "name"}) got, _ := readMap(data, "e", []string{"*", "name"})
assertResult(t, "[Fred Sam]", fmt.Sprintf("%v", got)) test.AssertResult(t, "[Fred Sam]", fmt.Sprintf("%v", got))
} }
func TestWrite_really_simple(t *testing.T) { func TestWrite_really_simple(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
b: 2 b: 2
`) `)
updated := writeMap(data, []string{"b"}, "4") updated := writeMap(data, []string{"b"}, "4")
assertResult(t, "[{b 4}]", fmt.Sprintf("%v", updated)) test.AssertResult(t, "[{b 4}]", fmt.Sprintf("%v", updated))
} }
func TestWrite_simple(t *testing.T) { func TestWrite_simple(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
b: b:
c: 2 c: 2
`) `)
updated := writeMap(data, []string{"b", "c"}, "4") updated := writeMap(data, []string{"b", "c"}, "4")
assertResult(t, "[{b [{c 4}]}]", fmt.Sprintf("%v", updated)) test.AssertResult(t, "[{b [{c 4}]}]", fmt.Sprintf("%v", updated))
} }
func TestWrite_new(t *testing.T) { func TestWrite_new(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
b: b:
c: 2 c: 2
`) `)
updated := writeMap(data, []string{"b", "d"}, "4") updated := writeMap(data, []string{"b", "d"}, "4")
assertResult(t, "[{b [{c 2} {d 4}]}]", fmt.Sprintf("%v", updated)) test.AssertResult(t, "[{b [{c 2} {d 4}]}]", fmt.Sprintf("%v", updated))
} }
func TestWrite_new_deep(t *testing.T) { func TestWrite_new_deep(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
b: b:
c: 2 c: 2
`) `)
updated := writeMap(data, []string{"b", "d", "f"}, "4") updated := writeMap(data, []string{"b", "d", "f"}, "4")
assertResult(t, "[{b [{c 2} {d [{f 4}]}]}]", fmt.Sprintf("%v", updated)) test.AssertResult(t, "[{b [{c 2} {d [{f 4}]}]}]", fmt.Sprintf("%v", updated))
} }
func TestWrite_array(t *testing.T) { func TestWrite_array(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
b: b:
- aa - aa
`) `)
updated := writeMap(data, []string{"b", "0"}, "bb") updated := writeMap(data, []string{"b", "0"}, "bb")
assertResult(t, "[{b [bb]}]", fmt.Sprintf("%v", updated)) test.AssertResult(t, "[{b [bb]}]", fmt.Sprintf("%v", updated))
} }
func TestWrite_new_array(t *testing.T) { func TestWrite_new_array(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
b: b:
c: 2 c: 2
`) `)
updated := writeMap(data, []string{"b", "0"}, "4") updated := writeMap(data, []string{"b", "0"}, "4")
assertResult(t, "[{b [{c 2} {0 4}]}]", fmt.Sprintf("%v", updated)) test.AssertResult(t, "[{b [{c 2} {0 4}]}]", fmt.Sprintf("%v", updated))
} }
func TestWrite_new_array_deep(t *testing.T) { func TestWrite_new_array_deep(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
a: apple a: apple
`) `)
@ -261,12 +262,12 @@ b:
- c: "4"` - c: "4"`
updated := writeMap(data, []string{"b", "+", "c"}, "4") updated := writeMap(data, []string{"b", "+", "c"}, "4")
got, _ := yamlToString(updated) got, _ := YamlToString(updated, true)
assertResult(t, expected, got) test.AssertResult(t, expected, got)
} }
func TestWrite_new_map_array_deep(t *testing.T) { func TestWrite_new_map_array_deep(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
b: b:
c: 2 c: 2
`) `)
@ -276,12 +277,12 @@ b:
- "4"` - "4"`
updated := writeMap(data, []string{"b", "d", "+"}, "4") updated := writeMap(data, []string{"b", "d", "+"}, "4")
got, _ := yamlToString(updated) got, _ := YamlToString(updated, true)
assertResult(t, expected, got) test.AssertResult(t, expected, got)
} }
func TestWrite_add_to_array(t *testing.T) { func TestWrite_add_to_array(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
b: b:
- aa - aa
`) `)
@ -291,109 +292,109 @@ b:
- bb` - bb`
updated := writeMap(data, []string{"b", "1"}, "bb") updated := writeMap(data, []string{"b", "1"}, "bb")
got, _ := yamlToString(updated) got, _ := YamlToString(updated, true)
assertResult(t, expected, got) test.AssertResult(t, expected, got)
} }
func TestWrite_with_no_tail(t *testing.T) { func TestWrite_with_no_tail(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
b: b:
c: 2 c: 2
`) `)
updated := writeMap(data, []string{"b"}, "4") updated := writeMap(data, []string{"b"}, "4")
assertResult(t, "[{b 4}]", fmt.Sprintf("%v", updated)) test.AssertResult(t, "[{b 4}]", fmt.Sprintf("%v", updated))
} }
func TestWriteMap_no_paths(t *testing.T) { func TestWriteMap_no_paths(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
b: 5 b: 5
`) `)
result := writeMap(data, []string{}, 4) result := writeMap(data, []string{}, 4)
assertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result)) test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
} }
func TestWriteArray_no_paths(t *testing.T) { func TestWriteArray_no_paths(t *testing.T) {
var data = make([]interface{}, 1) var data = make([]interface{}, 1)
data[0] = "mike" data[0] = "mike"
result := writeArray(data, []string{}, 4) result := writeArray(data, []string{}, 4)
assertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result)) test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
} }
func TestDelete_MapItem(t *testing.T) { func TestDelete_MapItem(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
a: 123 a: 123
b: 456 b: 456
`) `)
var expected = parseData(` var expected = test.ParseData(`
b: 456 b: 456
`) `)
result, _ := deleteMap(data, []string{"a"}) result, _ := deleteMap(data, []string{"a"})
assertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result)) test.AssertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result))
} }
// Ensure deleting an index into a string does nothing // Ensure deleting an index into a string does nothing
func TestDelete_index_to_string(t *testing.T) { func TestDelete_index_to_string(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
a: mystring a: mystring
`) `)
result, _ := deleteMap(data, []string{"a", "0"}) result, _ := deleteMap(data, []string{"a", "0"})
assertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result)) test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
} }
func TestDelete_list_index(t *testing.T) { func TestDelete_list_index(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
a: [3, 4] a: [3, 4]
`) `)
var expected = parseData(` var expected = test.ParseData(`
a: [3] a: [3]
`) `)
result, _ := deleteMap(data, []string{"a", "1"}) result, _ := deleteMap(data, []string{"a", "1"})
assertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result)) test.AssertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result))
} }
func TestDelete_list_index_beyond_bounds(t *testing.T) { func TestDelete_list_index_beyond_bounds(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
a: [3, 4] a: [3, 4]
`) `)
result, _ := deleteMap(data, []string{"a", "5"}) result, _ := deleteMap(data, []string{"a", "5"})
assertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result)) test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
} }
func TestDelete_list_index_out_of_bounds_by_1(t *testing.T) { func TestDelete_list_index_out_of_bounds_by_1(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
a: [3, 4] a: [3, 4]
`) `)
result, _ := deleteMap(data, []string{"a", "2"}) result, _ := deleteMap(data, []string{"a", "2"})
assertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result)) test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
} }
func TestDelete_no_paths(t *testing.T) { func TestDelete_no_paths(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
a: [3, 4] a: [3, 4]
b: b:
- name: test - name: test
`) `)
result, _ := deleteMap(data, []string{}) result, _ := deleteMap(data, []string{})
assertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result)) test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
} }
func TestDelete_array_map_item(t *testing.T) { func TestDelete_array_map_item(t *testing.T) {
var data = parseData(` var data = test.ParseData(`
b: b:
- name: fred - name: fred
value: blah value: blah
- name: john - name: john
value: test value: test
`) `)
var expected = parseData(` var expected = test.ParseData(`
b: b:
- value: blah - value: blah
- name: john - name: john
value: test value: test
`) `)
result, _ := deleteMap(data, []string{"b", "0", "name"}) result, _ := deleteMap(data, []string{"b", "0", "name"})
assertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result)) test.AssertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result))
} }

View File

@ -1,4 +1,4 @@
package main package yqlib
import ( import (
"encoding/json" "encoding/json"
@ -8,7 +8,7 @@ import (
yaml "github.com/mikefarah/yaml/v2" yaml "github.com/mikefarah/yaml/v2"
) )
func jsonToString(context interface{}) (string, error) { func JsonToString(context interface{}) (string, error) {
out, err := json.Marshal(toJSON(context)) out, err := json.Marshal(toJSON(context))
if err != nil { if err != nil {
return "", fmt.Errorf("error printing yaml as json: %v", err) return "", fmt.Errorf("error printing yaml as json: %v", err)

View File

@ -0,0 +1,47 @@
package yqlib
import (
"testing"
"github.com/mikefarah/yq/test"
)
func TestJsonToString(t *testing.T) {
var data = test.ParseData(`
---
b:
c: 2
`)
got, _ := JsonToString(data)
test.AssertResult(t, "{\"b\":{\"c\":2}}", got)
}
func TestJsonToString_withIntKey(t *testing.T) {
var data = test.ParseData(`
---
b:
2: c
`)
got, _ := JsonToString(data)
test.AssertResult(t, `{"b":{"2":"c"}}`, got)
}
func TestJsonToString_withBoolKey(t *testing.T) {
var data = test.ParseData(`
---
b:
false: c
`)
got, _ := JsonToString(data)
test.AssertResult(t, `{"b":{"false":"c"}}`, got)
}
func TestJsonToString_withArray(t *testing.T) {
var data = test.ParseData(`
---
b:
- item: one
- item: two
`)
got, _ := JsonToString(data)
test.AssertResult(t, "{\"b\":[{\"item\":\"one\"},{\"item\":\"two\"}]}", got)
}

54
pkg/yqlib/lib.go Normal file
View File

@ -0,0 +1,54 @@
package yqlib
import (
mergo "gopkg.in/imdario/mergo.v0"
logging "gopkg.in/op/go-logging.v1"
)
var log = logging.MustGetLogger("yq")
func SetLogger(l *logging.Logger) {
log = l
}
func ReadPath(dataBucket interface{}, path string) (interface{}, error) {
var paths = ParsePath(path)
return Recurse(dataBucket, paths[0], paths[1:])
}
func WritePath(dataBucket interface{}, path string, value interface{}) (interface{}) {
var paths = ParsePath(path)
return UpdatedChildValue(dataBucket, paths, value)
}
func PrefixPath(dataBucket interface{}, prefix string) (interface{}) {
var paths = ParsePath(prefix)
// Inverse order
for i := len(paths)/2 - 1; i >= 0; i-- {
opp := len(paths) - 1 - i
paths[i], paths[opp] = paths[opp], paths[i]
}
var mapDataBucket = dataBucket
for _, key := range paths {
singlePath := []string{key}
mapDataBucket = UpdatedChildValue(nil, singlePath, mapDataBucket)
}
return mapDataBucket
}
func DeletePath(dataBucket interface{}, path string) (interface{}, error) {
var paths = ParsePath(path)
return DeleteChildValue(dataBucket, paths)
}
func Merge(dst interface{}, src interface{}, overwrite bool, append bool) error {
if overwrite {
return mergo.Merge(dst, src, mergo.WithOverride)
} else if append {
return mergo.Merge(dst, src, mergo.WithAppendSlice)
}
return mergo.Merge(dst, src)
}

148
pkg/yqlib/lib_test.go Normal file
View File

@ -0,0 +1,148 @@
package yqlib
import (
"fmt"
"testing"
"github.com/mikefarah/yq/test"
)
func TestReadPath(t *testing.T) {
var data = test.ParseData(`
---
b:
2: c
`)
got, _ := ReadPath(data, "b.2")
test.AssertResult(t, `c`, got)
}
func TestReadPath_WithError(t *testing.T) {
var data = test.ParseData(`
---
b:
- c
`)
_, err := ReadPath(data, "b.[a]")
if err == nil {
t.Fatal("Expected error due to invalid path")
}
}
func TestWritePath(t *testing.T) {
var data = test.ParseData(`
---
b:
2: c
`)
got := WritePath(data, "b.3", "a")
test.AssertResult(t, `[{b [{2 c} {3 a}]}]`, fmt.Sprintf("%v", got))
}
func TestPrefixPath(t *testing.T) {
var data = test.ParseData(`
---
b:
2: c
`)
got := PrefixPath(data, "d")
test.AssertResult(t, `[{d [{b [{2 c}]}]}]`, fmt.Sprintf("%v", got))
}
func TestDeletePath(t *testing.T) {
var data = test.ParseData(`
---
b:
2: c
3: a
`)
got, _ := DeletePath(data, "b.2")
test.AssertResult(t, `[{b [{3 a}]}]`, fmt.Sprintf("%v", got))
}
func TestDeletePath_WithError(t *testing.T) {
var data = test.ParseData(`
---
b:
- c
`)
_, err := DeletePath(data, "b.[a]")
if err == nil {
t.Fatal("Expected error due to invalid path")
}
}
func TestMerge(t *testing.T) {
var dst = test.ParseData(`
---
a: b
c: d
`)
var src = test.ParseData(`
---
a: 1
b: 2
`)
var mergedData = make(map[interface{}]interface{})
mergedData["root"] = dst
var mapDataBucket = make(map[interface{}]interface{})
mapDataBucket["root"] = src
Merge(&mergedData, mapDataBucket, false, false)
test.AssertResult(t, `[{a b} {c d}]`, fmt.Sprintf("%v", mergedData["root"]))
}
func TestMerge_WithOverwrite(t *testing.T) {
var dst = test.ParseData(`
---
a: b
c: d
`)
var src = test.ParseData(`
---
a: 1
b: 2
`)
var mergedData = make(map[interface{}]interface{})
mergedData["root"] = dst
var mapDataBucket = make(map[interface{}]interface{})
mapDataBucket["root"] = src
Merge(&mergedData, mapDataBucket, true, false)
test.AssertResult(t, `[{a 1} {b 2}]`, fmt.Sprintf("%v", mergedData["root"]))
}
func TestMerge_WithAppend(t *testing.T) {
var dst = test.ParseData(`
---
a: b
c: d
`)
var src = test.ParseData(`
---
a: 1
b: 2
`)
var mergedData = make(map[interface{}]interface{})
mergedData["root"] = dst
var mapDataBucket = make(map[interface{}]interface{})
mapDataBucket["root"] = src
Merge(&mergedData, mapDataBucket, false, true)
test.AssertResult(t, `[{a b} {c d} {a 1} {b 2}]`, fmt.Sprintf("%v", mergedData["root"]))
}
func TestMerge_WithError(t *testing.T) {
err := Merge(nil, nil, false, false)
if err == nil {
t.Fatal("Expected error due to nil")
}
}

View File

@ -1,6 +1,6 @@
package main package yqlib
func parsePath(path string) []string { func ParsePath(path string) []string {
return parsePathAccum([]string{}, path) return parsePathAccum([]string{}, path)
} }

View File

@ -1,7 +1,8 @@
package main package yqlib
import ( import (
"testing" "testing"
"github.com/mikefarah/yq/test"
) )
var parsePathsTests = []struct { var parsePathsTests = []struct {
@ -15,7 +16,7 @@ var parsePathsTests = []struct {
func TestParsePath(t *testing.T) { func TestParsePath(t *testing.T) {
for _, tt := range parsePathsTests { for _, tt := range parsePathsTests {
assertResultComplex(t, tt.expectedPaths, parsePath(tt.path)) test.AssertResultComplex(t, tt.expectedPaths, ParsePath(tt.path))
} }
} }
@ -37,7 +38,7 @@ var nextYamlPathTests = []struct {
func TestNextYamlPath(t *testing.T) { func TestNextYamlPath(t *testing.T) {
for _, tt := range nextYamlPathTests { for _, tt := range nextYamlPathTests {
var element, remaining = nextYamlPath(tt.path) var element, remaining = nextYamlPath(tt.path)
assertResultWithContext(t, tt.expectedElement, element, tt) test.AssertResultWithContext(t, tt.expectedElement, element, tt)
assertResultWithContext(t, tt.expectedRemaining, remaining, tt) test.AssertResultWithContext(t, tt.expectedRemaining, remaining, tt)
} }
} }

View File

@ -0,0 +1,32 @@
package yqlib
import (
yaml "github.com/mikefarah/yaml/v2"
errors "github.com/pkg/errors"
"strings"
)
func YamlToString(context interface{}, trimOutput bool) (string, error) {
switch context := context.(type) {
case string:
return context, nil
default:
return marshalContext(context, trimOutput)
}
}
func marshalContext(context interface{}, trimOutput bool) (string, error) {
out, err := yaml.Marshal(context)
if err != nil {
return "", errors.Wrap(err, "error printing yaml")
}
outStr := string(out)
// trim the trailing new line as it's easier for a script to add
// it in if required than to remove it
if trimOutput {
return strings.Trim(outStr, "\n "), nil
}
return outStr, nil
}

View File

@ -1,4 +1,4 @@
package main package test
import ( import (
"bytes" "bytes"
@ -19,7 +19,7 @@ type resulter struct {
Command *cobra.Command Command *cobra.Command
} }
func runCmd(c *cobra.Command, input string) resulter { func RunCmd(c *cobra.Command, input string) resulter {
buf := new(bytes.Buffer) buf := new(bytes.Buffer)
c.SetOutput(buf) c.SetOutput(buf)
c.SetArgs(strings.Split(input, " ")) c.SetArgs(strings.Split(input, " "))
@ -30,7 +30,7 @@ func runCmd(c *cobra.Command, input string) resulter {
return resulter{err, output, c} return resulter{err, output, c}
} }
func parseData(rawData string) yaml.MapSlice { func ParseData(rawData string) yaml.MapSlice {
var parsedData yaml.MapSlice var parsedData yaml.MapSlice
err := yaml.Unmarshal([]byte(rawData), &parsedData) err := yaml.Unmarshal([]byte(rawData), &parsedData)
if err != nil { if err != nil {
@ -40,21 +40,21 @@ func parseData(rawData string) yaml.MapSlice {
return parsedData return parsedData
} }
func assertResult(t *testing.T, expectedValue interface{}, actualValue interface{}) { func AssertResult(t *testing.T, expectedValue interface{}, actualValue interface{}) {
t.Helper() t.Helper()
if expectedValue != actualValue { if expectedValue != actualValue {
t.Error("Expected <", expectedValue, "> but got <", actualValue, ">", fmt.Sprintf("%T", actualValue)) t.Error("Expected <", expectedValue, "> but got <", actualValue, ">", fmt.Sprintf("%T", actualValue))
} }
} }
func assertResultComplex(t *testing.T, expectedValue interface{}, actualValue interface{}) { func AssertResultComplex(t *testing.T, expectedValue interface{}, actualValue interface{}) {
t.Helper() t.Helper()
if !reflect.DeepEqual(expectedValue, actualValue) { if !reflect.DeepEqual(expectedValue, actualValue) {
t.Error("Expected <", expectedValue, "> but got <", actualValue, ">", fmt.Sprintf("%T", actualValue)) t.Error("Expected <", expectedValue, "> but got <", actualValue, ">", fmt.Sprintf("%T", actualValue))
} }
} }
func assertResultWithContext(t *testing.T, expectedValue interface{}, actualValue interface{}, context interface{}) { func AssertResultWithContext(t *testing.T, expectedValue interface{}, actualValue interface{}, context interface{}) {
t.Helper() t.Helper()
if expectedValue != actualValue { if expectedValue != actualValue {
t.Error(context) t.Error(context)
@ -62,7 +62,7 @@ func assertResultWithContext(t *testing.T, expectedValue interface{}, actualValu
} }
} }
func writeTempYamlFile(content string) string { func WriteTempYamlFile(content string) string {
tmpfile, _ := ioutil.TempFile("", "testyaml") tmpfile, _ := ioutil.TempFile("", "testyaml")
defer func() { defer func() {
_ = tmpfile.Close() _ = tmpfile.Close()
@ -72,11 +72,11 @@ func writeTempYamlFile(content string) string {
return tmpfile.Name() return tmpfile.Name()
} }
func readTempYamlFile(name string) string { func ReadTempYamlFile(name string) string {
content, _ := ioutil.ReadFile(name) content, _ := ioutil.ReadFile(name)
return string(content) return string(content)
} }
func removeTempYamlFile(name string) { func RemoveTempYamlFile(name string) {
_ = os.Remove(name) _ = os.Remove(name)
} }

79
yq.go
View File

@ -9,6 +9,7 @@ import (
"reflect" "reflect"
"strconv" "strconv"
"strings" "strings"
"github.com/mikefarah/yq/pkg/yqlib"
errors "github.com/pkg/errors" errors "github.com/pkg/errors"
@ -38,6 +39,7 @@ func main() {
} }
func newCommandCLI() *cobra.Command { func newCommandCLI() *cobra.Command {
yqlib.SetLogger(log)
yaml.DefaultMapType = reflect.TypeOf(yaml.MapSlice{}) yaml.DefaultMapType = reflect.TypeOf(yaml.MapSlice{})
var rootCmd = &cobra.Command{ var rootCmd = &cobra.Command{
Use: "yq", Use: "yq",
@ -270,13 +272,19 @@ func readProperty(cmd *cobra.Command, args []string) error {
log.Debugf("processing %v - requested index %v", currentIndex, docIndexInt) log.Debugf("processing %v - requested index %v", currentIndex, docIndexInt)
if updateAll || currentIndex == docIndexInt { if updateAll || currentIndex == docIndexInt {
log.Debugf("reading %v in index %v", path, currentIndex) log.Debugf("reading %v in index %v", path, currentIndex)
mappedDoc, errorParsing := readPath(dataBucket, path) if path == "" {
log.Debug("no path")
log.Debugf("%v", dataBucket)
mappedDocs = append(mappedDocs, dataBucket)
} else {
mappedDoc, errorParsing := yqlib.ReadPath(dataBucket, path)
log.Debugf("%v", mappedDoc) log.Debugf("%v", mappedDoc)
if errorParsing != nil { if errorParsing != nil {
return errors.Wrapf(errorParsing, "Error reading path in document index %v", currentIndex) return errors.Wrapf(errorParsing, "Error reading path in document index %v", currentIndex)
} }
mappedDocs = append(mappedDocs, mappedDoc) mappedDocs = append(mappedDocs, mappedDoc)
} }
}
currentIndex = currentIndex + 1 currentIndex = currentIndex + 1
} }
}) })
@ -299,15 +307,6 @@ func readProperty(cmd *cobra.Command, args []string) error {
return nil return nil
} }
func readPath(dataBucket interface{}, path string) (interface{}, error) {
if path == "" {
log.Debug("no path")
return dataBucket, nil
}
var paths = parsePath(path)
return recurse(dataBucket, paths[0], paths[1:])
}
func newProperty(cmd *cobra.Command, args []string) error { func newProperty(cmd *cobra.Command, args []string) error {
updatedData, err := newYaml(args) updatedData, err := newYaml(args)
if err != nil { if err != nil {
@ -339,8 +338,7 @@ func newYaml(args []string) (interface{}, error) {
path := entry.Key.(string) path := entry.Key.(string)
value := entry.Value value := entry.Value
log.Debugf("setting %v to %v", path, value) log.Debugf("setting %v to %v", path, value)
var paths = parsePath(path) dataBucket = yqlib.WritePath(dataBucket, path, value)
dataBucket = updatedChildValue(dataBucket, paths, value)
} }
return dataBucket, nil return dataBucket, nil
@ -416,8 +414,7 @@ func writeProperty(cmd *cobra.Command, args []string) error {
path := entry.Key.(string) path := entry.Key.(string)
value := entry.Value value := entry.Value
log.Debugf("setting %v to %v", path, value) log.Debugf("setting %v to %v", path, value)
var paths = parsePath(path) dataBucket = yqlib.WritePath(dataBucket, path, value)
dataBucket = updatedChildValue(dataBucket, paths, value)
} }
} }
return dataBucket, nil return dataBucket, nil
@ -429,28 +426,18 @@ func prefixProperty(cmd *cobra.Command, args []string) error {
if len(args) != 2 { if len(args) != 2 {
return errors.New("Must provide <filename> <prefixed_path>") return errors.New("Must provide <filename> <prefixed_path>")
} }
prefixPath := args[1]
var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex() var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex()
if errorParsingDocIndex != nil { if errorParsingDocIndex != nil {
return errorParsingDocIndex return errorParsingDocIndex
} }
var paths = parsePath(args[1])
// Inverse order
for i := len(paths)/2 - 1; i >= 0; i-- {
opp := len(paths) - 1 - i
paths[i], paths[opp] = paths[opp], paths[i]
}
var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) { var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) {
if updateAll || currentIndex == docIndexInt { if updateAll || currentIndex == docIndexInt {
log.Debugf("Prefixing %v to doc %v", paths, currentIndex) log.Debugf("Prefixing %v to doc %v", prefixPath, currentIndex)
var mapDataBucket = dataBucket var mapDataBucket = yqlib.PrefixPath(dataBucket, prefixPath)
for _, key := range paths {
singlePath := []string{key}
mapDataBucket = updatedChildValue(nil, singlePath, mapDataBucket)
}
return mapDataBucket, nil return mapDataBucket, nil
} }
return dataBucket, nil return dataBucket, nil
@ -496,7 +483,6 @@ func deleteProperty(cmd *cobra.Command, args []string) error {
return errors.New("Must provide <filename> <path_to_delete>") return errors.New("Must provide <filename> <path_to_delete>")
} }
var deletePath = args[1] var deletePath = args[1]
var paths = parsePath(deletePath)
var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex() var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex()
if errorParsingDocIndex != nil { if errorParsingDocIndex != nil {
return errorParsingDocIndex return errorParsingDocIndex
@ -505,7 +491,7 @@ func deleteProperty(cmd *cobra.Command, args []string) error {
var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) { var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) {
if updateAll || currentIndex == docIndexInt { if updateAll || currentIndex == docIndexInt {
log.Debugf("Deleting path in doc %v", currentIndex) log.Debugf("Deleting path in doc %v", currentIndex)
return deleteChildValue(dataBucket, paths) return yqlib.DeletePath(dataBucket, deletePath)
} }
return dataBucket, nil return dataBucket, nil
} }
@ -532,7 +518,7 @@ func mergeProperties(cmd *cobra.Command, args []string) error {
// map // map
var mapDataBucket = make(map[interface{}]interface{}) var mapDataBucket = make(map[interface{}]interface{})
mapDataBucket["root"] = dataBucket mapDataBucket["root"] = dataBucket
if err := merge(&mergedData, mapDataBucket, overwriteFlag, appendFlag); err != nil { if err := yqlib.Merge(&mergedData, mapDataBucket, overwriteFlag, appendFlag); err != nil {
return nil, err return nil, err
} }
for _, f := range filesToMerge { for _, f := range filesToMerge {
@ -544,7 +530,7 @@ func mergeProperties(cmd *cobra.Command, args []string) error {
return nil, err return nil, err
} }
mapDataBucket["root"] = fileToMerge mapDataBucket["root"] = fileToMerge
if err := merge(&mergedData, mapDataBucket, overwriteFlag, appendFlag); err != nil { if err := yqlib.Merge(&mergedData, mapDataBucket, overwriteFlag, appendFlag); err != nil {
return nil, err return nil, err
} }
} }
@ -594,34 +580,9 @@ func parseValue(argument string) interface{} {
func toString(context interface{}) (string, error) { func toString(context interface{}) (string, error) {
if outputToJSON { if outputToJSON {
return jsonToString(context) return yqlib.JsonToString(context)
} }
return yamlToString(context) return yqlib.YamlToString(context, trimOutput)
}
func yamlToString(context interface{}) (string, error) {
switch context := context.(type) {
case string:
return context, nil
default:
return marshalContext(context)
}
}
func marshalContext(context interface{}) (string, error) {
out, err := yaml.Marshal(context)
if err != nil {
return "", errors.Wrap(err, "error printing yaml")
}
outStr := string(out)
// trim the trailing new line as it's easier for a script to add
// it in if required than to remove it
if trimOutput {
return strings.Trim(outStr, "\n "), nil
}
return outStr, nil
} }
func safelyRenameFile(from string, to string) { func safelyRenameFile(from string, to string) {

View File

@ -4,6 +4,8 @@ import (
"fmt" "fmt"
"runtime" "runtime"
"testing" "testing"
"github.com/mikefarah/yq/test"
"github.com/mikefarah/yq/pkg/yqlib"
) )
var parseValueTests = []struct { var parseValueTests = []struct {
@ -20,7 +22,7 @@ var parseValueTests = []struct {
func TestParseValue(t *testing.T) { func TestParseValue(t *testing.T) {
for _, tt := range parseValueTests { for _, tt := range parseValueTests {
assertResultWithContext(t, tt.expectedResult, parseValue(tt.argument), tt.testDescription) test.AssertResultWithContext(t, tt.expectedResult, parseValue(tt.argument), tt.testDescription)
} }
} }
@ -28,14 +30,14 @@ func TestMultilineString(t *testing.T) {
testString := ` testString := `
abcd abcd
efg` efg`
formattedResult, _ := yamlToString(testString) formattedResult, _ := yqlib.YamlToString(testString, false)
assertResult(t, testString, formattedResult) test.AssertResult(t, testString, formattedResult)
} }
func TestNewYaml(t *testing.T) { func TestNewYaml(t *testing.T) {
result, _ := newYaml([]string{"b.c", "3"}) result, _ := newYaml([]string{"b.c", "3"})
formattedResult := fmt.Sprintf("%v", result) formattedResult := fmt.Sprintf("%v", result)
assertResult(t, test.AssertResult(t,
"[{b [{c 3}]}]", "[{b [{c 3}]}]",
formattedResult) formattedResult)
} }
@ -43,7 +45,7 @@ func TestNewYaml(t *testing.T) {
func TestNewYamlArray(t *testing.T) { func TestNewYamlArray(t *testing.T) {
result, _ := newYaml([]string{"[0].cat", "meow"}) result, _ := newYaml([]string{"[0].cat", "meow"})
formattedResult := fmt.Sprintf("%v", result) formattedResult := fmt.Sprintf("%v", result)
assertResult(t, test.AssertResult(t,
"[[{cat meow}]]", "[[{cat meow}]]",
formattedResult) formattedResult)
} }
@ -55,8 +57,8 @@ func TestNewYaml_WithScript(t *testing.T) {
e: e:
- name: Mike Farah` - name: Mike Farah`
result, _ := newYaml([]string{""}) result, _ := newYaml([]string{""})
actualResult, _ := yamlToString(result) actualResult, _ := yqlib.YamlToString(result, true)
assertResult(t, expectedResult, actualResult) test.AssertResult(t, expectedResult, actualResult)
} }
func TestNewYaml_WithUnknownScript(t *testing.T) { func TestNewYaml_WithUnknownScript(t *testing.T) {
@ -71,5 +73,5 @@ func TestNewYaml_WithUnknownScript(t *testing.T) {
} else { } else {
expectedOutput = `open fake-unknown: no such file or directory` expectedOutput = `open fake-unknown: no such file or directory`
} }
assertResult(t, expectedOutput, err.Error()) test.AssertResult(t, expectedOutput, err.Error())
} }