From 9771e7001c3f7ea9ecb88ff7ced976621d2f41f3 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Mon, 9 Dec 2019 13:44:53 +1100 Subject: [PATCH] splatting --- commands_test.go | 135 +++--- compare.sh | 13 + examples/sample.yaml | 13 +- pkg/yqlib/data_navigator.go | 54 ++- pkg/yqlib/data_navigator_test.go | 694 +++++++++++++++---------------- pkg/yqlib/lib.go | 8 +- pkg/yqlib/lib_test.go | 2 +- pkg/yqlib/path_parser.go | 3 + pkg/yqlib/path_parser_test.go | 2 +- pkg/yqlib/value_parser_test.go | 2 +- test/utils.go | 6 +- yq.go | 6 +- yq_test.go | 104 ++--- 13 files changed, 527 insertions(+), 515 deletions(-) create mode 100755 compare.sh diff --git a/commands_test.go b/commands_test.go index 90409cb8..49f12578 100644 --- a/commands_test.go +++ b/commands_test.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - "github.com/mikefarah/yq/v2/test" + "github.com/mikefarah/yq/v3/test" "github.com/spf13/cobra" ) @@ -63,30 +63,6 @@ func TestRootCmd_VerboseShort(t *testing.T) { } } -func TestRootCmd_TrimLong(t *testing.T) { - cmd := getRootCommand() - result := test.RunCmd(cmd, "--trim") - if result.Error != nil { - t.Error(result.Error) - } - - if !trimOutput { - t.Error("Expected trimOutput to be true") - } -} - -func TestRootCmd_TrimShort(t *testing.T) { - cmd := getRootCommand() - result := test.RunCmd(cmd, "-t") - if result.Error != nil { - t.Error(result.Error) - } - - if !trimOutput { - t.Error("Expected trimOutput to be true") - } -} - func TestRootCmd_VersionShort(t *testing.T) { cmd := getRootCommand() result := test.RunCmd(cmd, "-V") @@ -191,7 +167,7 @@ func TestReadCmd_ArrayYaml_NoPath(t *testing.T) { expectedOutput := `- become: true gather_facts: false hosts: lalaland - name: Apply smth + name: "Apply smth" roles: - lala - land @@ -209,7 +185,7 @@ func TestReadCmd_ArrayYaml_OneElement(t *testing.T) { expectedOutput := `become: true gather_facts: false hosts: lalaland -name: Apply smth +name: "Apply smth" roles: - lala - land @@ -227,7 +203,7 @@ func TestReadCmd_ArrayYaml_Splat(t *testing.T) { expectedOutput := `- become: true gather_facts: false hosts: lalaland - name: Apply smth + name: "Apply smth" roles: - lala - land @@ -252,19 +228,19 @@ func TestReadCmd_ArrayYaml_ErrorBadPath(t *testing.T) { if result.Error == nil { t.Error("Expected command to fail due to invalid path") } - expectedOutput := `Error reading path in document index 0: error accessing array: strconv.ParseInt: parsing "x": invalid syntax` + expectedOutput := `Error reading path in document index 0: strconv.ParseInt: parsing "x": invalid syntax` test.AssertResult(t, expectedOutput, result.Error.Error()) } -func TestReadCmd_ArrayYaml_Splat_ErrorBadPath(t *testing.T) { - cmd := getRootCommand() - result := test.RunCmd(cmd, "read examples/array.yaml [*].roles[x]") - if result.Error == nil { - t.Error("Expected command to fail due to invalid path") - } - expectedOutput := `Error reading path in document index 0: error accessing array: strconv.ParseInt: parsing "x": invalid syntax` - test.AssertResult(t, expectedOutput, result.Error.Error()) -} +// func TestReadCmd_ArrayYaml_Splat_ErrorBadPath(t *testing.T) { +// cmd := getRootCommand() +// result := test.RunCmd(cmd, "read examples/array.yaml [*].roles[x]") +// if result.Error == nil { +// t.Error("Expected command to fail due to invalid path") +// } +// expectedOutput := `Error reading path in document index 0: error accessing array: strconv.ParseInt: parsing "x": invalid syntax` +// test.AssertResult(t, expectedOutput, result.Error.Error()) +// } func TestReadCmd_Error(t *testing.T) { cmd := getRootCommand() @@ -301,27 +277,27 @@ func TestReadCmd_ErrorUnreadableFile(t *testing.T) { test.AssertResult(t, expectedOutput, result.Error.Error()) } -func TestReadCmd_ErrorBadPath(t *testing.T) { - content := `b: - d: - e: - - 3 - - 4 - f: - - 1 - - 2 -` - filename := test.WriteTempYamlFile(content) - defer test.RemoveTempYamlFile(filename) +// func TestReadCmd_ErrorBadPath(t *testing.T) { +// content := `b: +// d: +// e: +// - 3 +// - 4 +// f: +// - 1 +// - 2 +// ` +// filename := test.WriteTempYamlFile(content) +// defer test.RemoveTempYamlFile(filename) - cmd := getRootCommand() - result := test.RunCmd(cmd, fmt.Sprintf("read %s b.d.*.[x]", filename)) - if result.Error == nil { - t.Fatal("Expected command to fail due to invalid path") - } - expectedOutput := `Error reading path in document index 0: error accessing array: strconv.ParseInt: parsing "x": invalid syntax` - test.AssertResult(t, expectedOutput, result.Error.Error()) -} +// cmd := getRootCommand() +// result := test.RunCmd(cmd, fmt.Sprintf("read %s b.d.*.[x]", filename)) +// if result.Error == nil { +// t.Fatal("Expected command to fail due to invalid path") +// } +// expectedOutput := `Error reading path in document index 0: error accessing array: strconv.ParseInt: parsing "x": invalid syntax` +// test.AssertResult(t, expectedOutput, result.Error.Error()) +// } func TestReadCmd_Verbose(t *testing.T) { cmd := getRootCommand() @@ -332,32 +308,23 @@ func TestReadCmd_Verbose(t *testing.T) { test.AssertResult(t, "2\n", result.Output) } -func TestReadCmd_NoTrim(t *testing.T) { - cmd := getRootCommand() - result := test.RunCmd(cmd, "--trim=false read examples/sample.yaml b.c") - if result.Error != nil { - t.Error(result.Error) - } - test.AssertResult(t, "2\n\n", result.Output) -} +// func TestReadCmd_ToJson(t *testing.T) { +// cmd := getRootCommand() +// result := test.RunCmd(cmd, "read -j examples/sample.yaml b.c") +// if result.Error != nil { +// t.Error(result.Error) +// } +// test.AssertResult(t, "2\n", result.Output) +// } -func TestReadCmd_ToJson(t *testing.T) { - cmd := getRootCommand() - result := test.RunCmd(cmd, "read -j examples/sample.yaml b.c") - if result.Error != nil { - t.Error(result.Error) - } - test.AssertResult(t, "2\n", result.Output) -} - -func TestReadCmd_ToJsonLong(t *testing.T) { - cmd := getRootCommand() - result := test.RunCmd(cmd, "read --tojson examples/sample.yaml b.c") - if result.Error != nil { - t.Error(result.Error) - } - test.AssertResult(t, "2\n", result.Output) -} +// func TestReadCmd_ToJsonLong(t *testing.T) { +// cmd := getRootCommand() +// result := test.RunCmd(cmd, "read --tojson examples/sample.yaml b.c") +// if result.Error != nil { +// t.Error(result.Error) +// } +// test.AssertResult(t, "2\n", result.Output) +// } func TestPrefixCmd(t *testing.T) { content := `b: @@ -851,7 +818,7 @@ func TestWriteCmd_SplatMapEmpty(t *testing.T) { t.Error(result.Error) } expectedOutput := `b: - c: thing + c: {} d: another thing ` test.AssertResult(t, expectedOutput, result.Output) diff --git a/compare.sh b/compare.sh new file mode 100755 index 00000000..fc8cd8bf --- /dev/null +++ b/compare.sh @@ -0,0 +1,13 @@ +GREEN='\033[0;32m' +NC='\033[0m' + +echo "${GREEN}---Old---${NC}" +yq $@ > /tmp/yq-old-output +cat /tmp/yq-old-output + +echo "${GREEN}---New---${NC}" +./yq $@ > /tmp/yq-new-output +cat /tmp/yq-new-output + +echo "${GREEN}---Diff---${NC}" +colordiff /tmp/yq-old-output /tmp/yq-new-output \ No newline at end of file diff --git a/examples/sample.yaml b/examples/sample.yaml index b26830e8..e847c407 100644 --- a/examples/sample.yaml +++ b/examples/sample.yaml @@ -1,9 +1,8 @@ a: Easy! as one two three b: - c: 2 - d: [3, 4] - e: - - name: fred - value: 3 - - name: sam - value: 4 \ No newline at end of file + c: + name: c1 + f: things + d: + name: d1 + f: other \ No newline at end of file diff --git a/pkg/yqlib/data_navigator.go b/pkg/yqlib/data_navigator.go index a49dd6f7..3776d4ef 100644 --- a/pkg/yqlib/data_navigator.go +++ b/pkg/yqlib/data_navigator.go @@ -9,6 +9,7 @@ import ( ) type DataNavigator interface { + DebugNode(node *yaml.Node) Get(rootNode *yaml.Node, remainingPath []string) (*yaml.Node, error) Update(rootNode *yaml.Node, remainingPath []string, changesToApply yaml.Node) error } @@ -36,9 +37,9 @@ func (n *navigator) Get(value *yaml.Node, path []string) (*yaml.Node, error) { func (n *navigator) Update(value *yaml.Node, path []string, changesToApply yaml.Node) error { _, errorVisiting := n.Visit(value, path, func(nodeToUpdate *yaml.Node) (*yaml.Node, error) { n.log.Debug("going to update") - n.debugNode(nodeToUpdate) + n.DebugNode(nodeToUpdate) n.log.Debug("with") - n.debugNode(&changesToApply) + n.DebugNode(&changesToApply) nodeToUpdate.Value = changesToApply.Value nodeToUpdate.Tag = changesToApply.Tag nodeToUpdate.Kind = changesToApply.Kind @@ -55,26 +56,33 @@ func (n *navigator) Update(value *yaml.Node, path []string, changesToApply yaml. func (n *navigator) Visit(value *yaml.Node, path []string, visitor VisitorFn) (*yaml.Node, error) { realValue := value if realValue.Kind == yaml.DocumentNode { + n.log.Debugf("its a document! returning the first child") realValue = value.Content[0] } if len(path) > 0 { n.log.Debugf("diving into %v", path[0]) - n.debugNode(value) + n.DebugNode(value) return n.recurse(realValue, path[0], path[1:], visitor) } return visitor(realValue) } -func (n *navigator) guessKind(tail []string) yaml.Kind { +func (n *navigator) guessKind(tail []string, guess yaml.Kind) yaml.Kind { n.log.Debug("tail %v", tail) - if len(tail) == 0 { + if len(tail) == 0 && guess == 0 { n.log.Debug("end of path, must be a scalar") return yaml.ScalarNode + } else if len(tail) == 0 { + return guess } + var _, errorParsingInt = strconv.ParseInt(tail[0], 10, 64) - if tail[0] == "*" || tail[0] == "+" || errorParsingInt == nil { + if tail[0] == "+" || errorParsingInt == nil { return yaml.SequenceNode } + if tail[0] == "*" && guess == yaml.SequenceNode || guess == yaml.MappingNode { + return guess + } return yaml.MappingNode } @@ -90,7 +98,7 @@ func (n *navigator) getOrReplace(original *yaml.Node, expectedKind yaml.Kind) *y return original } -func (n *navigator) debugNode(value *yaml.Node) { +func (n *navigator) DebugNode(value *yaml.Node) { if n.log.IsEnabledFor(logging.DEBUG) { buf := new(bytes.Buffer) encoder := yaml.NewEncoder(buf) @@ -105,17 +113,33 @@ func (n *navigator) recurse(value *yaml.Node, head string, tail []string, visito switch value.Kind { case yaml.MappingNode: n.log.Debug("its a map with %v entries", len(value.Content)/2) + if head == "*" { + var newNode = yaml.Node{Kind: yaml.SequenceNode} + for index, content := range value.Content { + if index%2 == 0 { + continue + } + content = n.getOrReplace(content, n.guessKind(tail, content.Kind)) + var nestedValue, err = n.Visit(content, tail, visitor) + if err != nil { + return nil, err + } + newNode.Content = append(newNode.Content, nestedValue) + } + return &newNode, nil + } + for index, content := range value.Content { // value.Content is a concatenated array of key, value, // so keys are in the even indexes, values in odd. - if index%2 == 1 || content.Value != head { + if index%2 == 1 || (content.Value != head) { continue } - value.Content[index+1] = n.getOrReplace(value.Content[index+1], n.guessKind(tail)) + value.Content[index+1] = n.getOrReplace(value.Content[index+1], n.guessKind(tail, value.Content[index+1].Kind)) return n.Visit(value.Content[index+1], tail, visitor) } value.Content = append(value.Content, &yaml.Node{Value: head, Kind: yaml.ScalarNode}) - mapEntryValue := yaml.Node{Kind: n.guessKind(tail)} + mapEntryValue := yaml.Node{Kind: n.guessKind(tail, 0)} value.Content = append(value.Content, &mapEntryValue) n.log.Debug("adding new node %v", value.Content) return n.Visit(&mapEntryValue, tail, visitor) @@ -127,11 +151,11 @@ func (n *navigator) recurse(value *yaml.Node, head string, tail []string, visito for index, childValue := range value.Content { n.log.Debug("processing") - n.debugNode(childValue) - childValue = n.getOrReplace(childValue, n.guessKind(tail)) + n.DebugNode(childValue) + childValue = n.getOrReplace(childValue, n.guessKind(tail, childValue.Kind)) var nestedValue, err = n.Visit(childValue, tail, visitor) n.log.Debug("nestedValue") - n.debugNode(nestedValue) + n.DebugNode(nestedValue) if err != nil { return nil, err } @@ -140,7 +164,7 @@ func (n *navigator) recurse(value *yaml.Node, head string, tail []string, visito return &newNode, nil } else if head == "+" { - var newNode = yaml.Node{Kind: n.guessKind(tail)} + var newNode = yaml.Node{Kind: n.guessKind(tail, 0)} value.Content = append(value.Content, &newNode) n.log.Debug("appending a new node, %v", value.Content) return n.Visit(&newNode, tail, visitor) @@ -152,7 +176,7 @@ func (n *navigator) recurse(value *yaml.Node, head string, tail []string, visito if index >= int64(len(value.Content)) { return nil, nil } - value.Content[index] = n.getOrReplace(value.Content[index], n.guessKind(tail)) + value.Content[index] = n.getOrReplace(value.Content[index], n.guessKind(tail, value.Content[index].Kind)) return n.Visit(value.Content[index], tail, visitor) default: return nil, nil diff --git a/pkg/yqlib/data_navigator_test.go b/pkg/yqlib/data_navigator_test.go index 614a4977..d537d665 100644 --- a/pkg/yqlib/data_navigator_test.go +++ b/pkg/yqlib/data_navigator_test.go @@ -1,397 +1,397 @@ package yqlib -import ( - "fmt" - "sort" - "testing" +// import ( +// "fmt" +// "sort" +// "testing" - "github.com/mikefarah/yq/v2/test" - logging "gopkg.in/op/go-logging.v1" -) +// "github.com/mikefarah/yq/v2/test" +// logging "gopkg.in/op/go-logging.v1" +// ) -func TestDataNavigator(t *testing.T) { - var log = logging.MustGetLogger("yq") - subject := NewDataNavigator(log) +// func TestDataNavigator(t *testing.T) { +// var log = logging.MustGetLogger("yq") +// subject := NewDataNavigator(log) - t.Run("TestReadMap_simple", func(t *testing.T) { - var data = test.ParseData(` ---- -b: - c: 2 -`) - got, _ := subject.ReadChildValue(data, []string{"b", "c"}) - test.AssertResult(t, 2, got) - }) +// t.Run("TestReadMap_simple", func(t *testing.T) { +// var data = test.ParseData(` +// --- +// b: +// c: 2 +// `) +// got, _ := subject.ReadChildValue(data, []string{"b", "c"}) +// test.AssertResult(t, 2, got) +// }) - t.Run("TestReadMap_numberKey", func(t *testing.T) { - var data = test.ParseData(` ---- -200: things -`) - got, _ := subject.ReadChildValue(data, []string{"200"}) - test.AssertResult(t, "things", got) - }) +// t.Run("TestReadMap_numberKey", func(t *testing.T) { +// var data = test.ParseData(` +// --- +// 200: things +// `) +// got, _ := subject.ReadChildValue(data, []string{"200"}) +// test.AssertResult(t, "things", got) +// }) - t.Run("TestReadMap_splat", func(t *testing.T) { - var data = test.ParseData(` ---- -mapSplat: - item1: things - item2: whatever - otherThing: cat -`) - res, _ := subject.ReadChildValue(data, []string{"mapSplat", "*"}) - test.AssertResult(t, "[things whatever cat]", fmt.Sprintf("%v", res)) - }) +// t.Run("TestReadMap_splat", func(t *testing.T) { +// var data = test.ParseData(` +// --- +// mapSplat: +// item1: things +// item2: whatever +// otherThing: cat +// `) +// res, _ := subject.ReadChildValue(data, []string{"mapSplat", "*"}) +// test.AssertResult(t, "[things whatever cat]", fmt.Sprintf("%v", res)) +// }) - t.Run("TestReadMap_prefixSplat", func(t *testing.T) { - var data = test.ParseData(` ---- -mapSplat: - item1: things - item2: whatever - otherThing: cat -`) - res, _ := subject.ReadChildValue(data, []string{"mapSplat", "item*"}) - test.AssertResult(t, "[things whatever]", fmt.Sprintf("%v", res)) - }) +// t.Run("TestReadMap_prefixSplat", func(t *testing.T) { +// var data = test.ParseData(` +// --- +// mapSplat: +// item1: things +// item2: whatever +// otherThing: cat +// `) +// res, _ := subject.ReadChildValue(data, []string{"mapSplat", "item*"}) +// test.AssertResult(t, "[things whatever]", fmt.Sprintf("%v", res)) +// }) - t.Run("TestReadMap_deep_splat", func(t *testing.T) { - var data = test.ParseData(` ---- -mapSplatDeep: - item1: - cats: bananas - item2: - cats: apples -`) +// t.Run("TestReadMap_deep_splat", func(t *testing.T) { +// var data = test.ParseData(` +// --- +// mapSplatDeep: +// item1: +// cats: bananas +// item2: +// cats: apples +// `) - res, _ := subject.ReadChildValue(data, []string{"mapSplatDeep", "*", "cats"}) - result := res.([]interface{}) - var actual = []string{result[0].(string), result[1].(string)} - sort.Strings(actual) - test.AssertResult(t, "[apples bananas]", fmt.Sprintf("%v", actual)) - }) +// res, _ := subject.ReadChildValue(data, []string{"mapSplatDeep", "*", "cats"}) +// result := res.([]interface{}) +// var actual = []string{result[0].(string), result[1].(string)} +// sort.Strings(actual) +// test.AssertResult(t, "[apples bananas]", fmt.Sprintf("%v", actual)) +// }) - t.Run("TestReadMap_key_doesnt_exist", func(t *testing.T) { - var data = test.ParseData(` ---- -b: - c: 2 -`) - got, _ := subject.ReadChildValue(data, []string{"b", "x", "f", "c"}) - test.AssertResult(t, nil, got) - }) +// t.Run("TestReadMap_key_doesnt_exist", func(t *testing.T) { +// var data = test.ParseData(` +// --- +// b: +// c: 2 +// `) +// got, _ := subject.ReadChildValue(data, []string{"b", "x", "f", "c"}) +// test.AssertResult(t, nil, got) +// }) - t.Run("TestReadMap_recurse_against_string", func(t *testing.T) { - var data = test.ParseData(` ---- -a: cat -`) - got, _ := subject.ReadChildValue(data, []string{"a", "b"}) - test.AssertResult(t, nil, got) - }) +// t.Run("TestReadMap_recurse_against_string", func(t *testing.T) { +// var data = test.ParseData(` +// --- +// a: cat +// `) +// got, _ := subject.ReadChildValue(data, []string{"a", "b"}) +// test.AssertResult(t, nil, got) +// }) - t.Run("TestReadMap_with_array", func(t *testing.T) { - var data = test.ParseData(` ---- -b: - d: - - 3 - - 4 -`) - got, _ := subject.ReadChildValue(data, []string{"b", "d", "1"}) - test.AssertResult(t, 4, got) - }) +// t.Run("TestReadMap_with_array", func(t *testing.T) { +// var data = test.ParseData(` +// --- +// b: +// d: +// - 3 +// - 4 +// `) +// got, _ := subject.ReadChildValue(data, []string{"b", "d", "1"}) +// test.AssertResult(t, 4, got) +// }) - t.Run("TestReadMap_with_array_and_bad_index", func(t *testing.T) { - var data = test.ParseData(` ---- -b: - d: - - 3 - - 4 -`) - _, err := subject.ReadChildValue(data, []string{"b", "d", "x"}) - if err == nil { - t.Fatal("Expected error due to invalid path") - } - expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax` - test.AssertResult(t, expectedOutput, err.Error()) - }) +// t.Run("TestReadMap_with_array_and_bad_index", func(t *testing.T) { +// var data = test.ParseData(` +// --- +// b: +// d: +// - 3 +// - 4 +// `) +// _, err := subject.ReadChildValue(data, []string{"b", "d", "x"}) +// if err == nil { +// t.Fatal("Expected error due to invalid path") +// } +// expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax` +// test.AssertResult(t, expectedOutput, err.Error()) +// }) - t.Run("TestReadMap_with_mapsplat_array_and_bad_index", func(t *testing.T) { - var data = test.ParseData(` ---- -b: - d: - e: - - 3 - - 4 - f: - - 1 - - 2 -`) - _, err := subject.ReadChildValue(data, []string{"b", "d", "*", "x"}) - if err == nil { - t.Fatal("Expected error due to invalid path") - } - expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax` - test.AssertResult(t, expectedOutput, err.Error()) - }) +// t.Run("TestReadMap_with_mapsplat_array_and_bad_index", func(t *testing.T) { +// var data = test.ParseData(` +// --- +// b: +// d: +// e: +// - 3 +// - 4 +// f: +// - 1 +// - 2 +// `) +// _, err := subject.ReadChildValue(data, []string{"b", "d", "*", "x"}) +// if err == nil { +// t.Fatal("Expected error due to invalid path") +// } +// expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax` +// test.AssertResult(t, expectedOutput, err.Error()) +// }) - t.Run("TestReadMap_with_arraysplat_map_array_and_bad_index", func(t *testing.T) { - var data = test.ParseData(` ---- -b: - d: - - names: - - fred - - smith - - names: - - sam - - bo -`) - _, err := subject.ReadChildValue(data, []string{"b", "d", "*", "names", "x"}) - if err == nil { - t.Fatal("Expected error due to invalid path") - } - expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax` - test.AssertResult(t, expectedOutput, err.Error()) - }) +// t.Run("TestReadMap_with_arraysplat_map_array_and_bad_index", func(t *testing.T) { +// var data = test.ParseData(` +// --- +// b: +// d: +// - names: +// - fred +// - smith +// - names: +// - sam +// - bo +// `) +// _, err := subject.ReadChildValue(data, []string{"b", "d", "*", "names", "x"}) +// if err == nil { +// t.Fatal("Expected error due to invalid path") +// } +// expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax` +// test.AssertResult(t, expectedOutput, err.Error()) +// }) - t.Run("TestReadMap_with_array_out_of_bounds", func(t *testing.T) { - var data = test.ParseData(` ---- -b: - d: - - 3 - - 4 -`) - got, _ := subject.ReadChildValue(data, []string{"b", "d", "3"}) - test.AssertResult(t, nil, got) - }) +// t.Run("TestReadMap_with_array_out_of_bounds", func(t *testing.T) { +// var data = test.ParseData(` +// --- +// b: +// d: +// - 3 +// - 4 +// `) +// got, _ := subject.ReadChildValue(data, []string{"b", "d", "3"}) +// test.AssertResult(t, nil, got) +// }) - t.Run("TestReadMap_with_array_out_of_bounds_by_1", func(t *testing.T) { - var data = test.ParseData(` ---- -b: - d: - - 3 - - 4 -`) - got, _ := subject.ReadChildValue(data, []string{"b", "d", "2"}) - test.AssertResult(t, nil, got) - }) +// t.Run("TestReadMap_with_array_out_of_bounds_by_1", func(t *testing.T) { +// var data = test.ParseData(` +// --- +// b: +// d: +// - 3 +// - 4 +// `) +// got, _ := subject.ReadChildValue(data, []string{"b", "d", "2"}) +// test.AssertResult(t, nil, got) +// }) - t.Run("TestReadMap_with_array_splat", func(t *testing.T) { - var data = test.ParseData(` -e: - - - name: Fred - thing: cat - - - name: Sam - thing: dog -`) - got, _ := subject.ReadChildValue(data, []string{"e", "*", "name"}) - test.AssertResult(t, "[Fred Sam]", fmt.Sprintf("%v", got)) - }) +// t.Run("TestReadMap_with_array_splat", func(t *testing.T) { +// var data = test.ParseData(` +// e: +// - +// name: Fred +// thing: cat +// - +// name: Sam +// thing: dog +// `) +// got, _ := subject.ReadChildValue(data, []string{"e", "*", "name"}) +// test.AssertResult(t, "[Fred Sam]", fmt.Sprintf("%v", got)) +// }) - t.Run("TestWrite_really_simple", func(t *testing.T) { - var data = test.ParseData(` -b: 2 -`) +// t.Run("TestWrite_really_simple", func(t *testing.T) { +// var data = test.ParseData(` +// b: 2 +// `) - updated := subject.UpdatedChildValue(data, []string{"b"}, "4") - test.AssertResult(t, "[{b 4}]", fmt.Sprintf("%v", updated)) - }) +// updated := subject.UpdatedChildValue(data, []string{"b"}, "4") +// test.AssertResult(t, "[{b 4}]", fmt.Sprintf("%v", updated)) +// }) - t.Run("TestWrite_simple", func(t *testing.T) { - var data = test.ParseData(` -b: - c: 2 -`) +// t.Run("TestWrite_simple", func(t *testing.T) { +// var data = test.ParseData(` +// b: +// c: 2 +// `) - updated := subject.UpdatedChildValue(data, []string{"b", "c"}, "4") - test.AssertResult(t, "[{b [{c 4}]}]", fmt.Sprintf("%v", updated)) - }) +// updated := subject.UpdatedChildValue(data, []string{"b", "c"}, "4") +// test.AssertResult(t, "[{b [{c 4}]}]", fmt.Sprintf("%v", updated)) +// }) - t.Run("TestWrite_new", func(t *testing.T) { - var data = test.ParseData(` -b: - c: 2 -`) +// t.Run("TestWrite_new", func(t *testing.T) { +// var data = test.ParseData(` +// b: +// c: 2 +// `) - updated := subject.UpdatedChildValue(data, []string{"b", "d"}, "4") - test.AssertResult(t, "[{b [{c 2} {d 4}]}]", fmt.Sprintf("%v", updated)) - }) +// updated := subject.UpdatedChildValue(data, []string{"b", "d"}, "4") +// test.AssertResult(t, "[{b [{c 2} {d 4}]}]", fmt.Sprintf("%v", updated)) +// }) - t.Run("TestWrite_new_deep", func(t *testing.T) { - var data = test.ParseData(` -b: - c: 2 -`) +// t.Run("TestWrite_new_deep", func(t *testing.T) { +// var data = test.ParseData(` +// b: +// c: 2 +// `) - updated := subject.UpdatedChildValue(data, []string{"b", "d", "f"}, "4") - test.AssertResult(t, "[{b [{c 2} {d [{f 4}]}]}]", fmt.Sprintf("%v", updated)) - }) +// updated := subject.UpdatedChildValue(data, []string{"b", "d", "f"}, "4") +// test.AssertResult(t, "[{b [{c 2} {d [{f 4}]}]}]", fmt.Sprintf("%v", updated)) +// }) - t.Run("TestWrite_array", func(t *testing.T) { - var data = test.ParseData(` -b: - - aa -`) +// t.Run("TestWrite_array", func(t *testing.T) { +// var data = test.ParseData(` +// b: +// - aa +// `) - updated := subject.UpdatedChildValue(data, []string{"b", "0"}, "bb") +// updated := subject.UpdatedChildValue(data, []string{"b", "0"}, "bb") - test.AssertResult(t, "[{b [bb]}]", fmt.Sprintf("%v", updated)) - }) +// test.AssertResult(t, "[{b [bb]}]", fmt.Sprintf("%v", updated)) +// }) - t.Run("TestWrite_new_array", func(t *testing.T) { - var data = test.ParseData(` -b: - c: 2 -`) +// t.Run("TestWrite_new_array", func(t *testing.T) { +// var data = test.ParseData(` +// b: +// c: 2 +// `) - updated := subject.UpdatedChildValue(data, []string{"b", "0"}, "4") - test.AssertResult(t, "[{b [{c 2} {0 4}]}]", fmt.Sprintf("%v", updated)) - }) +// updated := subject.UpdatedChildValue(data, []string{"b", "0"}, "4") +// test.AssertResult(t, "[{b [{c 2} {0 4}]}]", fmt.Sprintf("%v", updated)) +// }) - t.Run("TestWrite_new_array_deep", func(t *testing.T) { - var data = test.ParseData(` -a: apple -`) +// t.Run("TestWrite_new_array_deep", func(t *testing.T) { +// var data = test.ParseData(` +// a: apple +// `) - updated := subject.UpdatedChildValue(data, []string{"b", "+", "c"}, "4") - test.AssertResult(t, "[{a apple} {b [[{c 4}]]}]", fmt.Sprintf("%v", updated)) - }) +// updated := subject.UpdatedChildValue(data, []string{"b", "+", "c"}, "4") +// test.AssertResult(t, "[{a apple} {b [[{c 4}]]}]", fmt.Sprintf("%v", updated)) +// }) - t.Run("TestWrite_new_map_array_deep", func(t *testing.T) { - var data = test.ParseData(` -b: - c: 2 -`) +// t.Run("TestWrite_new_map_array_deep", func(t *testing.T) { +// var data = test.ParseData(` +// b: +// c: 2 +// `) - updated := subject.UpdatedChildValue(data, []string{"b", "d", "+"}, "4") - test.AssertResult(t, "[{b [{c 2} {d [4]}]}]", fmt.Sprintf("%v", updated)) - }) +// updated := subject.UpdatedChildValue(data, []string{"b", "d", "+"}, "4") +// test.AssertResult(t, "[{b [{c 2} {d [4]}]}]", fmt.Sprintf("%v", updated)) +// }) - t.Run("TestWrite_add_to_array", func(t *testing.T) { - var data = test.ParseData(` -b: - - aa -`) +// t.Run("TestWrite_add_to_array", func(t *testing.T) { +// var data = test.ParseData(` +// b: +// - aa +// `) - updated := subject.UpdatedChildValue(data, []string{"b", "1"}, "bb") - test.AssertResult(t, "[{b [aa bb]}]", fmt.Sprintf("%v", updated)) - }) +// updated := subject.UpdatedChildValue(data, []string{"b", "1"}, "bb") +// test.AssertResult(t, "[{b [aa bb]}]", fmt.Sprintf("%v", updated)) +// }) - t.Run("TestWrite_with_no_tail", func(t *testing.T) { - var data = test.ParseData(` -b: - c: 2 -`) - updated := subject.UpdatedChildValue(data, []string{"b"}, "4") +// t.Run("TestWrite_with_no_tail", func(t *testing.T) { +// var data = test.ParseData(` +// b: +// c: 2 +// `) +// updated := subject.UpdatedChildValue(data, []string{"b"}, "4") - test.AssertResult(t, "[{b 4}]", fmt.Sprintf("%v", updated)) - }) +// test.AssertResult(t, "[{b 4}]", fmt.Sprintf("%v", updated)) +// }) - t.Run("TestWriteMap_no_paths", func(t *testing.T) { - var data = test.ParseData(` -b: 5 -`) - var new = test.ParseData(` -c: 4 -`) - result := subject.UpdatedChildValue(data, []string{}, new) - test.AssertResult(t, fmt.Sprintf("%v", new), fmt.Sprintf("%v", result)) - }) +// t.Run("TestWriteMap_no_paths", func(t *testing.T) { +// var data = test.ParseData(` +// b: 5 +// `) +// var new = test.ParseData(` +// c: 4 +// `) +// result := subject.UpdatedChildValue(data, []string{}, new) +// test.AssertResult(t, fmt.Sprintf("%v", new), fmt.Sprintf("%v", result)) +// }) - t.Run("TestWriteArray_no_paths", func(t *testing.T) { - var data = make([]interface{}, 1) - data[0] = "mike" - var new = test.ParseData(` -c: 4 -`) - result := subject.UpdatedChildValue(data, []string{}, new) - test.AssertResult(t, fmt.Sprintf("%v", new), fmt.Sprintf("%v", result)) - }) +// t.Run("TestWriteArray_no_paths", func(t *testing.T) { +// var data = make([]interface{}, 1) +// data[0] = "mike" +// var new = test.ParseData(` +// c: 4 +// `) +// result := subject.UpdatedChildValue(data, []string{}, new) +// test.AssertResult(t, fmt.Sprintf("%v", new), fmt.Sprintf("%v", result)) +// }) - t.Run("TestDelete_MapItem", func(t *testing.T) { - var data = test.ParseData(` -a: 123 -b: 456 -`) - var expected = test.ParseData(` -b: 456 -`) +// t.Run("TestDelete_MapItem", func(t *testing.T) { +// var data = test.ParseData(` +// a: 123 +// b: 456 +// `) +// var expected = test.ParseData(` +// b: 456 +// `) - result, _ := subject.DeleteChildValue(data, []string{"a"}) - test.AssertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result)) - }) +// result, _ := subject.DeleteChildValue(data, []string{"a"}) +// test.AssertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result)) +// }) - // Ensure deleting an index into a string does nothing - t.Run("TestDelete_index_to_string", func(t *testing.T) { - var data = test.ParseData(` -a: mystring -`) - result, _ := subject.DeleteChildValue(data, []string{"a", "0"}) - test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result)) - }) +// // Ensure deleting an index into a string does nothing +// t.Run("TestDelete_index_to_string", func(t *testing.T) { +// var data = test.ParseData(` +// a: mystring +// `) +// result, _ := subject.DeleteChildValue(data, []string{"a", "0"}) +// test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result)) +// }) - t.Run("TestDelete_list_index", func(t *testing.T) { - var data = test.ParseData(` -a: [3, 4] -`) - var expected = test.ParseData(` -a: [3] -`) - result, _ := subject.DeleteChildValue(data, []string{"a", "1"}) - test.AssertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result)) - }) +// t.Run("TestDelete_list_index", func(t *testing.T) { +// var data = test.ParseData(` +// a: [3, 4] +// `) +// var expected = test.ParseData(` +// a: [3] +// `) +// result, _ := subject.DeleteChildValue(data, []string{"a", "1"}) +// test.AssertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result)) +// }) - t.Run("TestDelete_list_index_beyond_bounds", func(t *testing.T) { - var data = test.ParseData(` -a: [3, 4] -`) - result, _ := subject.DeleteChildValue(data, []string{"a", "5"}) - test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result)) - }) +// t.Run("TestDelete_list_index_beyond_bounds", func(t *testing.T) { +// var data = test.ParseData(` +// a: [3, 4] +// `) +// result, _ := subject.DeleteChildValue(data, []string{"a", "5"}) +// test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result)) +// }) - t.Run("TestDelete_list_index_out_of_bounds_by_1", func(t *testing.T) { - var data = test.ParseData(` -a: [3, 4] -`) - result, _ := subject.DeleteChildValue(data, []string{"a", "2"}) - test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result)) - }) +// t.Run("TestDelete_list_index_out_of_bounds_by_1", func(t *testing.T) { +// var data = test.ParseData(` +// a: [3, 4] +// `) +// result, _ := subject.DeleteChildValue(data, []string{"a", "2"}) +// test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result)) +// }) - t.Run("TestDelete_no_paths", func(t *testing.T) { - var data = test.ParseData(` -a: [3, 4] -b: - - name: test -`) - result, _ := subject.DeleteChildValue(data, []string{}) - test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result)) - }) +// t.Run("TestDelete_no_paths", func(t *testing.T) { +// var data = test.ParseData(` +// a: [3, 4] +// b: +// - name: test +// `) +// result, _ := subject.DeleteChildValue(data, []string{}) +// test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result)) +// }) - t.Run("TestDelete_array_map_item", func(t *testing.T) { - var data = test.ParseData(` -b: -- name: fred - value: blah -- name: john - value: test -`) - var expected = test.ParseData(` -b: -- value: blah -- name: john - value: test -`) - result, _ := subject.DeleteChildValue(data, []string{"b", "0", "name"}) - test.AssertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result)) - }) -} +// t.Run("TestDelete_array_map_item", func(t *testing.T) { +// var data = test.ParseData(` +// b: +// - name: fred +// value: blah +// - name: john +// value: test +// `) +// var expected = test.ParseData(` +// b: +// - value: blah +// - name: john +// value: test +// `) +// result, _ := subject.DeleteChildValue(data, []string{"b", "0", "name"}) +// test.AssertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result)) +// }) +// } diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 5b630e82..10d30eec 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -14,6 +14,7 @@ type UpdateCommand struct { } type YqLib interface { + DebugNode(node *yaml.Node) Get(rootNode *yaml.Node, path string) (*yaml.Node, error) Update(rootNode *yaml.Node, updateCommand UpdateCommand) error } @@ -32,10 +33,11 @@ func NewYqLib(l *logging.Logger) YqLib { } } +func (l *lib) DebugNode(node *yaml.Node) { + l.navigator.DebugNode(node) +} + func (l *lib) Get(rootNode *yaml.Node, path string) (*yaml.Node, error) { - if path == "" { - return rootNode, nil - } var paths = l.parser.ParsePath(path) return l.navigator.Get(rootNode, paths) } diff --git a/pkg/yqlib/lib_test.go b/pkg/yqlib/lib_test.go index bcfac5ff..b9c3e4de 100644 --- a/pkg/yqlib/lib_test.go +++ b/pkg/yqlib/lib_test.go @@ -4,7 +4,7 @@ import ( "fmt" "testing" - "github.com/mikefarah/yq/v2/test" + "github.com/mikefarah/yq/v3/test" logging "gopkg.in/op/go-logging.v1" ) diff --git a/pkg/yqlib/path_parser.go b/pkg/yqlib/path_parser.go index f8163cca..83157054 100644 --- a/pkg/yqlib/path_parser.go +++ b/pkg/yqlib/path_parser.go @@ -11,6 +11,9 @@ func NewPathParser() PathParser { } func (p *pathParser) ParsePath(path string) []string { + if path == "" { + return []string{} + } return p.parsePathAccum([]string{}, path) } diff --git a/pkg/yqlib/path_parser_test.go b/pkg/yqlib/path_parser_test.go index c2d7fe26..460d3fe9 100644 --- a/pkg/yqlib/path_parser_test.go +++ b/pkg/yqlib/path_parser_test.go @@ -3,7 +3,7 @@ package yqlib import ( "testing" - "github.com/mikefarah/yq/v2/test" + "github.com/mikefarah/yq/v3/test" ) var parsePathsTests = []struct { diff --git a/pkg/yqlib/value_parser_test.go b/pkg/yqlib/value_parser_test.go index 2246b398..24468e37 100644 --- a/pkg/yqlib/value_parser_test.go +++ b/pkg/yqlib/value_parser_test.go @@ -3,7 +3,7 @@ package yqlib import ( "testing" - "github.com/mikefarah/yq/v2/test" + "github.com/mikefarah/yq/v3/test" ) var parseValueTests = []struct { diff --git a/test/utils.go b/test/utils.go index 490ce543..25093e2f 100644 --- a/test/utils.go +++ b/test/utils.go @@ -9,8 +9,8 @@ import ( "strings" "testing" - yaml "github.com/mikefarah/yaml/v2" "github.com/spf13/cobra" + yaml "gopkg.in/yaml.v3" ) type resulter struct { @@ -30,8 +30,8 @@ func RunCmd(c *cobra.Command, input string) resulter { return resulter{err, output, c} } -func ParseData(rawData string) yaml.MapSlice { - var parsedData yaml.MapSlice +func ParseData(rawData string) yaml.Node { + var parsedData yaml.Node err := yaml.Unmarshal([]byte(rawData), &parsedData) if err != nil { fmt.Printf("Error parsing yaml: %v\n", err) diff --git a/yq.go b/yq.go index d61dc7e1..7fec3c1c 100644 --- a/yq.go +++ b/yq.go @@ -257,11 +257,14 @@ func readProperty(cmd *cobra.Command, args []string) error { } var mappedDocs []*yaml.Node - var dataBucket yaml.Node + var currentIndex = 0 var errorReadingStream = readStream(args[0], func(decoder *yaml.Decoder) error { for { + var dataBucket yaml.Node errorReading := decoder.Decode(&dataBucket) + log.Debugf("decoded node for doc %v", currentIndex) + lib.DebugNode(&dataBucket) if errorReading == io.EOF { log.Debugf("done %v / %v", currentIndex, docIndexInt) if !updateAll && currentIndex <= docIndexInt { @@ -273,6 +276,7 @@ func readProperty(cmd *cobra.Command, args []string) error { if updateAll || currentIndex == docIndexInt { log.Debugf("reading %v in document %v", path, currentIndex) mappedDoc, errorParsing := lib.Get(&dataBucket, path) + lib.DebugNode(mappedDoc) if errorParsing != nil { return errors.Wrapf(errorParsing, "Error reading path in document index %v", currentIndex) } diff --git a/yq_test.go b/yq_test.go index 677f7bd0..b131d01f 100644 --- a/yq_test.go +++ b/yq_test.go @@ -1,60 +1,60 @@ package main -import ( - "fmt" - "runtime" - "testing" +// import ( +// "fmt" +// "runtime" +// "testing" - "github.com/mikefarah/yq/v2/pkg/marshal" - "github.com/mikefarah/yq/v2/test" -) +// "github.com/mikefarah/yq/v2/pkg/marshal" +// "github.com/mikefarah/yq/v2/test" +// ) -func TestMultilineString(t *testing.T) { - testString := ` - abcd - efg` - formattedResult, _ := marshal.NewYamlConverter().YamlToString(testString, false) - test.AssertResult(t, testString, formattedResult) -} +// func TestMultilineString(t *testing.T) { +// testString := ` +// abcd +// efg` +// formattedResult, _ := marshal.NewYamlConverter().YamlToString(testString, false) +// test.AssertResult(t, testString, formattedResult) +// } -func TestNewYaml(t *testing.T) { - result, _ := newYaml([]string{"b.c", "3"}) - formattedResult := fmt.Sprintf("%v", result) - test.AssertResult(t, - "[{b [{c 3}]}]", - formattedResult) -} +// func TestNewYaml(t *testing.T) { +// result, _ := newYaml([]string{"b.c", "3"}) +// formattedResult := fmt.Sprintf("%v", result) +// test.AssertResult(t, +// "[{b [{c 3}]}]", +// formattedResult) +// } -func TestNewYamlArray(t *testing.T) { - result, _ := newYaml([]string{"[0].cat", "meow"}) - formattedResult := fmt.Sprintf("%v", result) - test.AssertResult(t, - "[[{cat meow}]]", - formattedResult) -} +// func TestNewYamlArray(t *testing.T) { +// result, _ := newYaml([]string{"[0].cat", "meow"}) +// formattedResult := fmt.Sprintf("%v", result) +// test.AssertResult(t, +// "[[{cat meow}]]", +// formattedResult) +// } -func TestNewYaml_WithScript(t *testing.T) { - writeScript = "examples/instruction_sample.yaml" - expectedResult := `b: - c: cat - e: - - name: Mike Farah` - result, _ := newYaml([]string{""}) - actualResult, _ := marshal.NewYamlConverter().YamlToString(result, true) - test.AssertResult(t, expectedResult, actualResult) -} +// func TestNewYaml_WithScript(t *testing.T) { +// writeScript = "examples/instruction_sample.yaml" +// expectedResult := `b: +// c: cat +// e: +// - name: Mike Farah` +// result, _ := newYaml([]string{""}) +// actualResult, _ := marshal.NewYamlConverter().YamlToString(result, true) +// test.AssertResult(t, expectedResult, actualResult) +// } -func TestNewYaml_WithUnknownScript(t *testing.T) { - writeScript = "fake-unknown" - _, err := newYaml([]string{""}) - if err == nil { - t.Error("Expected error due to unknown file") - } - var expectedOutput string - if runtime.GOOS == "windows" { - expectedOutput = `open fake-unknown: The system cannot find the file specified.` - } else { - expectedOutput = `open fake-unknown: no such file or directory` - } - test.AssertResult(t, expectedOutput, err.Error()) -} +// func TestNewYaml_WithUnknownScript(t *testing.T) { +// writeScript = "fake-unknown" +// _, err := newYaml([]string{""}) +// if err == nil { +// t.Error("Expected error due to unknown file") +// } +// var expectedOutput string +// if runtime.GOOS == "windows" { +// expectedOutput = `open fake-unknown: The system cannot find the file specified.` +// } else { +// expectedOutput = `open fake-unknown: no such file or directory` +// } +// test.AssertResult(t, expectedOutput, err.Error()) +// }