splatting

This commit is contained in:
Mike Farah 2019-12-09 13:44:53 +11:00
parent 8da9a81702
commit 9771e7001c
13 changed files with 527 additions and 515 deletions

View File

@ -7,7 +7,7 @@ import (
"strings" "strings"
"testing" "testing"
"github.com/mikefarah/yq/v2/test" "github.com/mikefarah/yq/v3/test"
"github.com/spf13/cobra" "github.com/spf13/cobra"
) )
@ -63,30 +63,6 @@ func TestRootCmd_VerboseShort(t *testing.T) {
} }
} }
func TestRootCmd_TrimLong(t *testing.T) {
cmd := getRootCommand()
result := test.RunCmd(cmd, "--trim")
if result.Error != nil {
t.Error(result.Error)
}
if !trimOutput {
t.Error("Expected trimOutput to be true")
}
}
func TestRootCmd_TrimShort(t *testing.T) {
cmd := getRootCommand()
result := test.RunCmd(cmd, "-t")
if result.Error != nil {
t.Error(result.Error)
}
if !trimOutput {
t.Error("Expected trimOutput to be true")
}
}
func TestRootCmd_VersionShort(t *testing.T) { func TestRootCmd_VersionShort(t *testing.T) {
cmd := getRootCommand() cmd := getRootCommand()
result := test.RunCmd(cmd, "-V") result := test.RunCmd(cmd, "-V")
@ -191,7 +167,7 @@ func TestReadCmd_ArrayYaml_NoPath(t *testing.T) {
expectedOutput := `- become: true expectedOutput := `- become: true
gather_facts: false gather_facts: false
hosts: lalaland hosts: lalaland
name: Apply smth name: "Apply smth"
roles: roles:
- lala - lala
- land - land
@ -209,7 +185,7 @@ func TestReadCmd_ArrayYaml_OneElement(t *testing.T) {
expectedOutput := `become: true expectedOutput := `become: true
gather_facts: false gather_facts: false
hosts: lalaland hosts: lalaland
name: Apply smth name: "Apply smth"
roles: roles:
- lala - lala
- land - land
@ -227,7 +203,7 @@ func TestReadCmd_ArrayYaml_Splat(t *testing.T) {
expectedOutput := `- become: true expectedOutput := `- become: true
gather_facts: false gather_facts: false
hosts: lalaland hosts: lalaland
name: Apply smth name: "Apply smth"
roles: roles:
- lala - lala
- land - land
@ -252,19 +228,19 @@ func TestReadCmd_ArrayYaml_ErrorBadPath(t *testing.T) {
if result.Error == nil { if result.Error == nil {
t.Error("Expected command to fail due to invalid path") t.Error("Expected command to fail due to invalid path")
} }
expectedOutput := `Error reading path in document index 0: error accessing array: strconv.ParseInt: parsing "x": invalid syntax` expectedOutput := `Error reading path in document index 0: strconv.ParseInt: parsing "x": invalid syntax`
test.AssertResult(t, expectedOutput, result.Error.Error()) test.AssertResult(t, expectedOutput, result.Error.Error())
} }
func TestReadCmd_ArrayYaml_Splat_ErrorBadPath(t *testing.T) { // func TestReadCmd_ArrayYaml_Splat_ErrorBadPath(t *testing.T) {
cmd := getRootCommand() // cmd := getRootCommand()
result := test.RunCmd(cmd, "read examples/array.yaml [*].roles[x]") // result := test.RunCmd(cmd, "read examples/array.yaml [*].roles[x]")
if result.Error == nil { // if result.Error == nil {
t.Error("Expected command to fail due to invalid path") // t.Error("Expected command to fail due to invalid path")
} // }
expectedOutput := `Error reading path in document index 0: error accessing array: strconv.ParseInt: parsing "x": invalid syntax` // expectedOutput := `Error reading path in document index 0: error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
test.AssertResult(t, expectedOutput, result.Error.Error()) // test.AssertResult(t, expectedOutput, result.Error.Error())
} // }
func TestReadCmd_Error(t *testing.T) { func TestReadCmd_Error(t *testing.T) {
cmd := getRootCommand() cmd := getRootCommand()
@ -301,27 +277,27 @@ func TestReadCmd_ErrorUnreadableFile(t *testing.T) {
test.AssertResult(t, expectedOutput, result.Error.Error()) test.AssertResult(t, expectedOutput, result.Error.Error())
} }
func TestReadCmd_ErrorBadPath(t *testing.T) { // func TestReadCmd_ErrorBadPath(t *testing.T) {
content := `b: // content := `b:
d: // d:
e: // e:
- 3 // - 3
- 4 // - 4
f: // f:
- 1 // - 1
- 2 // - 2
` // `
filename := test.WriteTempYamlFile(content) // filename := test.WriteTempYamlFile(content)
defer test.RemoveTempYamlFile(filename) // defer test.RemoveTempYamlFile(filename)
cmd := getRootCommand() // cmd := getRootCommand()
result := test.RunCmd(cmd, fmt.Sprintf("read %s b.d.*.[x]", filename)) // result := test.RunCmd(cmd, fmt.Sprintf("read %s b.d.*.[x]", filename))
if result.Error == nil { // if result.Error == nil {
t.Fatal("Expected command to fail due to invalid path") // t.Fatal("Expected command to fail due to invalid path")
} // }
expectedOutput := `Error reading path in document index 0: error accessing array: strconv.ParseInt: parsing "x": invalid syntax` // expectedOutput := `Error reading path in document index 0: error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
test.AssertResult(t, expectedOutput, result.Error.Error()) // test.AssertResult(t, expectedOutput, result.Error.Error())
} // }
func TestReadCmd_Verbose(t *testing.T) { func TestReadCmd_Verbose(t *testing.T) {
cmd := getRootCommand() cmd := getRootCommand()
@ -332,32 +308,23 @@ func TestReadCmd_Verbose(t *testing.T) {
test.AssertResult(t, "2\n", result.Output) test.AssertResult(t, "2\n", result.Output)
} }
func TestReadCmd_NoTrim(t *testing.T) { // func TestReadCmd_ToJson(t *testing.T) {
cmd := getRootCommand() // cmd := getRootCommand()
result := test.RunCmd(cmd, "--trim=false read examples/sample.yaml b.c") // result := test.RunCmd(cmd, "read -j examples/sample.yaml b.c")
if result.Error != nil { // if result.Error != nil {
t.Error(result.Error) // t.Error(result.Error)
} // }
test.AssertResult(t, "2\n\n", result.Output) // test.AssertResult(t, "2\n", result.Output)
} // }
func TestReadCmd_ToJson(t *testing.T) { // func TestReadCmd_ToJsonLong(t *testing.T) {
cmd := getRootCommand() // cmd := getRootCommand()
result := test.RunCmd(cmd, "read -j examples/sample.yaml b.c") // result := test.RunCmd(cmd, "read --tojson examples/sample.yaml b.c")
if result.Error != nil { // if result.Error != nil {
t.Error(result.Error) // t.Error(result.Error)
} // }
test.AssertResult(t, "2\n", result.Output) // test.AssertResult(t, "2\n", result.Output)
} // }
func TestReadCmd_ToJsonLong(t *testing.T) {
cmd := getRootCommand()
result := test.RunCmd(cmd, "read --tojson examples/sample.yaml b.c")
if result.Error != nil {
t.Error(result.Error)
}
test.AssertResult(t, "2\n", result.Output)
}
func TestPrefixCmd(t *testing.T) { func TestPrefixCmd(t *testing.T) {
content := `b: content := `b:
@ -851,7 +818,7 @@ func TestWriteCmd_SplatMapEmpty(t *testing.T) {
t.Error(result.Error) t.Error(result.Error)
} }
expectedOutput := `b: expectedOutput := `b:
c: thing c: {}
d: another thing d: another thing
` `
test.AssertResult(t, expectedOutput, result.Output) test.AssertResult(t, expectedOutput, result.Output)

13
compare.sh Executable file
View File

@ -0,0 +1,13 @@
GREEN='\033[0;32m'
NC='\033[0m'
echo "${GREEN}---Old---${NC}"
yq $@ > /tmp/yq-old-output
cat /tmp/yq-old-output
echo "${GREEN}---New---${NC}"
./yq $@ > /tmp/yq-new-output
cat /tmp/yq-new-output
echo "${GREEN}---Diff---${NC}"
colordiff /tmp/yq-old-output /tmp/yq-new-output

View File

@ -1,9 +1,8 @@
a: Easy! as one two three a: Easy! as one two three
b: b:
c: 2 c:
d: [3, 4] name: c1
e: f: things
- name: fred d:
value: 3 name: d1
- name: sam f: other
value: 4

View File

@ -9,6 +9,7 @@ import (
) )
type DataNavigator interface { type DataNavigator interface {
DebugNode(node *yaml.Node)
Get(rootNode *yaml.Node, remainingPath []string) (*yaml.Node, error) Get(rootNode *yaml.Node, remainingPath []string) (*yaml.Node, error)
Update(rootNode *yaml.Node, remainingPath []string, changesToApply yaml.Node) error Update(rootNode *yaml.Node, remainingPath []string, changesToApply yaml.Node) error
} }
@ -36,9 +37,9 @@ func (n *navigator) Get(value *yaml.Node, path []string) (*yaml.Node, error) {
func (n *navigator) Update(value *yaml.Node, path []string, changesToApply yaml.Node) error { func (n *navigator) Update(value *yaml.Node, path []string, changesToApply yaml.Node) error {
_, errorVisiting := n.Visit(value, path, func(nodeToUpdate *yaml.Node) (*yaml.Node, error) { _, errorVisiting := n.Visit(value, path, func(nodeToUpdate *yaml.Node) (*yaml.Node, error) {
n.log.Debug("going to update") n.log.Debug("going to update")
n.debugNode(nodeToUpdate) n.DebugNode(nodeToUpdate)
n.log.Debug("with") n.log.Debug("with")
n.debugNode(&changesToApply) n.DebugNode(&changesToApply)
nodeToUpdate.Value = changesToApply.Value nodeToUpdate.Value = changesToApply.Value
nodeToUpdate.Tag = changesToApply.Tag nodeToUpdate.Tag = changesToApply.Tag
nodeToUpdate.Kind = changesToApply.Kind nodeToUpdate.Kind = changesToApply.Kind
@ -55,26 +56,33 @@ func (n *navigator) Update(value *yaml.Node, path []string, changesToApply yaml.
func (n *navigator) Visit(value *yaml.Node, path []string, visitor VisitorFn) (*yaml.Node, error) { func (n *navigator) Visit(value *yaml.Node, path []string, visitor VisitorFn) (*yaml.Node, error) {
realValue := value realValue := value
if realValue.Kind == yaml.DocumentNode { if realValue.Kind == yaml.DocumentNode {
n.log.Debugf("its a document! returning the first child")
realValue = value.Content[0] realValue = value.Content[0]
} }
if len(path) > 0 { if len(path) > 0 {
n.log.Debugf("diving into %v", path[0]) n.log.Debugf("diving into %v", path[0])
n.debugNode(value) n.DebugNode(value)
return n.recurse(realValue, path[0], path[1:], visitor) return n.recurse(realValue, path[0], path[1:], visitor)
} }
return visitor(realValue) return visitor(realValue)
} }
func (n *navigator) guessKind(tail []string) yaml.Kind { func (n *navigator) guessKind(tail []string, guess yaml.Kind) yaml.Kind {
n.log.Debug("tail %v", tail) n.log.Debug("tail %v", tail)
if len(tail) == 0 { if len(tail) == 0 && guess == 0 {
n.log.Debug("end of path, must be a scalar") n.log.Debug("end of path, must be a scalar")
return yaml.ScalarNode return yaml.ScalarNode
} else if len(tail) == 0 {
return guess
} }
var _, errorParsingInt = strconv.ParseInt(tail[0], 10, 64) var _, errorParsingInt = strconv.ParseInt(tail[0], 10, 64)
if tail[0] == "*" || tail[0] == "+" || errorParsingInt == nil { if tail[0] == "+" || errorParsingInt == nil {
return yaml.SequenceNode return yaml.SequenceNode
} }
if tail[0] == "*" && guess == yaml.SequenceNode || guess == yaml.MappingNode {
return guess
}
return yaml.MappingNode return yaml.MappingNode
} }
@ -90,7 +98,7 @@ func (n *navigator) getOrReplace(original *yaml.Node, expectedKind yaml.Kind) *y
return original return original
} }
func (n *navigator) debugNode(value *yaml.Node) { func (n *navigator) DebugNode(value *yaml.Node) {
if n.log.IsEnabledFor(logging.DEBUG) { if n.log.IsEnabledFor(logging.DEBUG) {
buf := new(bytes.Buffer) buf := new(bytes.Buffer)
encoder := yaml.NewEncoder(buf) encoder := yaml.NewEncoder(buf)
@ -105,17 +113,33 @@ func (n *navigator) recurse(value *yaml.Node, head string, tail []string, visito
switch value.Kind { switch value.Kind {
case yaml.MappingNode: case yaml.MappingNode:
n.log.Debug("its a map with %v entries", len(value.Content)/2) n.log.Debug("its a map with %v entries", len(value.Content)/2)
if head == "*" {
var newNode = yaml.Node{Kind: yaml.SequenceNode}
for index, content := range value.Content {
if index%2 == 0 {
continue
}
content = n.getOrReplace(content, n.guessKind(tail, content.Kind))
var nestedValue, err = n.Visit(content, tail, visitor)
if err != nil {
return nil, err
}
newNode.Content = append(newNode.Content, nestedValue)
}
return &newNode, nil
}
for index, content := range value.Content { for index, content := range value.Content {
// value.Content is a concatenated array of key, value, // value.Content is a concatenated array of key, value,
// so keys are in the even indexes, values in odd. // so keys are in the even indexes, values in odd.
if index%2 == 1 || content.Value != head { if index%2 == 1 || (content.Value != head) {
continue continue
} }
value.Content[index+1] = n.getOrReplace(value.Content[index+1], n.guessKind(tail)) value.Content[index+1] = n.getOrReplace(value.Content[index+1], n.guessKind(tail, value.Content[index+1].Kind))
return n.Visit(value.Content[index+1], tail, visitor) return n.Visit(value.Content[index+1], tail, visitor)
} }
value.Content = append(value.Content, &yaml.Node{Value: head, Kind: yaml.ScalarNode}) value.Content = append(value.Content, &yaml.Node{Value: head, Kind: yaml.ScalarNode})
mapEntryValue := yaml.Node{Kind: n.guessKind(tail)} mapEntryValue := yaml.Node{Kind: n.guessKind(tail, 0)}
value.Content = append(value.Content, &mapEntryValue) value.Content = append(value.Content, &mapEntryValue)
n.log.Debug("adding new node %v", value.Content) n.log.Debug("adding new node %v", value.Content)
return n.Visit(&mapEntryValue, tail, visitor) return n.Visit(&mapEntryValue, tail, visitor)
@ -127,11 +151,11 @@ func (n *navigator) recurse(value *yaml.Node, head string, tail []string, visito
for index, childValue := range value.Content { for index, childValue := range value.Content {
n.log.Debug("processing") n.log.Debug("processing")
n.debugNode(childValue) n.DebugNode(childValue)
childValue = n.getOrReplace(childValue, n.guessKind(tail)) childValue = n.getOrReplace(childValue, n.guessKind(tail, childValue.Kind))
var nestedValue, err = n.Visit(childValue, tail, visitor) var nestedValue, err = n.Visit(childValue, tail, visitor)
n.log.Debug("nestedValue") n.log.Debug("nestedValue")
n.debugNode(nestedValue) n.DebugNode(nestedValue)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -140,7 +164,7 @@ func (n *navigator) recurse(value *yaml.Node, head string, tail []string, visito
return &newNode, nil return &newNode, nil
} else if head == "+" { } else if head == "+" {
var newNode = yaml.Node{Kind: n.guessKind(tail)} var newNode = yaml.Node{Kind: n.guessKind(tail, 0)}
value.Content = append(value.Content, &newNode) value.Content = append(value.Content, &newNode)
n.log.Debug("appending a new node, %v", value.Content) n.log.Debug("appending a new node, %v", value.Content)
return n.Visit(&newNode, tail, visitor) return n.Visit(&newNode, tail, visitor)
@ -152,7 +176,7 @@ func (n *navigator) recurse(value *yaml.Node, head string, tail []string, visito
if index >= int64(len(value.Content)) { if index >= int64(len(value.Content)) {
return nil, nil return nil, nil
} }
value.Content[index] = n.getOrReplace(value.Content[index], n.guessKind(tail)) value.Content[index] = n.getOrReplace(value.Content[index], n.guessKind(tail, value.Content[index].Kind))
return n.Visit(value.Content[index], tail, visitor) return n.Visit(value.Content[index], tail, visitor)
default: default:
return nil, nil return nil, nil

View File

@ -1,397 +1,397 @@
package yqlib package yqlib
import ( // import (
"fmt" // "fmt"
"sort" // "sort"
"testing" // "testing"
"github.com/mikefarah/yq/v2/test" // "github.com/mikefarah/yq/v2/test"
logging "gopkg.in/op/go-logging.v1" // logging "gopkg.in/op/go-logging.v1"
) // )
func TestDataNavigator(t *testing.T) { // func TestDataNavigator(t *testing.T) {
var log = logging.MustGetLogger("yq") // var log = logging.MustGetLogger("yq")
subject := NewDataNavigator(log) // subject := NewDataNavigator(log)
t.Run("TestReadMap_simple", func(t *testing.T) { // t.Run("TestReadMap_simple", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
--- // ---
b: // b:
c: 2 // c: 2
`) // `)
got, _ := subject.ReadChildValue(data, []string{"b", "c"}) // got, _ := subject.ReadChildValue(data, []string{"b", "c"})
test.AssertResult(t, 2, got) // test.AssertResult(t, 2, got)
}) // })
t.Run("TestReadMap_numberKey", func(t *testing.T) { // t.Run("TestReadMap_numberKey", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
--- // ---
200: things // 200: things
`) // `)
got, _ := subject.ReadChildValue(data, []string{"200"}) // got, _ := subject.ReadChildValue(data, []string{"200"})
test.AssertResult(t, "things", got) // test.AssertResult(t, "things", got)
}) // })
t.Run("TestReadMap_splat", func(t *testing.T) { // t.Run("TestReadMap_splat", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
--- // ---
mapSplat: // mapSplat:
item1: things // item1: things
item2: whatever // item2: whatever
otherThing: cat // otherThing: cat
`) // `)
res, _ := subject.ReadChildValue(data, []string{"mapSplat", "*"}) // res, _ := subject.ReadChildValue(data, []string{"mapSplat", "*"})
test.AssertResult(t, "[things whatever cat]", fmt.Sprintf("%v", res)) // test.AssertResult(t, "[things whatever cat]", fmt.Sprintf("%v", res))
}) // })
t.Run("TestReadMap_prefixSplat", func(t *testing.T) { // t.Run("TestReadMap_prefixSplat", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
--- // ---
mapSplat: // mapSplat:
item1: things // item1: things
item2: whatever // item2: whatever
otherThing: cat // otherThing: cat
`) // `)
res, _ := subject.ReadChildValue(data, []string{"mapSplat", "item*"}) // res, _ := subject.ReadChildValue(data, []string{"mapSplat", "item*"})
test.AssertResult(t, "[things whatever]", fmt.Sprintf("%v", res)) // test.AssertResult(t, "[things whatever]", fmt.Sprintf("%v", res))
}) // })
t.Run("TestReadMap_deep_splat", func(t *testing.T) { // t.Run("TestReadMap_deep_splat", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
--- // ---
mapSplatDeep: // mapSplatDeep:
item1: // item1:
cats: bananas // cats: bananas
item2: // item2:
cats: apples // cats: apples
`) // `)
res, _ := subject.ReadChildValue(data, []string{"mapSplatDeep", "*", "cats"}) // res, _ := subject.ReadChildValue(data, []string{"mapSplatDeep", "*", "cats"})
result := res.([]interface{}) // result := res.([]interface{})
var actual = []string{result[0].(string), result[1].(string)} // var actual = []string{result[0].(string), result[1].(string)}
sort.Strings(actual) // sort.Strings(actual)
test.AssertResult(t, "[apples bananas]", fmt.Sprintf("%v", actual)) // test.AssertResult(t, "[apples bananas]", fmt.Sprintf("%v", actual))
}) // })
t.Run("TestReadMap_key_doesnt_exist", func(t *testing.T) { // t.Run("TestReadMap_key_doesnt_exist", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
--- // ---
b: // b:
c: 2 // c: 2
`) // `)
got, _ := subject.ReadChildValue(data, []string{"b", "x", "f", "c"}) // got, _ := subject.ReadChildValue(data, []string{"b", "x", "f", "c"})
test.AssertResult(t, nil, got) // test.AssertResult(t, nil, got)
}) // })
t.Run("TestReadMap_recurse_against_string", func(t *testing.T) { // t.Run("TestReadMap_recurse_against_string", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
--- // ---
a: cat // a: cat
`) // `)
got, _ := subject.ReadChildValue(data, []string{"a", "b"}) // got, _ := subject.ReadChildValue(data, []string{"a", "b"})
test.AssertResult(t, nil, got) // test.AssertResult(t, nil, got)
}) // })
t.Run("TestReadMap_with_array", func(t *testing.T) { // t.Run("TestReadMap_with_array", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
--- // ---
b: // b:
d: // d:
- 3 // - 3
- 4 // - 4
`) // `)
got, _ := subject.ReadChildValue(data, []string{"b", "d", "1"}) // got, _ := subject.ReadChildValue(data, []string{"b", "d", "1"})
test.AssertResult(t, 4, got) // test.AssertResult(t, 4, got)
}) // })
t.Run("TestReadMap_with_array_and_bad_index", func(t *testing.T) { // t.Run("TestReadMap_with_array_and_bad_index", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
--- // ---
b: // b:
d: // d:
- 3 // - 3
- 4 // - 4
`) // `)
_, err := subject.ReadChildValue(data, []string{"b", "d", "x"}) // _, err := subject.ReadChildValue(data, []string{"b", "d", "x"})
if err == nil { // if err == nil {
t.Fatal("Expected error due to invalid path") // t.Fatal("Expected error due to invalid path")
} // }
expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax` // expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
test.AssertResult(t, expectedOutput, err.Error()) // test.AssertResult(t, expectedOutput, err.Error())
}) // })
t.Run("TestReadMap_with_mapsplat_array_and_bad_index", func(t *testing.T) { // t.Run("TestReadMap_with_mapsplat_array_and_bad_index", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
--- // ---
b: // b:
d: // d:
e: // e:
- 3 // - 3
- 4 // - 4
f: // f:
- 1 // - 1
- 2 // - 2
`) // `)
_, err := subject.ReadChildValue(data, []string{"b", "d", "*", "x"}) // _, err := subject.ReadChildValue(data, []string{"b", "d", "*", "x"})
if err == nil { // if err == nil {
t.Fatal("Expected error due to invalid path") // t.Fatal("Expected error due to invalid path")
} // }
expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax` // expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
test.AssertResult(t, expectedOutput, err.Error()) // test.AssertResult(t, expectedOutput, err.Error())
}) // })
t.Run("TestReadMap_with_arraysplat_map_array_and_bad_index", func(t *testing.T) { // t.Run("TestReadMap_with_arraysplat_map_array_and_bad_index", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
--- // ---
b: // b:
d: // d:
- names: // - names:
- fred // - fred
- smith // - smith
- names: // - names:
- sam // - sam
- bo // - bo
`) // `)
_, err := subject.ReadChildValue(data, []string{"b", "d", "*", "names", "x"}) // _, err := subject.ReadChildValue(data, []string{"b", "d", "*", "names", "x"})
if err == nil { // if err == nil {
t.Fatal("Expected error due to invalid path") // t.Fatal("Expected error due to invalid path")
} // }
expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax` // expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax`
test.AssertResult(t, expectedOutput, err.Error()) // test.AssertResult(t, expectedOutput, err.Error())
}) // })
t.Run("TestReadMap_with_array_out_of_bounds", func(t *testing.T) { // t.Run("TestReadMap_with_array_out_of_bounds", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
--- // ---
b: // b:
d: // d:
- 3 // - 3
- 4 // - 4
`) // `)
got, _ := subject.ReadChildValue(data, []string{"b", "d", "3"}) // got, _ := subject.ReadChildValue(data, []string{"b", "d", "3"})
test.AssertResult(t, nil, got) // test.AssertResult(t, nil, got)
}) // })
t.Run("TestReadMap_with_array_out_of_bounds_by_1", func(t *testing.T) { // t.Run("TestReadMap_with_array_out_of_bounds_by_1", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
--- // ---
b: // b:
d: // d:
- 3 // - 3
- 4 // - 4
`) // `)
got, _ := subject.ReadChildValue(data, []string{"b", "d", "2"}) // got, _ := subject.ReadChildValue(data, []string{"b", "d", "2"})
test.AssertResult(t, nil, got) // test.AssertResult(t, nil, got)
}) // })
t.Run("TestReadMap_with_array_splat", func(t *testing.T) { // t.Run("TestReadMap_with_array_splat", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
e: // e:
- // -
name: Fred // name: Fred
thing: cat // thing: cat
- // -
name: Sam // name: Sam
thing: dog // thing: dog
`) // `)
got, _ := subject.ReadChildValue(data, []string{"e", "*", "name"}) // got, _ := subject.ReadChildValue(data, []string{"e", "*", "name"})
test.AssertResult(t, "[Fred Sam]", fmt.Sprintf("%v", got)) // test.AssertResult(t, "[Fred Sam]", fmt.Sprintf("%v", got))
}) // })
t.Run("TestWrite_really_simple", func(t *testing.T) { // t.Run("TestWrite_really_simple", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
b: 2 // b: 2
`) // `)
updated := subject.UpdatedChildValue(data, []string{"b"}, "4") // updated := subject.UpdatedChildValue(data, []string{"b"}, "4")
test.AssertResult(t, "[{b 4}]", fmt.Sprintf("%v", updated)) // test.AssertResult(t, "[{b 4}]", fmt.Sprintf("%v", updated))
}) // })
t.Run("TestWrite_simple", func(t *testing.T) { // t.Run("TestWrite_simple", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
b: // b:
c: 2 // c: 2
`) // `)
updated := subject.UpdatedChildValue(data, []string{"b", "c"}, "4") // updated := subject.UpdatedChildValue(data, []string{"b", "c"}, "4")
test.AssertResult(t, "[{b [{c 4}]}]", fmt.Sprintf("%v", updated)) // test.AssertResult(t, "[{b [{c 4}]}]", fmt.Sprintf("%v", updated))
}) // })
t.Run("TestWrite_new", func(t *testing.T) { // t.Run("TestWrite_new", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
b: // b:
c: 2 // c: 2
`) // `)
updated := subject.UpdatedChildValue(data, []string{"b", "d"}, "4") // updated := subject.UpdatedChildValue(data, []string{"b", "d"}, "4")
test.AssertResult(t, "[{b [{c 2} {d 4}]}]", fmt.Sprintf("%v", updated)) // test.AssertResult(t, "[{b [{c 2} {d 4}]}]", fmt.Sprintf("%v", updated))
}) // })
t.Run("TestWrite_new_deep", func(t *testing.T) { // t.Run("TestWrite_new_deep", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
b: // b:
c: 2 // c: 2
`) // `)
updated := subject.UpdatedChildValue(data, []string{"b", "d", "f"}, "4") // updated := subject.UpdatedChildValue(data, []string{"b", "d", "f"}, "4")
test.AssertResult(t, "[{b [{c 2} {d [{f 4}]}]}]", fmt.Sprintf("%v", updated)) // test.AssertResult(t, "[{b [{c 2} {d [{f 4}]}]}]", fmt.Sprintf("%v", updated))
}) // })
t.Run("TestWrite_array", func(t *testing.T) { // t.Run("TestWrite_array", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
b: // b:
- aa // - aa
`) // `)
updated := subject.UpdatedChildValue(data, []string{"b", "0"}, "bb") // updated := subject.UpdatedChildValue(data, []string{"b", "0"}, "bb")
test.AssertResult(t, "[{b [bb]}]", fmt.Sprintf("%v", updated)) // test.AssertResult(t, "[{b [bb]}]", fmt.Sprintf("%v", updated))
}) // })
t.Run("TestWrite_new_array", func(t *testing.T) { // t.Run("TestWrite_new_array", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
b: // b:
c: 2 // c: 2
`) // `)
updated := subject.UpdatedChildValue(data, []string{"b", "0"}, "4") // updated := subject.UpdatedChildValue(data, []string{"b", "0"}, "4")
test.AssertResult(t, "[{b [{c 2} {0 4}]}]", fmt.Sprintf("%v", updated)) // test.AssertResult(t, "[{b [{c 2} {0 4}]}]", fmt.Sprintf("%v", updated))
}) // })
t.Run("TestWrite_new_array_deep", func(t *testing.T) { // t.Run("TestWrite_new_array_deep", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
a: apple // a: apple
`) // `)
updated := subject.UpdatedChildValue(data, []string{"b", "+", "c"}, "4") // updated := subject.UpdatedChildValue(data, []string{"b", "+", "c"}, "4")
test.AssertResult(t, "[{a apple} {b [[{c 4}]]}]", fmt.Sprintf("%v", updated)) // test.AssertResult(t, "[{a apple} {b [[{c 4}]]}]", fmt.Sprintf("%v", updated))
}) // })
t.Run("TestWrite_new_map_array_deep", func(t *testing.T) { // t.Run("TestWrite_new_map_array_deep", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
b: // b:
c: 2 // c: 2
`) // `)
updated := subject.UpdatedChildValue(data, []string{"b", "d", "+"}, "4") // updated := subject.UpdatedChildValue(data, []string{"b", "d", "+"}, "4")
test.AssertResult(t, "[{b [{c 2} {d [4]}]}]", fmt.Sprintf("%v", updated)) // test.AssertResult(t, "[{b [{c 2} {d [4]}]}]", fmt.Sprintf("%v", updated))
}) // })
t.Run("TestWrite_add_to_array", func(t *testing.T) { // t.Run("TestWrite_add_to_array", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
b: // b:
- aa // - aa
`) // `)
updated := subject.UpdatedChildValue(data, []string{"b", "1"}, "bb") // updated := subject.UpdatedChildValue(data, []string{"b", "1"}, "bb")
test.AssertResult(t, "[{b [aa bb]}]", fmt.Sprintf("%v", updated)) // test.AssertResult(t, "[{b [aa bb]}]", fmt.Sprintf("%v", updated))
}) // })
t.Run("TestWrite_with_no_tail", func(t *testing.T) { // t.Run("TestWrite_with_no_tail", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
b: // b:
c: 2 // c: 2
`) // `)
updated := subject.UpdatedChildValue(data, []string{"b"}, "4") // updated := subject.UpdatedChildValue(data, []string{"b"}, "4")
test.AssertResult(t, "[{b 4}]", fmt.Sprintf("%v", updated)) // test.AssertResult(t, "[{b 4}]", fmt.Sprintf("%v", updated))
}) // })
t.Run("TestWriteMap_no_paths", func(t *testing.T) { // t.Run("TestWriteMap_no_paths", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
b: 5 // b: 5
`) // `)
var new = test.ParseData(` // var new = test.ParseData(`
c: 4 // c: 4
`) // `)
result := subject.UpdatedChildValue(data, []string{}, new) // result := subject.UpdatedChildValue(data, []string{}, new)
test.AssertResult(t, fmt.Sprintf("%v", new), fmt.Sprintf("%v", result)) // test.AssertResult(t, fmt.Sprintf("%v", new), fmt.Sprintf("%v", result))
}) // })
t.Run("TestWriteArray_no_paths", func(t *testing.T) { // t.Run("TestWriteArray_no_paths", func(t *testing.T) {
var data = make([]interface{}, 1) // var data = make([]interface{}, 1)
data[0] = "mike" // data[0] = "mike"
var new = test.ParseData(` // var new = test.ParseData(`
c: 4 // c: 4
`) // `)
result := subject.UpdatedChildValue(data, []string{}, new) // result := subject.UpdatedChildValue(data, []string{}, new)
test.AssertResult(t, fmt.Sprintf("%v", new), fmt.Sprintf("%v", result)) // test.AssertResult(t, fmt.Sprintf("%v", new), fmt.Sprintf("%v", result))
}) // })
t.Run("TestDelete_MapItem", func(t *testing.T) { // t.Run("TestDelete_MapItem", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
a: 123 // a: 123
b: 456 // b: 456
`) // `)
var expected = test.ParseData(` // var expected = test.ParseData(`
b: 456 // b: 456
`) // `)
result, _ := subject.DeleteChildValue(data, []string{"a"}) // result, _ := subject.DeleteChildValue(data, []string{"a"})
test.AssertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result)) // test.AssertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result))
}) // })
// Ensure deleting an index into a string does nothing // // Ensure deleting an index into a string does nothing
t.Run("TestDelete_index_to_string", func(t *testing.T) { // t.Run("TestDelete_index_to_string", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
a: mystring // a: mystring
`) // `)
result, _ := subject.DeleteChildValue(data, []string{"a", "0"}) // result, _ := subject.DeleteChildValue(data, []string{"a", "0"})
test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result)) // test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
}) // })
t.Run("TestDelete_list_index", func(t *testing.T) { // t.Run("TestDelete_list_index", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
a: [3, 4] // a: [3, 4]
`) // `)
var expected = test.ParseData(` // var expected = test.ParseData(`
a: [3] // a: [3]
`) // `)
result, _ := subject.DeleteChildValue(data, []string{"a", "1"}) // result, _ := subject.DeleteChildValue(data, []string{"a", "1"})
test.AssertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result)) // test.AssertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result))
}) // })
t.Run("TestDelete_list_index_beyond_bounds", func(t *testing.T) { // t.Run("TestDelete_list_index_beyond_bounds", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
a: [3, 4] // a: [3, 4]
`) // `)
result, _ := subject.DeleteChildValue(data, []string{"a", "5"}) // result, _ := subject.DeleteChildValue(data, []string{"a", "5"})
test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result)) // test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
}) // })
t.Run("TestDelete_list_index_out_of_bounds_by_1", func(t *testing.T) { // t.Run("TestDelete_list_index_out_of_bounds_by_1", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
a: [3, 4] // a: [3, 4]
`) // `)
result, _ := subject.DeleteChildValue(data, []string{"a", "2"}) // result, _ := subject.DeleteChildValue(data, []string{"a", "2"})
test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result)) // test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
}) // })
t.Run("TestDelete_no_paths", func(t *testing.T) { // t.Run("TestDelete_no_paths", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
a: [3, 4] // a: [3, 4]
b: // b:
- name: test // - name: test
`) // `)
result, _ := subject.DeleteChildValue(data, []string{}) // result, _ := subject.DeleteChildValue(data, []string{})
test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result)) // test.AssertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result))
}) // })
t.Run("TestDelete_array_map_item", func(t *testing.T) { // t.Run("TestDelete_array_map_item", func(t *testing.T) {
var data = test.ParseData(` // var data = test.ParseData(`
b: // b:
- name: fred // - name: fred
value: blah // value: blah
- name: john // - name: john
value: test // value: test
`) // `)
var expected = test.ParseData(` // var expected = test.ParseData(`
b: // b:
- value: blah // - value: blah
- name: john // - name: john
value: test // value: test
`) // `)
result, _ := subject.DeleteChildValue(data, []string{"b", "0", "name"}) // result, _ := subject.DeleteChildValue(data, []string{"b", "0", "name"})
test.AssertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result)) // test.AssertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result))
}) // })
} // }

View File

@ -14,6 +14,7 @@ type UpdateCommand struct {
} }
type YqLib interface { type YqLib interface {
DebugNode(node *yaml.Node)
Get(rootNode *yaml.Node, path string) (*yaml.Node, error) Get(rootNode *yaml.Node, path string) (*yaml.Node, error)
Update(rootNode *yaml.Node, updateCommand UpdateCommand) error Update(rootNode *yaml.Node, updateCommand UpdateCommand) error
} }
@ -32,10 +33,11 @@ func NewYqLib(l *logging.Logger) YqLib {
} }
} }
func (l *lib) DebugNode(node *yaml.Node) {
l.navigator.DebugNode(node)
}
func (l *lib) Get(rootNode *yaml.Node, path string) (*yaml.Node, error) { func (l *lib) Get(rootNode *yaml.Node, path string) (*yaml.Node, error) {
if path == "" {
return rootNode, nil
}
var paths = l.parser.ParsePath(path) var paths = l.parser.ParsePath(path)
return l.navigator.Get(rootNode, paths) return l.navigator.Get(rootNode, paths)
} }

View File

@ -4,7 +4,7 @@ import (
"fmt" "fmt"
"testing" "testing"
"github.com/mikefarah/yq/v2/test" "github.com/mikefarah/yq/v3/test"
logging "gopkg.in/op/go-logging.v1" logging "gopkg.in/op/go-logging.v1"
) )

View File

@ -11,6 +11,9 @@ func NewPathParser() PathParser {
} }
func (p *pathParser) ParsePath(path string) []string { func (p *pathParser) ParsePath(path string) []string {
if path == "" {
return []string{}
}
return p.parsePathAccum([]string{}, path) return p.parsePathAccum([]string{}, path)
} }

View File

@ -3,7 +3,7 @@ package yqlib
import ( import (
"testing" "testing"
"github.com/mikefarah/yq/v2/test" "github.com/mikefarah/yq/v3/test"
) )
var parsePathsTests = []struct { var parsePathsTests = []struct {

View File

@ -3,7 +3,7 @@ package yqlib
import ( import (
"testing" "testing"
"github.com/mikefarah/yq/v2/test" "github.com/mikefarah/yq/v3/test"
) )
var parseValueTests = []struct { var parseValueTests = []struct {

View File

@ -9,8 +9,8 @@ import (
"strings" "strings"
"testing" "testing"
yaml "github.com/mikefarah/yaml/v2"
"github.com/spf13/cobra" "github.com/spf13/cobra"
yaml "gopkg.in/yaml.v3"
) )
type resulter struct { type resulter struct {
@ -30,8 +30,8 @@ func RunCmd(c *cobra.Command, input string) resulter {
return resulter{err, output, c} return resulter{err, output, c}
} }
func ParseData(rawData string) yaml.MapSlice { func ParseData(rawData string) yaml.Node {
var parsedData yaml.MapSlice var parsedData yaml.Node
err := yaml.Unmarshal([]byte(rawData), &parsedData) err := yaml.Unmarshal([]byte(rawData), &parsedData)
if err != nil { if err != nil {
fmt.Printf("Error parsing yaml: %v\n", err) fmt.Printf("Error parsing yaml: %v\n", err)

6
yq.go
View File

@ -257,11 +257,14 @@ func readProperty(cmd *cobra.Command, args []string) error {
} }
var mappedDocs []*yaml.Node var mappedDocs []*yaml.Node
var dataBucket yaml.Node
var currentIndex = 0 var currentIndex = 0
var errorReadingStream = readStream(args[0], func(decoder *yaml.Decoder) error { var errorReadingStream = readStream(args[0], func(decoder *yaml.Decoder) error {
for { for {
var dataBucket yaml.Node
errorReading := decoder.Decode(&dataBucket) errorReading := decoder.Decode(&dataBucket)
log.Debugf("decoded node for doc %v", currentIndex)
lib.DebugNode(&dataBucket)
if errorReading == io.EOF { if errorReading == io.EOF {
log.Debugf("done %v / %v", currentIndex, docIndexInt) log.Debugf("done %v / %v", currentIndex, docIndexInt)
if !updateAll && currentIndex <= docIndexInt { if !updateAll && currentIndex <= docIndexInt {
@ -273,6 +276,7 @@ func readProperty(cmd *cobra.Command, args []string) error {
if updateAll || currentIndex == docIndexInt { if updateAll || currentIndex == docIndexInt {
log.Debugf("reading %v in document %v", path, currentIndex) log.Debugf("reading %v in document %v", path, currentIndex)
mappedDoc, errorParsing := lib.Get(&dataBucket, path) mappedDoc, errorParsing := lib.Get(&dataBucket, path)
lib.DebugNode(mappedDoc)
if errorParsing != nil { if errorParsing != nil {
return errors.Wrapf(errorParsing, "Error reading path in document index %v", currentIndex) return errors.Wrapf(errorParsing, "Error reading path in document index %v", currentIndex)
} }

View File

@ -1,60 +1,60 @@
package main package main
import ( // import (
"fmt" // "fmt"
"runtime" // "runtime"
"testing" // "testing"
"github.com/mikefarah/yq/v2/pkg/marshal" // "github.com/mikefarah/yq/v2/pkg/marshal"
"github.com/mikefarah/yq/v2/test" // "github.com/mikefarah/yq/v2/test"
) // )
func TestMultilineString(t *testing.T) { // func TestMultilineString(t *testing.T) {
testString := ` // testString := `
abcd // abcd
efg` // efg`
formattedResult, _ := marshal.NewYamlConverter().YamlToString(testString, false) // formattedResult, _ := marshal.NewYamlConverter().YamlToString(testString, false)
test.AssertResult(t, testString, formattedResult) // test.AssertResult(t, testString, formattedResult)
} // }
func TestNewYaml(t *testing.T) { // func TestNewYaml(t *testing.T) {
result, _ := newYaml([]string{"b.c", "3"}) // result, _ := newYaml([]string{"b.c", "3"})
formattedResult := fmt.Sprintf("%v", result) // formattedResult := fmt.Sprintf("%v", result)
test.AssertResult(t, // test.AssertResult(t,
"[{b [{c 3}]}]", // "[{b [{c 3}]}]",
formattedResult) // formattedResult)
} // }
func TestNewYamlArray(t *testing.T) { // func TestNewYamlArray(t *testing.T) {
result, _ := newYaml([]string{"[0].cat", "meow"}) // result, _ := newYaml([]string{"[0].cat", "meow"})
formattedResult := fmt.Sprintf("%v", result) // formattedResult := fmt.Sprintf("%v", result)
test.AssertResult(t, // test.AssertResult(t,
"[[{cat meow}]]", // "[[{cat meow}]]",
formattedResult) // formattedResult)
} // }
func TestNewYaml_WithScript(t *testing.T) { // func TestNewYaml_WithScript(t *testing.T) {
writeScript = "examples/instruction_sample.yaml" // writeScript = "examples/instruction_sample.yaml"
expectedResult := `b: // expectedResult := `b:
c: cat // c: cat
e: // e:
- name: Mike Farah` // - name: Mike Farah`
result, _ := newYaml([]string{""}) // result, _ := newYaml([]string{""})
actualResult, _ := marshal.NewYamlConverter().YamlToString(result, true) // actualResult, _ := marshal.NewYamlConverter().YamlToString(result, true)
test.AssertResult(t, expectedResult, actualResult) // test.AssertResult(t, expectedResult, actualResult)
} // }
func TestNewYaml_WithUnknownScript(t *testing.T) { // func TestNewYaml_WithUnknownScript(t *testing.T) {
writeScript = "fake-unknown" // writeScript = "fake-unknown"
_, err := newYaml([]string{""}) // _, err := newYaml([]string{""})
if err == nil { // if err == nil {
t.Error("Expected error due to unknown file") // t.Error("Expected error due to unknown file")
} // }
var expectedOutput string // var expectedOutput string
if runtime.GOOS == "windows" { // if runtime.GOOS == "windows" {
expectedOutput = `open fake-unknown: The system cannot find the file specified.` // expectedOutput = `open fake-unknown: The system cannot find the file specified.`
} else { // } else {
expectedOutput = `open fake-unknown: no such file or directory` // expectedOutput = `open fake-unknown: no such file or directory`
} // }
test.AssertResult(t, expectedOutput, err.Error()) // test.AssertResult(t, expectedOutput, err.Error())
} // }