diff --git a/.gitignore b/.gitignore index 06ed739b..1561eba9 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ _obj _test bin build +build-done .DS_Store # Architecture specific extensions/prefixes diff --git a/.travis.yml b/.travis.yml index eaac2a95..3d51d69a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: go go: - - 1.9.x + - 1.11.x script: - scripts/devtools.sh - make local build diff --git a/Dockerfile b/Dockerfile index c89f0ec1..63867d44 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.9 as builder +FROM golang:1.11 as builder WORKDIR /go/src/mikefarah/yq @@ -16,9 +16,12 @@ RUN CGO_ENABLED=0 make local build # Choose alpine as a base image to make this useful for CI, as many # CI tools expect an interactive shell inside the container -FROM alpine:3.7 +FROM alpine:3.8 as production COPY --from=builder /go/src/mikefarah/yq/yq /usr/bin/yq RUN chmod +x /usr/bin/yq +ARG VERSION=none +LABEL version=${VERSION} + WORKDIR /workdir diff --git a/Makefile b/Makefile index 5d6c2e6d..d854d8fb 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,6 @@ help: @echo ' make build Build yq binary.' @echo ' make install Install yq.' @echo ' make xcompile Build cross-compiled binaries of yq.' - @echo ' make snap Build a snap package of yq.' @echo ' make vendor Install dependencies using govendor.' @echo ' make format Run code formatter.' @echo ' make check Run static code analysis (lint).' @@ -65,10 +64,6 @@ xcompile: check @find build -type d -exec chmod 755 {} \; || : @find build -type f -exec chmod 755 {} \; || : -.PHONY: snap -snap: - snapcraft - .PHONY: install install: build ${DOCKRUN} go install diff --git a/README.md b/README.md index 8786e4b5..40461bcc 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# yq +# yq -[![Build Status](https://travis-ci.org/mikefarah/yq.svg?branch=master)](https://travis-ci.org/mikefarah/yq) ![Docker Pulls](https://img.shields.io/docker/pulls/mikefarah/yq.svg) ![Github Releases (by Release)](https://img.shields.io/github/downloads/mikefarah/yq/total.svg) +[![Build Status](https://travis-ci.org/mikefarah/yq.svg?branch=master)](https://travis-ci.org/mikefarah/yq) ![Docker Pulls](https://img.shields.io/docker/pulls/mikefarah/yq.svg) ![Github Releases (by Release)](https://img.shields.io/github/downloads/mikefarah/yq/total.svg) ![Go Report](https://goreportcard.com/badge/github.com/mikefarah/yq) a lightweight and portable command-line YAML processor @@ -8,21 +8,40 @@ a lightweight and portable command-line YAML processor The aim of the project is to be the [jq](https://github.com/stedolan/jq) or sed of yaml files. ## Install -On MacOS: +### On MacOS: ``` brew install yq ``` -On Ubuntu and other Linux distros supporting `snap` packages: +### On Ubuntu and other Linux distros supporting `snap` packages: ``` snap install yq ``` -On Ubuntu 16.04 or higher from Debian package: + +#### Snap notes +`yq` installs with with [_strict confinement_](https://docs.snapcraft.io/snap-confinement/6233) in snap, this means it doesn't have direct access to root files. To read root files you can: + +``` +sudo cat /etc/myfile | yq -r - somecommand +``` + +And to write to a root file you can either use [sponge](https://linux.die.net/man/1/sponge): +``` +sudo cat /etc/myfile | yq -r - somecommand | sudo sponge /etc/myfile +``` +or write to a temporary file: +``` +sudo cat /etc/myfile | yq -r - somecommand | sudo tee /etc/myfile.tmp +sudo mv /etc/myfile.tmp /etc/myfile +rm /etc/myfile.tmp +``` + +### On Ubuntu 16.04 or higher from Debian package: ``` sudo add-apt-repository ppa:rmescandon/yq sudo apt update sudo apt install yq -y ``` -or, [Download latest binary](https://github.com/mikefarah/yq/releases/latest) or alternatively: +### or, [Download latest binary](https://github.com/mikefarah/yq/releases/latest) or alternatively: ``` go get gopkg.in/mikefarah/yq.v2 ``` @@ -32,13 +51,21 @@ go get gopkg.in/mikefarah/yq.v2 Oneshot use: ```bash -docker run -v ${PWD}:/workdir mikefarah/yq yq [flags] FILE... +docker run --rm -v ${PWD}:/workdir mikefarah/yq yq [flags] FILE... ``` Run commands interactively: ```bash -docker run -it -v ${PWD}:/workdir mikefarah/yq sh +docker run --rm -it -v ${PWD}:/workdir mikefarah/yq sh +``` + +It can be useful to have a bash function to avoid typing the whole docker command: + +```bash +yq() { + docker run --rm -i -v ${PWD}:/workdir mikefarah/yq yq $@ +} ``` ## Features @@ -49,6 +76,7 @@ docker run -it -v ${PWD}:/workdir mikefarah/yq sh - Update creates any missing entries in the path on the fly - Create a yaml file given a deep path and value - Create a yaml file given a script file +- Prefix a path to a yaml file - Convert from json to yaml - Convert from yaml to json - Pipe data in by using '-' @@ -62,6 +90,8 @@ docker run -it -v ${PWD}:/workdir mikefarah/yq sh Check out the [documentation](http://mikefarah.github.io/yq/) for more detailed and advanced usage. ``` +yq is a lightweight and portable command-line YAML processor. It aims to be the jq or sed of yaml files. + Usage: yq [flags] yq [command] @@ -71,6 +101,7 @@ Available Commands: help Help about any command merge yq m [--inplace/-i] [--doc/-d index] [--overwrite/-x] [--append/-a] sample.yaml sample2.yaml new yq n [--script/-s script_file] a.b.c newValue + prefix yq p [--inplace/-i] [--doc/-d index] sample.yaml a.b.c read yq r [--doc/-d index] sample.yaml a.b.c write yq w [--inplace/-i] [--script/-s script_file] [--doc/-d index] sample.yaml a.b.c newValue @@ -89,8 +120,8 @@ Use "yq [command] --help" for more information about a command. 3. add unit tests 4. apply changes (use govendor with a preference to [gopkg](https://gopkg.in/) for package dependencies) 5. `make [local] build` -6. If required, update the user documentation +6. If required, update the user documentation - Update README.md and/or documentation under the mkdocs folder - `make [local] build-docs` - - browse to docs/index.html and check your changes + - browse to docs/index.html and check your changes 7. profit diff --git a/commands_test.go b/commands_test.go index ee52333d..102f71b7 100644 --- a/commands_test.go +++ b/commands_test.go @@ -2,10 +2,12 @@ package main import ( "fmt" + "os" + "runtime" "strings" "testing" - "gopkg.in/spf13/cobra.v0" + cobra "gopkg.in/spf13/cobra.v0" ) func getRootCommand() *cobra.Command { @@ -22,7 +24,18 @@ func TestRootCmd(t *testing.T) { if !strings.Contains(result.Output, "Usage:") { t.Error("Expected usage message to be printed out, but the usage message was not found.") } +} +func TestRootCmd_Help(t *testing.T) { + cmd := getRootCommand() + result := runCmd(cmd, "--help") + if result.Error != nil { + t.Error(result.Error) + } + + if !strings.Contains(result.Output, "yq is a lightweight and portable command-line YAML processor. It aims to be the jq or sed of yaml files.") { + t.Error("Expected usage message to be printed out, but the usage message was not found.") + } } func TestRootCmd_VerboseLong(t *testing.T) { @@ -120,7 +133,7 @@ func TestReadBadDocumentIndexCmd(t *testing.T) { if result.Error == nil { t.Error("Expected command to fail due to invalid path") } - expectedOutput := `Asked to process document index 1 but there are only 1 document(s)` + expectedOutput := `asked to process document index 1 but there are only 1 document(s)` assertResult(t, expectedOutput, result.Error.Error()) } @@ -238,7 +251,7 @@ func TestReadCmd_ArrayYaml_ErrorBadPath(t *testing.T) { if result.Error == nil { t.Error("Expected command to fail due to invalid path") } - expectedOutput := `Error reading path in document index 0: Error accessing array: strconv.ParseInt: parsing "x": invalid syntax` + expectedOutput := `Error reading path in document index 0: error accessing array: strconv.ParseInt: parsing "x": invalid syntax` assertResult(t, expectedOutput, result.Error.Error()) } @@ -248,7 +261,7 @@ func TestReadCmd_ArrayYaml_Splat_ErrorBadPath(t *testing.T) { if result.Error == nil { t.Error("Expected command to fail due to invalid path") } - expectedOutput := `Error reading path in document index 0: Error accessing array: strconv.ParseInt: parsing "x": invalid syntax` + expectedOutput := `Error reading path in document index 0: error accessing array: strconv.ParseInt: parsing "x": invalid syntax` assertResult(t, expectedOutput, result.Error.Error()) } @@ -278,7 +291,12 @@ func TestReadCmd_ErrorUnreadableFile(t *testing.T) { if result.Error == nil { t.Error("Expected command to fail due to unknown file") } - expectedOutput := `open fake-unknown: no such file or directory` + var expectedOutput string + if runtime.GOOS == "windows" { + expectedOutput = `open fake-unknown: The system cannot find the file specified.` + } else { + expectedOutput = `open fake-unknown: no such file or directory` + } assertResult(t, expectedOutput, result.Error.Error()) } @@ -300,7 +318,7 @@ func TestReadCmd_ErrorBadPath(t *testing.T) { if result.Error == nil { t.Fatal("Expected command to fail due to invalid path") } - expectedOutput := `Error reading path in document index 0: Error accessing array: strconv.ParseInt: parsing "x": invalid syntax` + expectedOutput := `Error reading path in document index 0: error accessing array: strconv.ParseInt: parsing "x": invalid syntax` assertResult(t, expectedOutput, result.Error.Error()) } @@ -340,6 +358,204 @@ func TestReadCmd_ToJsonLong(t *testing.T) { assertResult(t, "2\n", result.Output) } +func TestPrefixCmd(t *testing.T) { + content := `b: + c: 3 +` + filename := writeTempYamlFile(content) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("prefix %s d", filename)) + if result.Error != nil { + t.Error(result.Error) + } + expectedOutput := `d: + b: + c: 3 +` + assertResult(t, expectedOutput, result.Output) +} + +func TestPrefixCmdArray(t *testing.T) { + content := `b: + c: 3 +` + filename := writeTempYamlFile(content) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("prefix %s [+].d.[+]", filename)) + if result.Error != nil { + t.Error(result.Error) + } + expectedOutput := `- d: + - b: + c: 3 +` + assertResult(t, expectedOutput, result.Output) +} + +func TestPrefixCmd_MultiLayer(t *testing.T) { + content := `b: + c: 3 +` + filename := writeTempYamlFile(content) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("prefix %s d.e.f", filename)) + if result.Error != nil { + t.Error(result.Error) + } + expectedOutput := `d: + e: + f: + b: + c: 3 +` + assertResult(t, expectedOutput, result.Output) +} + +func TestPrefixMultiCmd(t *testing.T) { + content := `b: + c: 3 +--- +apples: great +` + filename := writeTempYamlFile(content) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("prefix %s -d 1 d", filename)) + if result.Error != nil { + t.Error(result.Error) + } + expectedOutput := `b: + c: 3 +--- +d: + apples: great +` + assertResult(t, expectedOutput, result.Output) +} +func TestPrefixInvalidDocumentIndexCmd(t *testing.T) { + content := `b: + c: 3 +` + filename := writeTempYamlFile(content) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("prefix %s -df d", filename)) + if result.Error == nil { + t.Error("Expected command to fail due to invalid path") + } + expectedOutput := `Document index f is not a integer or *: strconv.ParseInt: parsing "f": invalid syntax` + assertResult(t, expectedOutput, result.Error.Error()) +} + +func TestPrefixBadDocumentIndexCmd(t *testing.T) { + content := `b: + c: 3 +` + filename := writeTempYamlFile(content) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("prefix %s -d 1 d", filename)) + if result.Error == nil { + t.Error("Expected command to fail due to invalid path") + } + expectedOutput := `asked to process document index 1 but there are only 1 document(s)` + assertResult(t, expectedOutput, result.Error.Error()) +} +func TestPrefixMultiAllCmd(t *testing.T) { + content := `b: + c: 3 +--- +apples: great +` + filename := writeTempYamlFile(content) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("prefix %s -d * d", filename)) + if result.Error != nil { + t.Error(result.Error) + } + expectedOutput := `d: + b: + c: 3 +--- +d: + apples: great` + assertResult(t, expectedOutput, strings.Trim(result.Output, "\n ")) +} + +func TestPrefixCmd_Error(t *testing.T) { + cmd := getRootCommand() + result := runCmd(cmd, "prefix") + if result.Error == nil { + t.Error("Expected command to fail due to missing arg") + } + expectedOutput := `Must provide ` + assertResult(t, expectedOutput, result.Error.Error()) +} + +func TestPrefixCmd_ErrorUnreadableFile(t *testing.T) { + cmd := getRootCommand() + result := runCmd(cmd, "prefix fake-unknown a.b") + if result.Error == nil { + t.Error("Expected command to fail due to unknown file") + } + var expectedOutput string + if runtime.GOOS == "windows" { + expectedOutput = `open fake-unknown: The system cannot find the file specified.` + } else { + expectedOutput = `open fake-unknown: no such file or directory` + } + assertResult(t, expectedOutput, result.Error.Error()) +} + +func TestPrefixCmd_Verbose(t *testing.T) { + content := `b: + c: 3 +` + filename := writeTempYamlFile(content) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("-v prefix %s x", filename)) + if result.Error != nil { + t.Error(result.Error) + } + expectedOutput := `x: + b: + c: 3 +` + assertResult(t, expectedOutput, result.Output) +} + +func TestPrefixCmd_Inplace(t *testing.T) { + content := `b: + c: 3 +` + filename := writeTempYamlFile(content) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("prefix -i %s d", filename)) + if result.Error != nil { + t.Error(result.Error) + } + gotOutput := readTempYamlFile(filename) + expectedOutput := `d: + b: + c: 3` + assertResult(t, expectedOutput, strings.Trim(gotOutput, "\n ")) +} + func TestNewCmd(t *testing.T) { cmd := getRootCommand() result := runCmd(cmd, "new b.c 3") @@ -441,7 +657,7 @@ func TestWriteBadDocumentIndexCmd(t *testing.T) { if result.Error == nil { t.Error("Expected command to fail due to invalid path") } - expectedOutput := `Asked to process document index 1 but there are only 1 document(s)` + expectedOutput := `asked to process document index 1 but there are only 1 document(s)` assertResult(t, expectedOutput, result.Error.Error()) } func TestWriteMultiAllCmd(t *testing.T) { @@ -498,7 +714,12 @@ func TestWriteCmd_ErrorUnreadableFile(t *testing.T) { if result.Error == nil { t.Error("Expected command to fail due to unknown file") } - expectedOutput := `open fake-unknown: no such file or directory` + var expectedOutput string + if runtime.GOOS == "windows" { + expectedOutput = `open fake-unknown: The system cannot find the file specified.` + } else { + expectedOutput = `open fake-unknown: no such file or directory` + } assertResult(t, expectedOutput, result.Error.Error()) } @@ -564,7 +785,7 @@ func TestWriteCmd_AppendEmptyArray(t *testing.T) { defer removeTempYamlFile(filename) cmd := getRootCommand() - result := runCmd(cmd, fmt.Sprintf("write %s b[+] v", filename)) + result := runCmd(cmd, fmt.Sprintf("write -v %s b[+] v", filename)) if result.Error != nil { t.Error(result.Error) } @@ -575,6 +796,66 @@ b: assertResult(t, expectedOutput, result.Output) } +func TestWriteCmd_SplatArray(t *testing.T) { + content := `b: +- c: thing +- c: another thing +` + filename := writeTempYamlFile(content) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("write -v %s b[*].c new", filename)) + if result.Error != nil { + t.Error(result.Error) + } + expectedOutput := `b: +- c: new +- c: new +` + assertResult(t, expectedOutput, result.Output) +} + +func TestWriteCmd_SplatMap(t *testing.T) { + content := `b: + c: thing + d: another thing +` + filename := writeTempYamlFile(content) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("write -v %s b.* new", filename)) + if result.Error != nil { + t.Error(result.Error) + } + expectedOutput := `b: + c: new + d: new +` + assertResult(t, expectedOutput, result.Output) +} + +func TestWriteCmd_SplatMapEmpty(t *testing.T) { + content := `b: + c: thing + d: another thing +` + filename := writeTempYamlFile(content) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("write -v %s b.c.* new", filename)) + if result.Error != nil { + t.Error(result.Error) + } + expectedOutput := `b: + c: thing + d: another thing +` + assertResult(t, expectedOutput, result.Output) +} + func TestDeleteYaml(t *testing.T) { content := `a: 2 b: @@ -597,6 +878,102 @@ b: assertResult(t, expectedOutput, result.Output) } +func TestDeleteSplatYaml(t *testing.T) { + content := `a: 2 +b: + hi: + c: things + d: something else + hello: + c: things2 + d: something else2 + there: + c: more things + d: more something else +` + filename := writeTempYamlFile(content) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("delete -v %s b.*.c", filename)) + if result.Error != nil { + t.Error(result.Error) + } + + expectedOutput := `a: 2 +b: + hi: + d: something else + hello: + d: something else2 + there: + d: more something else +` + assertResult(t, expectedOutput, result.Output) +} + +func TestDeleteSplatArrayYaml(t *testing.T) { + content := `a: 2 +b: + hi: + - thing: item1 + name: fred + - thing: item2 + name: sam +` + filename := writeTempYamlFile(content) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("delete -v %s b.hi[*].thing", filename)) + if result.Error != nil { + t.Error(result.Error) + } + + expectedOutput := `a: 2 +b: + hi: + - name: fred + - name: sam +` + assertResult(t, expectedOutput, result.Output) +} + +func TestDeleteSplatPrefixYaml(t *testing.T) { + content := `a: 2 +b: + hi: + c: things + d: something else + there: + c: more things + d: more something else + there2: + c: more things also + d: more something else also +` + filename := writeTempYamlFile(content) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("delete -v %s b.there*.c", filename)) + if result.Error != nil { + t.Error(result.Error) + } + + expectedOutput := `a: 2 +b: + hi: + c: things + d: something else + there: + d: more something else + there2: + d: more something else also +` + assertResult(t, expectedOutput, result.Output) +} + func TestDeleteYamlArray(t *testing.T) { content := `- 1 - 2 @@ -824,7 +1201,12 @@ func TestMergeCmd_ErrorUnreadableFile(t *testing.T) { if result.Error == nil { t.Error("Expected command to fail due to unknown file") } - expectedOutput := `Error updating document at index 0: open fake-unknown: no such file or directory` + var expectedOutput string + if runtime.GOOS == "windows" { + expectedOutput = `Error updating document at index 0: open fake-unknown: The system cannot find the file specified.` + } else { + expectedOutput = `Error updating document at index 0: open fake-unknown: no such file or directory` + } assertResult(t, expectedOutput, result.Error.Error()) } @@ -846,6 +1228,10 @@ c: func TestMergeCmd_Inplace(t *testing.T) { filename := writeTempYamlFile(readTempYamlFile("examples/data1.yaml")) + err := os.Chmod(filename, os.FileMode(int(0666))) + if err != nil { + t.Error(err) + } defer removeTempYamlFile(filename) cmd := getRootCommand() @@ -853,6 +1239,7 @@ func TestMergeCmd_Inplace(t *testing.T) { if result.Error != nil { t.Error(result.Error) } + info, _ := os.Stat(filename) gotOutput := readTempYamlFile(filename) expectedOutput := `a: simple b: @@ -861,6 +1248,21 @@ b: c: test: 1` assertResult(t, expectedOutput, strings.Trim(gotOutput, "\n ")) + assertResult(t, os.FileMode(int(0666)), info.Mode()) +} + +func TestMergeAllowEmptyCmd(t *testing.T) { + cmd := getRootCommand() + result := runCmd(cmd, "merge --allow-empty examples/data1.yaml examples/empty.yaml") + if result.Error != nil { + t.Error(result.Error) + } + expectedOutput := `a: simple +b: +- 1 +- 2 +` + assertResult(t, expectedOutput, result.Output) } func TestLintCmd_OK(t *testing.T) { diff --git a/data_navigator.go b/data_navigator.go index 589257a5..79157dd4 100644 --- a/data_navigator.go +++ b/data_navigator.go @@ -2,26 +2,38 @@ package main import ( "fmt" + "reflect" "strconv" + "strings" yaml "gopkg.in/mikefarah/yaml.v2" ) -func entryInSlice(context yaml.MapSlice, key interface{}) *yaml.MapItem { +func matchesKey(key string, actual interface{}) bool { + var actualString = fmt.Sprintf("%v", actual) + var prefixMatch = strings.TrimSuffix(key, "*") + if prefixMatch != key { + return strings.HasPrefix(actualString, prefixMatch) + } + return actualString == key +} + +func entriesInSlice(context yaml.MapSlice, key string) []*yaml.MapItem { + var matches = make([]*yaml.MapItem, 0) for idx := range context { var entry = &context[idx] - if entry.Key == key { - return entry + if matchesKey(key, entry.Key) { + matches = append(matches, entry) } } - return nil + return matches } func getMapSlice(context interface{}) yaml.MapSlice { var mapSlice yaml.MapSlice - switch context.(type) { + switch context := context.(type) { case yaml.MapSlice: - mapSlice = context.(yaml.MapSlice) + mapSlice = context default: mapSlice = make(yaml.MapSlice, 0) } @@ -29,9 +41,9 @@ func getMapSlice(context interface{}) yaml.MapSlice { } func getArray(context interface{}) (array []interface{}, ok bool) { - switch context.(type) { + switch context := context.(type) { case []interface{}: - array = context.([]interface{}) + array = context ok = true default: array = make([]interface{}, 0) @@ -40,27 +52,33 @@ func getArray(context interface{}) (array []interface{}, ok bool) { return } -func writeMap(context interface{}, paths []string, value interface{}) yaml.MapSlice { - log.Debugf("writeMap for %v for %v with value %v\n", paths, context, value) +func writeMap(context interface{}, paths []string, value interface{}) interface{} { + log.Debugf("writeMap with path %v for %v to set value %v\n", paths, context, value) mapSlice := getMapSlice(context) if len(paths) == 0 { - return mapSlice + return context } - child := entryInSlice(mapSlice, paths[0]) - if child == nil { + children := entriesInSlice(mapSlice, paths[0]) + + if len(children) == 0 && paths[0] == "*" { + log.Debugf("\tNo matches, return map as is") + return context + } + + if len(children) == 0 { newChild := yaml.MapItem{Key: paths[0]} mapSlice = append(mapSlice, newChild) - child = entryInSlice(mapSlice, paths[0]) + children = entriesInSlice(mapSlice, paths[0]) log.Debugf("\tAppended child at %v for mapSlice %v\n", paths[0], mapSlice) } - log.Debugf("\tchild.Value %v\n", child.Value) - remainingPaths := paths[1:] - child.Value = updatedChildValue(child.Value, remainingPaths, value) + for _, child := range children { + child.Value = updatedChildValue(child.Value, remainingPaths, value) + } log.Debugf("\tReturning mapSlice %v\n", mapSlice) return mapSlice } @@ -69,19 +87,26 @@ func updatedChildValue(child interface{}, remainingPaths []string, value interfa if len(remainingPaths) == 0 { return value } + log.Debugf("updatedChildValue for child %v with path %v to set value %v", child, remainingPaths, value) + log.Debugf("type of child is %v", reflect.TypeOf(child)) - _, nextIndexErr := strconv.ParseInt(remainingPaths[0], 10, 64) - if nextIndexErr != nil && remainingPaths[0] != "+" { - // must be a map - return writeMap(child, remainingPaths, value) + switch child := child.(type) { + case nil: + if remainingPaths[0] == "+" || remainingPaths[0] == "*" { + return writeArray(child, remainingPaths, value) + } + case []interface{}: + _, nextIndexErr := strconv.ParseInt(remainingPaths[0], 10, 64) + arrayCommand := nextIndexErr == nil || remainingPaths[0] == "+" || remainingPaths[0] == "*" + if arrayCommand { + return writeArray(child, remainingPaths, value) + } } - - // must be an array - return writeArray(child, remainingPaths, value) + return writeMap(child, remainingPaths, value) } func writeArray(context interface{}, paths []string, value interface{}) []interface{} { - log.Debugf("writeArray for %v for %v with value %v\n", paths, context, value) + log.Debugf("writeArray with path %v for %v to set value %v\n", paths, context, value) array, _ := getArray(context) if len(paths) == 0 { @@ -91,10 +116,16 @@ func writeArray(context interface{}, paths []string, value interface{}) []interf log.Debugf("\tarray %v\n", array) rawIndex := paths[0] + remainingPaths := paths[1:] var index int64 // the append array indicator if rawIndex == "+" { index = int64(len(array)) + } else if rawIndex == "*" { + for index, oldChild := range array { + array[index] = updatedChildValue(oldChild, remainingPaths, value) + } + return array } else { index, _ = strconv.ParseInt(rawIndex, 10, 64) // nolint // writeArray is only called by updatedChildValue which handles parsing the @@ -108,23 +139,34 @@ func writeArray(context interface{}, paths []string, value interface{}) []interf log.Debugf("\tcurrentChild %v\n", currentChild) - remainingPaths := paths[1:] array[index] = updatedChildValue(currentChild, remainingPaths, value) log.Debugf("\tReturning array %v\n", array) return array } func readMap(context yaml.MapSlice, head string, tail []string) (interface{}, error) { + log.Debugf("readingMap %v with key %v\n", context, head) if head == "*" { return readMapSplat(context, tail) } - var value interface{} - entry := entryInSlice(context, head) - if entry != nil { - value = entry.Value + entries := entriesInSlice(context, head) + if len(entries) == 1 { + return calculateValue(entries[0].Value, tail) + } else if len(entries) == 0 { + return nil, nil } - return calculateValue(value, tail) + var errInIdx error + values := make([]interface{}, len(entries)) + for idx, entry := range entries { + values[idx], errInIdx = calculateValue(entry.Value, tail) + if errInIdx != nil { + log.Errorf("Error updating index %v in %v", idx, context) + return nil, errInIdx + } + + } + return values, nil } func readMapSplat(context yaml.MapSlice, tail []string) (interface{}, error) { @@ -146,18 +188,18 @@ func readMapSplat(context yaml.MapSlice, tail []string) (interface{}, error) { } func recurse(value interface{}, head string, tail []string) (interface{}, error) { - switch value.(type) { + switch value := value.(type) { case []interface{}: if head == "*" { - return readArraySplat(value.([]interface{}), tail) + return readArraySplat(value, tail) } index, err := strconv.ParseInt(head, 10, 64) if err != nil { - return nil, fmt.Errorf("Error accessing array: %v", err) + return nil, fmt.Errorf("error accessing array: %v", err) } - return readArray(value.([]interface{}), index, tail) + return readArray(value, index, tail) case yaml.MapSlice: - return readMap(value.(yaml.MapSlice), head, tail) + return readMap(value, head, tail) default: return nil, nil } @@ -191,39 +233,47 @@ func calculateValue(value interface{}, tail []string) (interface{}, error) { return value, nil } -func deleteMap(context interface{}, paths []string) yaml.MapSlice { +func deleteMap(context interface{}, paths []string) (yaml.MapSlice, error) { log.Debugf("deleteMap for %v for %v\n", paths, context) mapSlice := getMapSlice(context) if len(paths) == 0 { - return mapSlice + return mapSlice, nil } - var found bool var index int var child yaml.MapItem for index, child = range mapSlice { - if child.Key == paths[0] { - found = true - break + if matchesKey(paths[0], child.Key) { + log.Debugf("\tMatched [%v] with [%v] at index %v", paths[0], child.Key, index) + var badDelete error + mapSlice, badDelete = deleteEntryInMap(mapSlice, child, index, paths) + if badDelete != nil { + return nil, badDelete + } } } - if !found { - return mapSlice - } + return mapSlice, nil +} + +func deleteEntryInMap(original yaml.MapSlice, child yaml.MapItem, index int, paths []string) (yaml.MapSlice, error) { remainingPaths := paths[1:] var newSlice yaml.MapSlice if len(remainingPaths) > 0 { newChild := yaml.MapItem{Key: child.Key} - newChild.Value = deleteChildValue(child.Value, remainingPaths) + var errorDeleting error + newChild.Value, errorDeleting = deleteChildValue(child.Value, remainingPaths) + if errorDeleting != nil { + return nil, errorDeleting + } - newSlice = make(yaml.MapSlice, len(mapSlice)) - for i := range mapSlice { - item := mapSlice[i] + newSlice = make(yaml.MapSlice, len(original)) + for i := range original { + item := original[i] if i == index { item = newChild } @@ -231,32 +281,43 @@ func deleteMap(context interface{}, paths []string) yaml.MapSlice { } } else { // Delete item from slice at index - newSlice = append(mapSlice[:index], mapSlice[index+1:]...) - log.Debugf("\tDeleted item index %d from mapSlice", index) + newSlice = append(original[:index], original[index+1:]...) + log.Debugf("\tDeleted item index %d from original", index) } - log.Debugf("\t\tlen: %d\tcap: %d\tslice: %v", len(mapSlice), cap(mapSlice), mapSlice) - log.Debugf("\tReturning mapSlice %v\n", mapSlice) - return newSlice + log.Debugf("\tReturning original %v\n", original) + return newSlice, nil } -func deleteArray(context interface{}, paths []string, index int64) interface{} { - log.Debugf("deleteArray for %v for %v\n", paths, context) - - array, ok := getArray(context) - if !ok { - // did not get an array - return context +func deleteArraySplat(array []interface{}, tail []string) (interface{}, error) { + log.Debugf("deleteArraySplat for %v for %v\n", tail, array) + var newArray = make([]interface{}, len(array)) + for index, value := range array { + val, err := deleteChildValue(value, tail) + if err != nil { + return nil, err + } + newArray[index] = val } + return newArray, nil +} + +func deleteArray(array []interface{}, paths []string, index int64) (interface{}, error) { + log.Debugf("deleteArray for %v for %v\n", paths, array) if index >= int64(len(array)) { - return array + return array, nil } remainingPaths := paths[1:] if len(remainingPaths) > 0 { // Recurse into the array element at index - array[index] = deleteMap(array[index], remainingPaths) + var errorDeleting error + array[index], errorDeleting = deleteMap(array[index], remainingPaths) + if errorDeleting != nil { + return nil, errorDeleting + } + } else { // Delete the array element at index array = append(array[:index], array[index+1:]...) @@ -264,19 +325,25 @@ func deleteArray(context interface{}, paths []string, index int64) interface{} { } log.Debugf("\tReturning array: %v\n", array) - return array + return array, nil } -func deleteChildValue(child interface{}, remainingPaths []string) interface{} { +func deleteChildValue(child interface{}, remainingPaths []string) (interface{}, error) { log.Debugf("deleteChildValue for %v for %v\n", remainingPaths, child) - - idx, nextIndexErr := strconv.ParseInt(remainingPaths[0], 10, 64) - if nextIndexErr != nil { - // must be a map - log.Debugf("\tdetected a map, invoking deleteMap\n") + var head = remainingPaths[0] + var tail = remainingPaths[1:] + switch child := child.(type) { + case yaml.MapSlice: return deleteMap(child, remainingPaths) + case []interface{}: + if head == "*" { + return deleteArraySplat(child, tail) + } + index, err := strconv.ParseInt(head, 10, 64) + if err != nil { + return nil, fmt.Errorf("error accessing array: %v", err) + } + return deleteArray(child, remainingPaths, index) } - - log.Debugf("\tdetected an array, so traversing element with index %d\n", idx) - return deleteArray(child, remainingPaths, idx) + return child, nil } diff --git a/data_navigator_test.go b/data_navigator_test.go index 9a3fce08..206346ff 100644 --- a/data_navigator_test.go +++ b/data_navigator_test.go @@ -4,8 +4,6 @@ import ( "fmt" "sort" "testing" - - yaml "gopkg.in/mikefarah/yaml.v2" ) func TestReadMap_simple(t *testing.T) { @@ -18,18 +16,37 @@ b: assertResult(t, 2, got) } +func TestReadMap_numberKey(t *testing.T) { + var data = parseData(` +--- +200: things +`) + got, _ := readMap(data, "200", []string{}) + assertResult(t, "things", got) +} + func TestReadMap_splat(t *testing.T) { var data = parseData(` --- mapSplat: item1: things item2: whatever + otherThing: cat `) res, _ := readMap(data, "mapSplat", []string{"*"}) - result := res.([]interface{}) - var actual = []string{result[0].(string), result[1].(string)} - sort.Strings(actual) - assertResult(t, "[things whatever]", fmt.Sprintf("%v", actual)) + assertResult(t, "[things whatever cat]", fmt.Sprintf("%v", res)) +} + +func TestReadMap_prefixSplat(t *testing.T) { + var data = parseData(` +--- +mapSplat: + item1: things + item2: whatever + otherThing: cat +`) + res, _ := readMap(data, "mapSplat", []string{"item*"}) + assertResult(t, "[things whatever]", fmt.Sprintf("%v", res)) } func TestReadMap_deep_splat(t *testing.T) { @@ -92,7 +109,7 @@ b: if err == nil { t.Fatal("Expected error due to invalid path") } - expectedOutput := `Error accessing array: strconv.ParseInt: parsing "x": invalid syntax` + expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax` assertResult(t, expectedOutput, err.Error()) } @@ -112,7 +129,7 @@ b: if err == nil { t.Fatal("Expected error due to invalid path") } - expectedOutput := `Error accessing array: strconv.ParseInt: parsing "x": invalid syntax` + expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax` assertResult(t, expectedOutput, err.Error()) } @@ -132,7 +149,7 @@ b: if err == nil { t.Fatal("Expected error due to invalid path") } - expectedOutput := `Error accessing array: strconv.ParseInt: parsing "x": invalid syntax` + expectedOutput := `error accessing array: strconv.ParseInt: parsing "x": invalid syntax` assertResult(t, expectedOutput, err.Error()) } @@ -180,8 +197,7 @@ func TestWrite_really_simple(t *testing.T) { `) updated := writeMap(data, []string{"b"}, "4") - b := entryInSlice(updated, "b").Value - assertResult(t, "4", b) + assertResult(t, "[{b 4}]", fmt.Sprintf("%v", updated)) } func TestWrite_simple(t *testing.T) { @@ -191,9 +207,7 @@ b: `) updated := writeMap(data, []string{"b", "c"}, "4") - b := entryInSlice(updated, "b").Value.(yaml.MapSlice) - c := entryInSlice(b, "c").Value - assertResult(t, "4", c) + assertResult(t, "[{b [{c 4}]}]", fmt.Sprintf("%v", updated)) } func TestWrite_new(t *testing.T) { @@ -203,9 +217,7 @@ b: `) updated := writeMap(data, []string{"b", "d"}, "4") - b := entryInSlice(updated, "b").Value.(yaml.MapSlice) - d := entryInSlice(b, "d").Value - assertResult(t, "4", d) + assertResult(t, "[{b [{c 2} {d 4}]}]", fmt.Sprintf("%v", updated)) } func TestWrite_new_deep(t *testing.T) { @@ -215,8 +227,7 @@ b: `) updated := writeMap(data, []string{"b", "d", "f"}, "4") - got, _ := readMap(updated, "b", []string{"d", "f"}) - assertResult(t, "4", got) + assertResult(t, "[{b [{c 2} {d [{f 4}]}]}]", fmt.Sprintf("%v", updated)) } func TestWrite_array(t *testing.T) { @@ -227,8 +238,7 @@ b: updated := writeMap(data, []string{"b", "0"}, "bb") - b := entryInSlice(updated, "b").Value.([]interface{}) - assertResult(t, "bb", b[0].(string)) + assertResult(t, "[{b [bb]}]", fmt.Sprintf("%v", updated)) } func TestWrite_new_array(t *testing.T) { @@ -238,20 +248,19 @@ b: `) updated := writeMap(data, []string{"b", "0"}, "4") - got, _ := readMap(updated, "b", []string{"0"}) - assertResult(t, "4", got) + assertResult(t, "[{b [{c 2} {0 4}]}]", fmt.Sprintf("%v", updated)) } func TestWrite_new_array_deep(t *testing.T) { var data = parseData(` -b: - c: 2 +a: apple `) - var expected = `b: + var expected = `a: apple +b: - c: "4"` - updated := writeMap(data, []string{"b", "0", "c"}, "4") + updated := writeMap(data, []string{"b", "+", "c"}, "4") got, _ := yamlToString(updated) assertResult(t, expected, got) } @@ -261,10 +270,14 @@ func TestWrite_new_map_array_deep(t *testing.T) { b: c: 2 `) + var expected = `b: + c: 2 + d: + - "4"` - updated := writeMap(data, []string{"b", "d", "0"}, "4") - got, _ := readMap(updated, "b", []string{"d", "0"}) - assertResult(t, "4", got) + updated := writeMap(data, []string{"b", "d", "+"}, "4") + got, _ := yamlToString(updated) + assertResult(t, expected, got) } func TestWrite_add_to_array(t *testing.T) { @@ -289,8 +302,7 @@ b: `) updated := writeMap(data, []string{"b"}, "4") - b := entryInSlice(updated, "b").Value - assertResult(t, "4", fmt.Sprintf("%v", b)) + assertResult(t, "[{b 4}]", fmt.Sprintf("%v", updated)) } func TestWriteMap_no_paths(t *testing.T) { @@ -318,7 +330,7 @@ b: 456 b: 456 `) - result := deleteMap(data, []string{"a"}) + result, _ := deleteMap(data, []string{"a"}) assertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result)) } @@ -327,7 +339,7 @@ func TestDelete_index_to_string(t *testing.T) { var data = parseData(` a: mystring `) - result := deleteMap(data, []string{"a", "0"}) + result, _ := deleteMap(data, []string{"a", "0"}) assertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result)) } @@ -338,7 +350,7 @@ a: [3, 4] var expected = parseData(` a: [3] `) - result := deleteMap(data, []string{"a", "1"}) + result, _ := deleteMap(data, []string{"a", "1"}) assertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result)) } @@ -346,7 +358,7 @@ func TestDelete_list_index_beyond_bounds(t *testing.T) { var data = parseData(` a: [3, 4] `) - result := deleteMap(data, []string{"a", "5"}) + result, _ := deleteMap(data, []string{"a", "5"}) assertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result)) } @@ -354,7 +366,7 @@ func TestDelete_list_index_out_of_bounds_by_1(t *testing.T) { var data = parseData(` a: [3, 4] `) - result := deleteMap(data, []string{"a", "2"}) + result, _ := deleteMap(data, []string{"a", "2"}) assertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result)) } @@ -364,7 +376,7 @@ a: [3, 4] b: - name: test `) - result := deleteMap(data, []string{}) + result, _ := deleteMap(data, []string{}) assertResult(t, fmt.Sprintf("%v", data), fmt.Sprintf("%v", result)) } @@ -382,6 +394,6 @@ b: - name: john value: test `) - result := deleteMap(data, []string{"b", "0", "name"}) + result, _ := deleteMap(data, []string{"b", "0", "name"}) assertResult(t, fmt.Sprintf("%v", expected), fmt.Sprintf("%v", result)) } diff --git a/debian/changelog b/debian/changelog index e56acca2..60b55ba1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,15 @@ +yq (2.2-1) bionic; urgency=medium + + * Added Windows support for the "--inplace" command flag + * Prefix now supports arrays + * Add prefix command + * Bump Alpine version to 3.8 + * Improved docker build process + * Lint fixes + * Build support for all linux architectures supported by gox + + -- Roberto Mier Escandon Sat, 19 Jan 2019 15:50:47 +0100 + yq (2.1-0) bionic; urgency=medium * Ability to read multiple documents in a single file diff --git a/debian/control b/debian/control index 65a184c0..53cee5f9 100644 --- a/debian/control +++ b/debian/control @@ -12,7 +12,7 @@ Vcs-Browser: https://github.com/mikefarah/yq.git Vcs-Git: https://github.com/mikefarah/yq.git Package: yq -Architecture: all +Architecture: any Built-Using: ${misc:Built-Using} Depends: ${shlibs:Depends}, ${misc:Depends} diff --git a/docs/404.html b/docs/404.html index 38e09cfb..d9cfab02 100644 --- a/docs/404.html +++ b/docs/404.html @@ -2,7 +2,7 @@ - + @@ -32,7 +32,7 @@ - + @@ -40,16 +40,16 @@ - + - + - + @@ -57,6 +57,10 @@ + + + + @@ -95,7 +99,7 @@