From 8d6e3a6a750849b783e2ea86f2864e95f5d91e86 Mon Sep 17 00:00:00 2001 From: Thad Craft Date: Wed, 24 Oct 2018 15:00:43 -0500 Subject: [PATCH 01/54] copying file permissions from original file when inline merging - Closes #180 --- commands_test.go | 6 +++++- yq.go | 12 ++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/commands_test.go b/commands_test.go index 6bbc8bce..fec2151d 100644 --- a/commands_test.go +++ b/commands_test.go @@ -2,6 +2,7 @@ package main import ( "fmt" + "os" "strings" "testing" @@ -600,7 +601,7 @@ b: func TestDeleteYamlArray(t *testing.T) { content := `- 1 - 2 -- 3 +- 3 ` filename := writeTempYamlFile(content) defer removeTempYamlFile(filename) @@ -846,6 +847,7 @@ c: func TestMergeCmd_Inplace(t *testing.T) { filename := writeTempYamlFile(readTempYamlFile("examples/data1.yaml")) + os.Chmod(filename, os.FileMode(int(0666))) defer removeTempYamlFile(filename) cmd := getRootCommand() @@ -853,6 +855,7 @@ func TestMergeCmd_Inplace(t *testing.T) { if result.Error != nil { t.Error(result.Error) } + info, _ := os.Stat(filename) gotOutput := readTempYamlFile(filename) expectedOutput := `a: simple b: @@ -861,4 +864,5 @@ b: c: test: 1` assertResult(t, expectedOutput, strings.Trim(gotOutput, "\n ")) + assertResult(t, os.FileMode(int(0666)), info.Mode()) } diff --git a/yq.go b/yq.go index b21bb01d..2e403f89 100644 --- a/yq.go +++ b/yq.go @@ -11,10 +11,10 @@ import ( "strings" errors "github.com/pkg/errors" - "gopkg.in/spf13/cobra.v0" yaml "gopkg.in/mikefarah/yaml.v2" logging "gopkg.in/op/go-logging.v1" + cobra "gopkg.in/spf13/cobra.v0" ) var trimOutput = true @@ -398,7 +398,15 @@ func readAndUpdate(stdOut io.Writer, inputFile string, updateData updateDataFn) var destination io.Writer var destinationName string if writeInplace { - var tempFile, err = ioutil.TempFile("", "temp") + info, err := os.Stat(inputFile) + if err != nil { + return err + } + tempFile, err := ioutil.TempFile("", "temp") + if err != nil { + return err + } + err = tempFile.Chmod(info.Mode()) if err != nil { return err } From a9ade5a832951f900d976d0e35391b94c4b853f2 Mon Sep 17 00:00:00 2001 From: Thad Craft Date: Wed, 24 Oct 2018 15:04:05 -0500 Subject: [PATCH 02/54] fixing test lint --- commands_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/commands_test.go b/commands_test.go index fec2151d..f3032e75 100644 --- a/commands_test.go +++ b/commands_test.go @@ -847,7 +847,10 @@ c: func TestMergeCmd_Inplace(t *testing.T) { filename := writeTempYamlFile(readTempYamlFile("examples/data1.yaml")) - os.Chmod(filename, os.FileMode(int(0666))) + err := os.Chmod(filename, os.FileMode(int(0666))) + if err != nil { + t.Error(err) + } defer removeTempYamlFile(filename) cmd := getRootCommand() From 90fe9c651259ac53f6f8bb18b4dd2c671688d0ea Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Thu, 25 Oct 2018 15:57:41 +1100 Subject: [PATCH 03/54] Version bump --- snap/snapcraft.yaml | 2 +- test.yml | 2 ++ version.go | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 test.yml diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 7234f47c..ee0112d7 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,5 +1,5 @@ name: yq -version: 2.1.0 +version: 2.1.2 summary: A lightweight and portable command-line YAML processor description: | The aim of the project is to be the jq or sed of yaml files. diff --git a/test.yml b/test.yml new file mode 100644 index 00000000..71f1098c --- /dev/null +++ b/test.yml @@ -0,0 +1,2 @@ +a: apple +c: cat diff --git a/version.go b/version.go index ec21812c..35613018 100644 --- a/version.go +++ b/version.go @@ -11,7 +11,7 @@ var ( GitDescribe string // Version is main version number that is being run at the moment. - Version = "2.1.1" + Version = "2.1.2" // VersionPrerelease is a pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release From 785ee68a7679efb91c62bf4a342566b0f977914d Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Thu, 25 Oct 2018 17:49:46 +1100 Subject: [PATCH 04/54] Improved docker build process --- Dockerfile | 5 ++++- release_instructions.txt | 1 + scripts/publish-docker.sh | 9 +++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) create mode 100755 scripts/publish-docker.sh diff --git a/Dockerfile b/Dockerfile index c89f0ec1..b82fac25 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,9 +16,12 @@ RUN CGO_ENABLED=0 make local build # Choose alpine as a base image to make this useful for CI, as many # CI tools expect an interactive shell inside the container -FROM alpine:3.7 +FROM alpine:3.7 as production COPY --from=builder /go/src/mikefarah/yq/yq /usr/bin/yq RUN chmod +x /usr/bin/yq +ARG VERSION=none +LABEL version=${VERSION} + WORKDIR /workdir diff --git a/release_instructions.txt b/release_instructions.txt index 1e45b2c8..db873d8f 100644 --- a/release_instructions.txt +++ b/release_instructions.txt @@ -25,6 +25,7 @@ - docker - build and push latest and new version tag + - docker build . --arg -t mikefarah/yq:latest -t mikefarah/yq:VERSION - debian package - execute diff --git a/scripts/publish-docker.sh b/scripts/publish-docker.sh new file mode 100755 index 00000000..315eb197 --- /dev/null +++ b/scripts/publish-docker.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -ex +VERSION="$(git describe --tags --abbrev=0)" +docker build \ + --target production \ + --build-arg VERSION=${VERSION} \ + -t mikefarah/yq:latest \ + -t mikefarah/yq:${VERSION} \ + . \ No newline at end of file From ef579d4ccf6ea4eb6dd7b58c4604c495f174c4ef Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Tue, 30 Oct 2018 13:18:52 +1100 Subject: [PATCH 05/54] Updated release instructions --- release_instructions.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release_instructions.txt b/release_instructions.txt index db873d8f..28458346 100644 --- a/release_instructions.txt +++ b/release_instructions.txt @@ -17,11 +17,11 @@ sudo snap remove yq sudo snap install --edge yq - then on the mac snapcraft release yq stable + then use the UI (https://snapcraft.io/yq/release) - brew - - create pull request pointing to latest git release + - brew bump-formula-pr --url=https://github.com/mikefarah/yq/archive/2.1.2.tar.gz yq - docker - build and push latest and new version tag From d7040e393307ad28493aa8ab1e4e7acfe808005d Mon Sep 17 00:00:00 2001 From: Matthias Fax Date: Sun, 18 Nov 2018 16:55:24 +0100 Subject: [PATCH 06/54] Bump Alpine version to 3.8 There has been a security issue with older Alpine versions and their package manager. https://justi.cz/security/2018/09/13/alpine-apk-rce.html Not sure about 3.7, but the latest 3.8 image has it fixed. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b82fac25..e7d75a42 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN CGO_ENABLED=0 make local build # Choose alpine as a base image to make this useful for CI, as many # CI tools expect an interactive shell inside the container -FROM alpine:3.7 as production +FROM alpine:3.8 as production COPY --from=builder /go/src/mikefarah/yq/yq /usr/bin/yq RUN chmod +x /usr/bin/yq From 48dcc152813999e37fda53b50f1e3c6281be823a Mon Sep 17 00:00:00 2001 From: matfax Date: Sun, 18 Nov 2018 16:35:28 +0100 Subject: [PATCH 07/54] feat: add prefix command --- commands_test.go | 174 +++++++++++++++++++++++++++++++++++++++++++++++ mkdocs/prefix.md | 96 ++++++++++++++++++++++++++ yq.go | 57 ++++++++++++++++ 3 files changed, 327 insertions(+) create mode 100644 mkdocs/prefix.md diff --git a/commands_test.go b/commands_test.go index f3032e75..7ff5efb4 100644 --- a/commands_test.go +++ b/commands_test.go @@ -341,6 +341,180 @@ func TestReadCmd_ToJsonLong(t *testing.T) { assertResult(t, "2\n", result.Output) } +func TestPrefixCmd(t *testing.T) { + content := `b: + c: 3 +` + filename := writeTempYamlFile(content) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("prefix %s d", filename)) + if result.Error != nil { + t.Error(result.Error) + } + expectedOutput := `d: + b: + c: 3 +` + assertResult(t, expectedOutput, result.Output) +} + +func TestPrefixCmd_MultiLayer(t *testing.T) { + content := `b: + c: 3 +` + filename := writeTempYamlFile(content) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("prefix %s d.e.f", filename)) + if result.Error != nil { + t.Error(result.Error) + } + expectedOutput := `d: + e: + f: + b: + c: 3 +` + assertResult(t, expectedOutput, result.Output) +} + +func TestPrefixMultiCmd(t *testing.T) { + content := `b: + c: 3 +--- +apples: great +` + filename := writeTempYamlFile(content) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("prefix %s -d 1 d", filename)) + if result.Error != nil { + t.Error(result.Error) + } + expectedOutput := `b: + c: 3 +--- +d: + apples: great +` + assertResult(t, expectedOutput, result.Output) +} +func TestPrefixInvalidDocumentIndexCmd(t *testing.T) { + content := `b: + c: 3 +` + filename := writeTempYamlFile(content) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("prefix %s -df d", filename)) + if result.Error == nil { + t.Error("Expected command to fail due to invalid path") + } + expectedOutput := `Document index f is not a integer or *: strconv.ParseInt: parsing "f": invalid syntax` + assertResult(t, expectedOutput, result.Error.Error()) +} + +func TestPrefixBadDocumentIndexCmd(t *testing.T) { + content := `b: + c: 3 +` + filename := writeTempYamlFile(content) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("prefix %s -d 1 d", filename)) + if result.Error == nil { + t.Error("Expected command to fail due to invalid path") + } + expectedOutput := `Asked to process document index 1 but there are only 1 document(s)` + assertResult(t, expectedOutput, result.Error.Error()) +} +func TestPrefixMultiAllCmd(t *testing.T) { + content := `b: + c: 3 +--- +apples: great +` + filename := writeTempYamlFile(content) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("prefix %s -d * d", filename)) + if result.Error != nil { + t.Error(result.Error) + } + expectedOutput := `d: + b: + c: 3 +--- +d: + apples: great` + assertResult(t, expectedOutput, strings.Trim(result.Output, "\n ")) +} + +func TestPrefixCmd_Error(t *testing.T) { + cmd := getRootCommand() + result := runCmd(cmd, "prefix") + if result.Error == nil { + t.Error("Expected command to fail due to missing arg") + } + expectedOutput := `Must provide ` + assertResult(t, expectedOutput, result.Error.Error()) +} + +func TestPrefixCmd_ErrorUnreadableFile(t *testing.T) { + cmd := getRootCommand() + result := runCmd(cmd, "prefix fake-unknown a.b") + if result.Error == nil { + t.Error("Expected command to fail due to unknown file") + } + expectedOutput := `open fake-unknown: no such file or directory` + assertResult(t, expectedOutput, result.Error.Error()) +} + +func TestPrefixCmd_Verbose(t *testing.T) { + content := `b: + c: 3 +` + filename := writeTempYamlFile(content) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("-v prefix %s x", filename)) + if result.Error != nil { + t.Error(result.Error) + } + expectedOutput := `x: + b: + c: 3 +` + assertResult(t, expectedOutput, result.Output) +} + +func TestPrefixCmd_Inplace(t *testing.T) { + content := `b: + c: 3 +` + filename := writeTempYamlFile(content) + defer removeTempYamlFile(filename) + + cmd := getRootCommand() + result := runCmd(cmd, fmt.Sprintf("prefix -i %s d", filename)) + if result.Error != nil { + t.Error(result.Error) + } + gotOutput := readTempYamlFile(filename) + expectedOutput := `d: + b: + c: 3` + assertResult(t, expectedOutput, strings.Trim(gotOutput, "\n ")) +} + func TestNewCmd(t *testing.T) { cmd := getRootCommand() result := runCmd(cmd, "new b.c 3") diff --git a/mkdocs/prefix.md b/mkdocs/prefix.md new file mode 100644 index 00000000..0e0995fd --- /dev/null +++ b/mkdocs/prefix.md @@ -0,0 +1,96 @@ +Paths can be prefixed using the 'prefix' command. +The complete yaml content will be nested inside the new prefix path. + +``` +yq p +``` + +### To Stdout +Given a data1.yaml file of: +```yaml +a: simple +b: [1, 2] +``` +then +```bash +yq p data1.yaml c +``` +will output: +```yaml +c: + a: simple + b: [1, 2] +``` + +### Arbitrary depth +Given a data1.yaml file of: +```yaml +a: + b: [1, 2] +``` +then +```bash +yq p data1.yaml c.d +``` +will output: +```yaml +c: + d: + a: + b: [1, 2] +``` + +### Updating files in-place +Given a data1.yaml file of: +```yaml +a: simple +b: [1, 2] +``` +then +```bash +yq p -i data1.yaml c +``` +will update the data1.yaml file so that the path 'c' is prefixed to all other paths. + +### Multiple Documents - update a single document +Given a data1.yaml file of: +```yaml +something: else +--- +a: simple +b: cat +``` +then +```bash +yq p -d1 data1.yaml c +``` +will output: +```yaml +something: else +--- +c: + a: simple + b: cat +``` + +### Multiple Documents - update a single document +Given a data1.yaml file of: +```yaml +something: else +--- +a: simple +b: cat +``` +then +```bash +yq p -d'*' data1.yaml c +``` +will output: +```yaml +c: + something: else +--- +c: + a: simple + b: cat +``` diff --git a/yq.go b/yq.go index 2e403f89..293c731e 100644 --- a/yq.go +++ b/yq.go @@ -73,6 +73,7 @@ func newCommandCLI() *cobra.Command { rootCmd.AddCommand( createReadCmd(), createWriteCmd(), + createPrefixCmd(), createDeleteCmd(), createNewCmd(), createMergeCmd(), @@ -137,6 +138,28 @@ a.b.e: return cmdWrite } +func createPrefixCmd() *cobra.Command { + var cmdWrite = &cobra.Command{ + Use: "prefix [yaml_file] [path]", + Aliases: []string{"p"}, + Short: "yq p [--inplace/-i] [--doc/-d index] sample.yaml a.b.c", + Example: ` +yq prefix things.yaml a.b.c +yq prefix --inplace things.yaml a.b.c +yq p -i things.yaml a.b.c +yq p --doc 2 things.yaml a.b.d +yq p -d2 things.yaml a.b.d + `, + Long: `Prefixes w.r.t to the yaml file at the given path. +Outputs to STDOUT unless the inplace flag is used, in which case the file is updated instead. +`, + RunE: prefixProperty, + } + cmdWrite.PersistentFlags().BoolVarP(&writeInplace, "inplace", "i", false, "update the yaml file inplace") + cmdWrite.PersistentFlags().StringVarP(&docIndex, "doc", "d", "0", "process document index number (0 based, * for all documents)") + return cmdWrite +} + func createDeleteCmd() *cobra.Command { var cmdDelete = &cobra.Command{ Use: "delete [yaml_file] [path]", @@ -394,6 +417,40 @@ func writeProperty(cmd *cobra.Command, args []string) error { return readAndUpdate(cmd.OutOrStdout(), args[0], updateData) } +func prefixProperty(cmd *cobra.Command, args []string) error { + if len(args) != 2 { + return errors.New("Must provide ") + } + var updateAll, docIndexInt, errorParsingDocIndex = parseDocumentIndex() + if errorParsingDocIndex != nil { + return errorParsingDocIndex + } + + var paths = parsePath(args[1]) + + // Inverse order + for i := len(paths)/2 - 1; i >= 0; i-- { + opp := len(paths) - 1 - i + paths[i], paths[opp] = paths[opp], paths[i] + } + + var updateData = func(dataBucket interface{}, currentIndex int) (interface{}, error) { + + if updateAll || currentIndex == docIndexInt { + log.Debugf("Prefixing %v to doc %v", paths, currentIndex) + var mapDataBucket = dataBucket + for _, key := range paths { + nestedBucket := make(map[string]interface{}) + nestedBucket[key] = mapDataBucket + mapDataBucket = nestedBucket + } + return mapDataBucket, nil + } + return dataBucket, nil + } + return readAndUpdate(cmd.OutOrStdout(), args[0], updateData) +} + func readAndUpdate(stdOut io.Writer, inputFile string, updateData updateDataFn) error { var destination io.Writer var destinationName string From 8a3fb32f36a8d93cd3ca5cd52b69f14c3188b017 Mon Sep 17 00:00:00 2001 From: matfax Date: Sun, 18 Nov 2018 18:40:51 +0100 Subject: [PATCH 08/54] Update documentation --- README.md | 2 + docs/404.html | 33 +- .../javascripts/application.9e1f3b71.js | 1 + .../javascripts/application.a59e2a89.js | 1 - docs/assets/javascripts/lunr/lunr.da.js | 2 +- docs/assets/javascripts/lunr/lunr.de.js | 2 +- docs/assets/javascripts/lunr/lunr.du.js | 2 +- docs/assets/javascripts/lunr/lunr.es.js | 2 +- docs/assets/javascripts/lunr/lunr.fi.js | 2 +- docs/assets/javascripts/lunr/lunr.fr.js | 2 +- docs/assets/javascripts/lunr/lunr.hu.js | 2 +- docs/assets/javascripts/lunr/lunr.it.js | 2 +- docs/assets/javascripts/lunr/lunr.jp.js | 2 +- docs/assets/javascripts/lunr/lunr.multi.js | 2 +- docs/assets/javascripts/lunr/lunr.no.js | 2 +- docs/assets/javascripts/lunr/lunr.pt.js | 2 +- docs/assets/javascripts/lunr/lunr.ro.js | 2 +- docs/assets/javascripts/lunr/lunr.ru.js | 2 +- .../javascripts/lunr/lunr.stemmer.support.js | 2 +- docs/assets/javascripts/lunr/lunr.sv.js | 2 +- docs/assets/javascripts/lunr/lunr.tr.js | 2 +- docs/assets/javascripts/lunr/tinyseg.js | 2 +- docs/assets/javascripts/modernizr.1aa3b519.js | 1 - docs/assets/javascripts/modernizr.20ef595d.js | 1 + .../application-palette.22915126.css | 1176 ++++++++ .../application-palette.6079476c.css | 2 - .../stylesheets/application.11e41852.css | 2563 +++++++++++++++++ .../stylesheets/application.ba0fd1a6.css | 2 - docs/convert/index.html | 29 +- docs/create/index.html | 29 +- docs/delete/index.html | 33 +- docs/index.html | 35 +- docs/merge/index.html | 29 +- docs/prefix/index.html | 641 +++++ docs/read/index.html | 29 +- docs/search/search_index.json | 235 +- docs/sitemap.xml | 33 +- docs/sitemap.xml.gz | Bin 0 -> 199 bytes docs/snippets/keys_with_dots/index.html | 441 +++ docs/snippets/works_with_json/index.html | 395 +++ docs/write/index.html | 33 +- mkdocs.yml | 4 +- mkdocs/prefix.md | 4 +- 43 files changed, 5440 insertions(+), 348 deletions(-) create mode 100644 docs/assets/javascripts/application.9e1f3b71.js delete mode 100644 docs/assets/javascripts/application.a59e2a89.js delete mode 100644 docs/assets/javascripts/modernizr.1aa3b519.js create mode 100644 docs/assets/javascripts/modernizr.20ef595d.js create mode 100644 docs/assets/stylesheets/application-palette.22915126.css delete mode 100644 docs/assets/stylesheets/application-palette.6079476c.css create mode 100644 docs/assets/stylesheets/application.11e41852.css delete mode 100644 docs/assets/stylesheets/application.ba0fd1a6.css create mode 100644 docs/prefix/index.html create mode 100644 docs/sitemap.xml.gz create mode 100644 docs/snippets/keys_with_dots/index.html create mode 100644 docs/snippets/works_with_json/index.html diff --git a/README.md b/README.md index 8786e4b5..a8250093 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,7 @@ docker run -it -v ${PWD}:/workdir mikefarah/yq sh - Update creates any missing entries in the path on the fly - Create a yaml file given a deep path and value - Create a yaml file given a script file +- Prefix a path to a yaml file - Convert from json to yaml - Convert from yaml to json - Pipe data in by using '-' @@ -73,6 +74,7 @@ Available Commands: new yq n [--script/-s script_file] a.b.c newValue read yq r [--doc/-d index] sample.yaml a.b.c write yq w [--inplace/-i] [--script/-s script_file] [--doc/-d index] sample.yaml a.b.c newValue + prefix yq p [--inplace/-i] [--doc/-d index] sample.yaml a.b.c Flags: -h, --help help for yq diff --git a/docs/404.html b/docs/404.html index 38e09cfb..7a52591c 100644 --- a/docs/404.html +++ b/docs/404.html @@ -32,7 +32,7 @@ - + @@ -40,16 +40,16 @@ - + - + - + @@ -57,6 +57,7 @@ + @@ -95,7 +96,7 @@