From dde183386cd56bbe6148a31dd5e0020ec86a87b6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 12 Jul 2023 12:27:51 +1000 Subject: [PATCH 01/43] Bump golang.org/x/net from 0.11.0 to 0.12.0 (#1719) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.11.0 to 0.12.0. - [Commits](https://github.com/golang/net/compare/v0.11.0...v0.12.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 9c99e723..623f7f26 100644 --- a/go.mod +++ b/go.mod @@ -15,8 +15,8 @@ require ( github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 - golang.org/x/net v0.11.0 - golang.org/x/text v0.10.0 + golang.org/x/net v0.12.0 + golang.org/x/text v0.11.0 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 gopkg.in/yaml.v3 v3.0.1 ) @@ -25,7 +25,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect - golang.org/x/sys v0.9.0 // indirect + golang.org/x/sys v0.10.0 // indirect golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect ) diff --git a/go.sum b/go.sum index edaed309..597cb5fa 100644 --- a/go.sum +++ b/go.sum @@ -50,13 +50,13 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= -golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= -golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= -golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= From 9cb09182068b572030da606cbbefdce0cb826ca8 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Wed, 12 Jul 2023 12:31:25 +1000 Subject: [PATCH 02/43] Release notes --- release_notes.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/release_notes.txt b/release_notes.txt index 3b1a79a2..4f8dfac5 100644 --- a/release_notes.txt +++ b/release_notes.txt @@ -1,3 +1,7 @@ +4.34.2: + - Bumped dependencies + + 4.34.1: - Added shell output format thanks @giorgiga - Fixed nil pointer dereference (#1649) thanks @ArthurFritz From 0b34c9a00de1c575a34eea05af1d956a525c4fc1 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Wed, 12 Jul 2023 12:32:14 +1000 Subject: [PATCH 03/43] Bumping version --- cmd/version.go | 2 +- snap/snapcraft.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/version.go b/cmd/version.go index 8efc91cf..c0898fec 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -11,7 +11,7 @@ var ( GitDescribe string // Version is main version number that is being run at the moment. - Version = "v4.34.1" + Version = "v4.34.2" // VersionPrerelease is a pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index ea9a14a0..2ef9c1e7 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,5 +1,5 @@ name: yq -version: 'v4.34.1' +version: 'v4.34.2' summary: A lightweight and portable command-line YAML processor description: | The aim of the project is to be the jq or sed of yaml files. From a68be7debf7c76a1579e75c30ea31eb497bede4a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Jul 2023 08:52:19 +1000 Subject: [PATCH 04/43] Bump golang from 1.20.5 to 1.20.6 (#1725) Bumps golang from 1.20.5 to 1.20.6. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- Dockerfile.dev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9dc3e67d..6102d53a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.5 as builder +FROM golang:1.20.6 as builder WORKDIR /go/src/mikefarah/yq diff --git a/Dockerfile.dev b/Dockerfile.dev index 6b6150e0..bb36cb54 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM golang:1.20.5 +FROM golang:1.20.6 COPY scripts/devtools.sh /opt/devtools.sh From 1dee08aa38504dc16ec6528d833024129ba67090 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 28 Jul 2023 13:28:05 +1000 Subject: [PATCH 05/43] Bump github.com/pelletier/go-toml/v2 from 2.0.8 to 2.0.9 (#1732) Bumps [github.com/pelletier/go-toml/v2](https://github.com/pelletier/go-toml) from 2.0.8 to 2.0.9. - [Release notes](https://github.com/pelletier/go-toml/releases) - [Changelog](https://github.com/pelletier/go-toml/blob/v2/.goreleaser.yaml) - [Commits](https://github.com/pelletier/go-toml/compare/v2.0.8...v2.0.9) --- updated-dependencies: - dependency-name: github.com/pelletier/go-toml/v2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 623f7f26..850d2dee 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/goccy/go-yaml v1.11.0 github.com/jinzhu/copier v0.3.5 github.com/magiconair/properties v1.8.7 - github.com/pelletier/go-toml/v2 v2.0.8 + github.com/pelletier/go-toml/v2 v2.0.9 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 597cb5fa..7ff0c506 100644 --- a/go.sum +++ b/go.sum @@ -31,8 +31,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= -github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0= +github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -48,8 +48,8 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From 1d611f04fb0d6949388665700a63f87d9ecb8313 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Aug 2023 13:52:40 +1000 Subject: [PATCH 06/43] Bump golang.org/x/net from 0.12.0 to 0.13.0 (#1750) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.12.0 to 0.13.0. - [Commits](https://github.com/golang/net/compare/v0.12.0...v0.13.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 850d2dee..47d764ba 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 - golang.org/x/net v0.12.0 + golang.org/x/net v0.13.0 golang.org/x/text v0.11.0 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 7ff0c506..6e5f4ff7 100644 --- a/go.sum +++ b/go.sum @@ -50,8 +50,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY= +golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From 9b4082919bf50bb6be38742adf46f888e9f5683a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Aug 2023 13:52:57 +1000 Subject: [PATCH 07/43] Bump golang from 1.20.6 to 1.20.7 (#1749) Bumps golang from 1.20.6 to 1.20.7. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- Dockerfile.dev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 6102d53a..e091cc50 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.6 as builder +FROM golang:1.20.7 as builder WORKDIR /go/src/mikefarah/yq diff --git a/Dockerfile.dev b/Dockerfile.dev index bb36cb54..e1b15e72 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM golang:1.20.6 +FROM golang:1.20.7 COPY scripts/devtools.sh /opt/devtools.sh From d302d75c77b6fe18fe3c25210659d3e2ed01e788 Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Fri, 11 Aug 2023 04:56:49 +0200 Subject: [PATCH 08/43] Implement basic Lua output support (#1745) * Implement basic Lua output support Ref #1700 Basic but working serialization to Lua tables. * Escape larger set of characters in Lua output Started with a minimum of replacements, this should be more complete, tho not all substitutions are strictly required in Lua. * Print simple keys unquoted in Lua output String keys that satisfy the requirements for variable names can be used as keys without quotes in tables. * Quote Lua keywords in table keys Keywords are not valid as unquoted keys, thus must be quoted * Make output of unquoted Lua table keys optional Generally safer and simpler to not do it. * Hook up settings for Lua output * Allow special characters in Lua prefix and suffix --lua-suffix='});^M' didn't work, so taking this approach instead * Panic on unhandled YAML Kind in Lua encoder * Handle YAML case varied booleans in Lua encoder * Handle special-case numbers in Lua encoder * Reject unhandled scalar Tags in Lua encoder * Add note about how Lua nil is unsuitable as table key Could add some context tracking in the future to allow rejecting nil in a table key context. * Return error instead of panic in Lua encoder * Add initial test for Lua encoder Boilerplate mostly copied from toml_test.go * Additional Lua output tests * Generate Lua encoder documentation Mostly just for the boilerplate * Convert octal for Lua output Lua doesn't have the 0oNNN syntax for octal integers, only decimal and hexadecimal, hence those can be passed trough as is while octal needs special treatment. * Implement indentation in in Lua output * Respect string Style in Lua encoder Lua has 'single', "double" and [[ long ]] strings. * Expand Lua examples * Output line comments in Lua output * Implement Lua globals output mode --- cmd/root.go | 5 + cmd/utils.go | 2 + pkg/yqlib/doc/usage/lua.md | 144 ++++++++++++++++ pkg/yqlib/encoder_lua.go | 328 +++++++++++++++++++++++++++++++++++++ pkg/yqlib/lua.go | 19 +++ pkg/yqlib/lua_test.go | 257 +++++++++++++++++++++++++++++ pkg/yqlib/printer.go | 5 +- 7 files changed, 759 insertions(+), 1 deletion(-) create mode 100644 pkg/yqlib/doc/usage/lua.md create mode 100644 pkg/yqlib/encoder_lua.go create mode 100644 pkg/yqlib/lua.go create mode 100644 pkg/yqlib/lua_test.go diff --git a/cmd/root.go b/cmd/root.go index 96f9815d..45cb9de6 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -78,6 +78,11 @@ yq -P sample.json rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", yqlib.ConfiguredXMLPreferences.SkipProcInst, "skip over process instructions (e.g. )") rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", yqlib.ConfiguredXMLPreferences.SkipDirectives, "skip over directives (e.g. )") + rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocPrefix, "lua-prefix", yqlib.ConfiguredLuaPreferences.DocPrefix, "prefix") + rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredLuaPreferences.DocSuffix, "lua-suffix", yqlib.ConfiguredLuaPreferences.DocSuffix, "suffix") + rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.UnquotedKeys, "lua-unquoted", yqlib.ConfiguredLuaPreferences.UnquotedKeys, "output unquoted string keys (e.g. {foo=\"bar\"})") + rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.Globals, "lua-globals", yqlib.ConfiguredLuaPreferences.Globals, "output keys as top-level global variables") + rootCmd.PersistentFlags().BoolVarP(&nullInput, "null-input", "n", false, "Don't read input, simply evaluate the expression given. Useful for creating docs from scratch.") rootCmd.PersistentFlags().BoolVarP(&noDocSeparators, "no-doc", "N", false, "Don't print document separators (---)") diff --git a/cmd/utils.go b/cmd/utils.go index 7d34c723..a50f7048 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -197,6 +197,8 @@ func createEncoder(format yqlib.PrinterOutputFormat) (yqlib.Encoder, error) { return yqlib.NewTomlEncoder(), nil case yqlib.ShellVariablesOutputFormat: return yqlib.NewShellVariablesEncoder(), nil + case yqlib.LuaOutputFormat: + return yqlib.NewLuaEncoder(yqlib.ConfiguredLuaPreferences), nil } return nil, fmt.Errorf("invalid encoder: %v", format) } diff --git a/pkg/yqlib/doc/usage/lua.md b/pkg/yqlib/doc/usage/lua.md new file mode 100644 index 00000000..32141707 --- /dev/null +++ b/pkg/yqlib/doc/usage/lua.md @@ -0,0 +1,144 @@ + +## Basic example +Given a sample.yml file of: +```yaml +--- +country: Australia # this place +cities: +- Sydney +- Melbourne +- Brisbane +- Perth +``` +then +```bash +yq -o=lua '.' sample.yml +``` +will output +```lua +return { + ["country"] = "Australia"; -- this place + ["cities"] = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", + }; +}; +``` + +## Unquoted keys +Uses the `--lua-unquoted` option to produce a nicer-looking output. + +Given a sample.yml file of: +```yaml +--- +country: Australia # this place +cities: +- Sydney +- Melbourne +- Brisbane +- Perth +``` +then +```bash +yq -o=lua '.' sample.yml +``` +will output +```lua +return { + country = "Australia"; -- this place + cities = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", + }; +}; +``` + +## Globals +Uses the `--lua-globals` option to export the values into the global scope. + +Given a sample.yml file of: +```yaml +--- +country: Australia # this place +cities: +- Sydney +- Melbourne +- Brisbane +- Perth +``` +then +```bash +yq -o=lua '.' sample.yml +``` +will output +```lua +country = "Australia"; -- this place +cities = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", +}; +``` + +## Elaborate example +Given a sample.yml file of: +```yaml +--- +hello: world +tables: + like: this + keys: values + ? look: non-string keys + : True +numbers: + - decimal: 12345 + - hex: 0x7fabc123 + - octal: 0o30 + - float: 123.45 + - infinity: .inf + - not: .nan + +``` +then +```bash +yq -o=lua '.' sample.yml +``` +will output +```lua +return { + ["hello"] = "world"; + ["tables"] = { + ["like"] = "this"; + ["keys"] = "values"; + [{ + ["look"] = "non-string keys"; + }] = true; + }; + ["numbers"] = { + { + ["decimal"] = 12345; + }, + { + ["hex"] = 0x7fabc123; + }, + { + ["octal"] = 24; + }, + { + ["float"] = 123.45; + }, + { + ["infinity"] = (1/0); + }, + { + ["not"] = (0/0); + }, + }; +}; +``` + diff --git a/pkg/yqlib/encoder_lua.go b/pkg/yqlib/encoder_lua.go new file mode 100644 index 00000000..898b3855 --- /dev/null +++ b/pkg/yqlib/encoder_lua.go @@ -0,0 +1,328 @@ +package yqlib + +import ( + "fmt" + "io" + "strings" + + yaml "gopkg.in/yaml.v3" +) + +type luaEncoder struct { + docPrefix string + docSuffix string + indent int + indentStr string + unquoted bool + globals bool + escape *strings.Replacer +} + +func (le *luaEncoder) CanHandleAliases() bool { + return false +} + +func NewLuaEncoder(prefs LuaPreferences) Encoder { + escape := strings.NewReplacer( + "\000", "\\000", + "\001", "\\001", + "\002", "\\002", + "\003", "\\003", + "\004", "\\004", + "\005", "\\005", + "\006", "\\006", + "\007", "\\a", + "\010", "\\b", + "\011", "\\t", + "\012", "\\n", + "\013", "\\v", + "\014", "\\f", + "\015", "\\r", + "\016", "\\014", + "\017", "\\015", + "\020", "\\016", + "\021", "\\017", + "\022", "\\018", + "\023", "\\019", + "\024", "\\020", + "\025", "\\021", + "\026", "\\022", + "\027", "\\023", + "\030", "\\024", + "\031", "\\025", + "\032", "\\026", + "\033", "\\027", + "\034", "\\028", + "\035", "\\029", + "\036", "\\030", + "\037", "\\031", + "\"", "\\\"", + "'", "\\'", + "\\", "\\\\", + "\177", "\\127", + ) + unescape := strings.NewReplacer( + "\\'", "'", + "\\\"", "\"", + "\\n", "\n", + "\\r", "\r", + "\\t", "\t", + "\\\\", "\\", + ) + return &luaEncoder{unescape.Replace(prefs.DocPrefix), unescape.Replace(prefs.DocSuffix), 0, "\t", prefs.UnquotedKeys, prefs.Globals, escape} +} + +func (le *luaEncoder) PrintDocumentSeparator(writer io.Writer) error { + return nil +} + +func (le *luaEncoder) PrintLeadingContent(writer io.Writer, content string) error { + return nil +} + +func (le *luaEncoder) encodeString(writer io.Writer, node *yaml.Node) error { + quote := "\"" + switch node.Style { + case yaml.LiteralStyle, yaml.FoldedStyle, yaml.FlowStyle: + for i := 0; i < 10; i++ { + if !strings.Contains(node.Value, "]"+strings.Repeat("=", i)+"]") { + err := writeString(writer, "["+strings.Repeat("=", i)+"[\n") + if err != nil { + return err + } + err = writeString(writer, node.Value) + if err != nil { + return err + } + return writeString(writer, "]"+strings.Repeat("=", i)+"]") + } + } + case yaml.SingleQuotedStyle: + quote = "'" + + // falltrough to regular ol' string + } + return writeString(writer, quote+le.escape.Replace(node.Value)+quote) +} + +func (le *luaEncoder) writeIndent(writer io.Writer) error { + if le.indentStr == "" { + return nil + } + err := writeString(writer, "\n") + if err != nil { + return err + } + return writeString(writer, strings.Repeat(le.indentStr, le.indent)) +} + +func (le *luaEncoder) encodeArray(writer io.Writer, node *yaml.Node) error { + err := writeString(writer, "{") + if err != nil { + return err + } + le.indent++ + for _, child := range node.Content { + err = le.writeIndent(writer) + if err != nil { + return err + } + err := le.Encode(writer, child) + if err != nil { + return err + } + err = writeString(writer, ",") + if err != nil { + return err + } + if child.LineComment != "" { + sansPrefix, _ := strings.CutPrefix(child.LineComment, "#") + err = writeString(writer, " --"+sansPrefix) + if err != nil { + return err + } + } + } + le.indent-- + if len(node.Content) != 0 { + err = le.writeIndent(writer) + if err != nil { + return err + } + } + return writeString(writer, "}") +} + +func needsQuoting(s string) bool { + // known keywords as of Lua 5.4 + switch s { + case "do", "and", "else", "break", + "if", "end", "goto", "false", + "in", "for", "then", "local", + "or", "nil", "true", "until", + "elseif", "function", "not", + "repeat", "return", "while": + return true + } + // [%a_][%w_]* + for i, c := range s { + if i == 0 { + if !((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_') { + return true + } + } else { + if !((c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c == '_') { + return true + } + } + } + return false +} + +func (le *luaEncoder) encodeMap(writer io.Writer, node *yaml.Node, global bool) error { + if !global { + err := writeString(writer, "{") + if err != nil { + return err + } + le.indent++ + } + for i, child := range node.Content { + if (i % 2) == 1 { + // value + err := le.Encode(writer, child) + if err != nil { + return err + } + err = writeString(writer, ";") + if err != nil { + return err + } + } else { + // key + if !global || i > 0 { + err := le.writeIndent(writer) + if err != nil { + return err + } + } + if (le.unquoted || global) && child.Tag == "!!str" && !needsQuoting(child.Value) { + err := writeString(writer, child.Value+" = ") + if err != nil { + return err + } + } else { + if global { + // This only works in Lua 5.2+ + err := writeString(writer, "_ENV") + if err != nil { + return err + } + } + err := writeString(writer, "[") + if err != nil { + return err + } + err = le.encodeAny(writer, child) + if err != nil { + return err + } + err = writeString(writer, "] = ") + if err != nil { + return err + } + } + } + if child.LineComment != "" { + sansPrefix, _ := strings.CutPrefix(child.LineComment, "#") + err := writeString(writer, strings.Repeat(" ", i%2)+"--"+sansPrefix) + if err != nil { + return err + } + if (i % 2) == 0 { + // newline and indent after comments on keys + err = le.writeIndent(writer) + if err != nil { + return err + } + } + } + } + if global { + return writeString(writer, "\n") + } + le.indent-- + if len(node.Content) != 0 { + err := le.writeIndent(writer) + if err != nil { + return err + } + } + return writeString(writer, "}") +} + +func (le *luaEncoder) encodeAny(writer io.Writer, node *yaml.Node) error { + switch node.Kind { + case yaml.SequenceNode: + return le.encodeArray(writer, node) + case yaml.MappingNode: + return le.encodeMap(writer, node, false) + case yaml.ScalarNode: + switch node.Tag { + case "!!str": + return le.encodeString(writer, node) + case "!!null": + // TODO reject invalid use as a table key + return writeString(writer, "nil") + case "!!bool": + // Yaml 1.2 has case variation e.g. True, FALSE etc but Lua only has + // lower case + return writeString(writer, strings.ToLower(node.Value)) + case "!!int": + if strings.HasPrefix(node.Value, "0o") { + var octalValue int + err := node.Decode(&octalValue) + if err != nil { + return err + } + return writeString(writer, fmt.Sprintf("%d", octalValue)) + } + return writeString(writer, strings.ToLower(node.Value)) + case "!!float": + switch strings.ToLower(node.Value) { + case ".inf": + return writeString(writer, "(1/0)") + case "-.inf": + return writeString(writer, "(-1/0)") + case ".nan": + return writeString(writer, "(0/0)") + default: + return writeString(writer, node.Value) + } + default: + return fmt.Errorf("Lua encoder NYI -- %s", node.ShortTag()) + } + case yaml.DocumentNode: + if le.globals { + if node.Content[0].Kind != yaml.MappingNode { + return fmt.Errorf("--lua-global requires a top level MappingNode") + } + return le.encodeMap(writer, node.Content[0], true) + } + err := writeString(writer, le.docPrefix) + if err != nil { + return err + } + err = le.encodeAny(writer, node.Content[0]) + if err != nil { + return err + } + return writeString(writer, le.docSuffix) + default: + return fmt.Errorf("Lua encoder NYI -- %s", node.ShortTag()) + } +} + +func (le *luaEncoder) Encode(writer io.Writer, node *yaml.Node) error { + return le.encodeAny(writer, node) +} diff --git a/pkg/yqlib/lua.go b/pkg/yqlib/lua.go new file mode 100644 index 00000000..ba63e634 --- /dev/null +++ b/pkg/yqlib/lua.go @@ -0,0 +1,19 @@ +package yqlib + +type LuaPreferences struct { + DocPrefix string + DocSuffix string + UnquotedKeys bool + Globals bool +} + +func NewDefaultLuaPreferences() LuaPreferences { + return LuaPreferences{ + DocPrefix: "return ", + DocSuffix: ";\n", + UnquotedKeys: false, + Globals: false, + } +} + +var ConfiguredLuaPreferences = NewDefaultLuaPreferences() diff --git a/pkg/yqlib/lua_test.go b/pkg/yqlib/lua_test.go new file mode 100644 index 00000000..3f8133d6 --- /dev/null +++ b/pkg/yqlib/lua_test.go @@ -0,0 +1,257 @@ +package yqlib + +import ( + "bufio" + "fmt" + "testing" + + "github.com/mikefarah/yq/v4/test" +) + +var luaScenarios = []formatScenario{ + { + description: "Basic example", + scenarioType: "encode", + input: `--- +country: Australia # this place +cities: +- Sydney +- Melbourne +- Brisbane +- Perth`, + expected: `return { + ["country"] = "Australia"; -- this place + ["cities"] = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", + }; +}; +`, + }, + { + description: "Unquoted keys", + subdescription: "Uses the `--lua-unquoted` option to produce a nicer-looking output.", + scenarioType: "unquoted-encode", + input: `--- +country: Australia # this place +cities: +- Sydney +- Melbourne +- Brisbane +- Perth`, + expected: `return { + country = "Australia"; -- this place + cities = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", + }; +}; +`, + }, + { + description: "Globals", + subdescription: "Uses the `--lua-globals` option to export the values into the global scope.", + scenarioType: "globals-encode", + input: `--- +country: Australia # this place +cities: +- Sydney +- Melbourne +- Brisbane +- Perth`, + expected: `country = "Australia"; -- this place +cities = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", +}; +`, + }, + { + description: "Elaborate example", + input: `--- +hello: world +tables: + like: this + keys: values + ? look: non-string keys + : True +numbers: + - decimal: 12345 + - hex: 0x7fabc123 + - octal: 0o30 + - float: 123.45 + - infinity: .inf + - not: .nan +`, + expected: `return { + ["hello"] = "world"; + ["tables"] = { + ["like"] = "this"; + ["keys"] = "values"; + [{ + ["look"] = "non-string keys"; + }] = true; + }; + ["numbers"] = { + { + ["decimal"] = 12345; + }, + { + ["hex"] = 0x7fabc123; + }, + { + ["octal"] = 24; + }, + { + ["float"] = 123.45; + }, + { + ["infinity"] = (1/0); + }, + { + ["not"] = (0/0); + }, + }; +}; +`, + scenarioType: "encode", + }, + { + skipDoc: true, + description: "Sequence", + input: "- a\n- b\n- c\n", + expected: "return {\n\t\"a\",\n\t\"b\",\n\t\"c\",\n};\n", + scenarioType: "encode", + }, + { + skipDoc: true, + description: "Mapping", + input: "a: b\nc:\n d: e\nf: 0\n", + expected: "return {\n\t[\"a\"] = \"b\";\n\t[\"c\"] = {\n\t\t[\"d\"] = \"e\";\n\t};\n\t[\"f\"] = 0;\n};\n", + scenarioType: "encode", + }, + { + skipDoc: true, + description: "Scalar str", + input: "str: |\n foo\n bar\nanother: 'single'\nand: \"double\"", + expected: "return {\n\t[\"str\"] = [[\nfoo\nbar\n]];\n\t[\"another\"] = 'single';\n\t[\"and\"] = \"double\";\n};\n", + scenarioType: "encode", + }, + { + skipDoc: true, + description: "Scalar null", + input: "x: null\n", + expected: "return {\n\t[\"x\"] = nil;\n};\n", + scenarioType: "encode", + }, + { + skipDoc: true, + description: "Scalar int", + input: "- 1\n- 2\n- 0x10\n- 0o30\n- -999\n", + expected: "return {\n\t1,\n\t2,\n\t0x10,\n\t24,\n\t-999,\n};\n", + scenarioType: "encode", + }, + { + skipDoc: true, + description: "Scalar float", + input: "- 1.0\n- 3.14\n- 1e100\n- .Inf\n- .NAN\n", + expected: "return {\n\t1.0,\n\t3.14,\n\t1e100,\n\t(1/0),\n\t(0/0),\n};\n", + scenarioType: "encode", + }, +} + +func testLuaScenario(t *testing.T, s formatScenario) { + switch s.scenarioType { + case "encode": + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(ConfiguredLuaPreferences)), s.description) + case "unquoted-encode": + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(LuaPreferences{ + DocPrefix: "return ", + DocSuffix: ";\n", + UnquotedKeys: true, + Globals: false, + })), s.description) + case "globals-encode": + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(LuaPreferences{ + DocPrefix: "return ", + DocSuffix: ";\n", + UnquotedKeys: false, + Globals: true, + })), s.description) + default: + panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) + } +} + +func documentLuaScenario(t *testing.T, w *bufio.Writer, i interface{}) { + s := i.(formatScenario) + + if s.skipDoc { + return + } + switch s.scenarioType { + case "encode", "unquoted-encode", "globals-encode": + documentLuaEncodeScenario(w, s) + default: + panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) + } +} + +func documentLuaEncodeScenario(w *bufio.Writer, s formatScenario) { + writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) + + if s.subdescription != "" { + writeOrPanic(w, s.subdescription) + writeOrPanic(w, "\n\n") + } + + prefs := ConfiguredLuaPreferences + switch s.scenarioType { + case "unquoted-encode": + prefs = LuaPreferences{ + DocPrefix: "return ", + DocSuffix: ";\n", + UnquotedKeys: true, + Globals: false, + } + case "globals-encode": + prefs = LuaPreferences{ + DocPrefix: "return ", + DocSuffix: ";\n", + UnquotedKeys: false, + Globals: true, + } + } + writeOrPanic(w, "Given a sample.yml file of:\n") + writeOrPanic(w, fmt.Sprintf("```yaml\n%v\n```\n", s.input)) + + writeOrPanic(w, "then\n") + switch s.scenarioType { + case "unquoted-encode": + writeOrPanic(w, "```bash\nyq -o=lua --lua-unquoted '.' sample.yml\n```\n") + case "globals-encode": + writeOrPanic(w, "```bash\nyq -o=lua --lua-globals '.' sample.yml\n```\n") + default: + writeOrPanic(w, "```bash\nyq -o=lua '.' sample.yml\n```\n") + } + writeOrPanic(w, "will output\n") + + writeOrPanic(w, fmt.Sprintf("```lua\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(prefs)))) +} + +func TestLuaScenarios(t *testing.T) { + for _, tt := range luaScenarios { + testLuaScenario(t, tt) + } + genericScenarios := make([]interface{}, len(luaScenarios)) + for i, s := range luaScenarios { + genericScenarios[i] = s + } + documentScenarios(t, "usage", "lua", genericScenarios, documentLuaScenario) +} diff --git a/pkg/yqlib/printer.go b/pkg/yqlib/printer.go index 9cd65753..c430fcfe 100644 --- a/pkg/yqlib/printer.go +++ b/pkg/yqlib/printer.go @@ -33,6 +33,7 @@ const ( ShOutputFormat TomlOutputFormat ShellVariablesOutputFormat + LuaOutputFormat ) func OutputFormatFromString(format string) (PrinterOutputFormat, error) { @@ -53,8 +54,10 @@ func OutputFormatFromString(format string) (PrinterOutputFormat, error) { return TomlOutputFormat, nil case "shell", "s", "sh": return ShellVariablesOutputFormat, nil + case "lua", "l": + return LuaOutputFormat, nil default: - return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml|shell]", format) + return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml|shell|lua]", format) } } From 804b204f3c0832e12847922622978c455f576c89 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Aug 2023 12:57:02 +1000 Subject: [PATCH 09/43] Bump golang from 1.20.7 to 1.21.0 (#1753) Bumps golang from 1.20.7 to 1.21.0. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- Dockerfile.dev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e091cc50..8ba25356 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20.7 as builder +FROM golang:1.21.0 as builder WORKDIR /go/src/mikefarah/yq diff --git a/Dockerfile.dev b/Dockerfile.dev index e1b15e72..55d70477 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM golang:1.20.7 +FROM golang:1.21.0 COPY scripts/devtools.sh /opt/devtools.sh From 1c4f92ecd18af09a4631af34453db1f746f73350 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Aug 2023 12:57:12 +1000 Subject: [PATCH 10/43] Bump golang.org/x/text from 0.11.0 to 0.12.0 (#1752) Bumps [golang.org/x/text](https://github.com/golang/text) from 0.11.0 to 0.12.0. - [Release notes](https://github.com/golang/text/releases) - [Commits](https://github.com/golang/text/compare/v0.11.0...v0.12.0) --- updated-dependencies: - dependency-name: golang.org/x/text dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 47d764ba..d97b8238 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 golang.org/x/net v0.13.0 - golang.org/x/text v0.11.0 + golang.org/x/text v0.12.0 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 6e5f4ff7..95dcbef4 100644 --- a/go.sum +++ b/go.sum @@ -55,8 +55,8 @@ golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= From ba1d5c1a84b8bd59b140a5057089898a16396519 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 11 Aug 2023 13:50:00 +1000 Subject: [PATCH 11/43] Bump golang.org/x/net from 0.13.0 to 0.14.0 (#1751) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.13.0 to 0.14.0. - [Commits](https://github.com/golang/net/compare/v0.13.0...v0.14.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index d97b8238..44dc7838 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 - golang.org/x/net v0.13.0 + golang.org/x/net v0.14.0 golang.org/x/text v0.12.0 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 gopkg.in/yaml.v3 v3.0.1 @@ -25,7 +25,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect - golang.org/x/sys v0.10.0 // indirect + golang.org/x/sys v0.11.0 // indirect golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect ) diff --git a/go.sum b/go.sum index 95dcbef4..2514ee94 100644 --- a/go.sum +++ b/go.sum @@ -50,11 +50,11 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY= -golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= From aaf805d8938417f7060dfb355532df6a40c4f0a9 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 11 Aug 2023 13:51:04 +1000 Subject: [PATCH 12/43] Added checksum in bsd format --- scripts/xcompile.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/xcompile.sh b/scripts/xcompile.sh index 2b259167..08e8d662 100755 --- a/scripts/xcompile.sh +++ b/scripts/xcompile.sh @@ -19,6 +19,8 @@ rm yq.1 rhash -r -a . -o checksums +rhash -r -a --bsd . -o checksums-bsd + rhash --list-hashes > checksums_hashes_order cp ../scripts/extract-checksum.sh . From 44c7f3aec342a88b969431f6d49cdad69d594872 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 11 Aug 2023 13:52:50 +1000 Subject: [PATCH 13/43] Preparing release notes --- pkg/yqlib/doc/usage/lua.md | 4 ++-- release_notes.txt | 5 +++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pkg/yqlib/doc/usage/lua.md b/pkg/yqlib/doc/usage/lua.md index 32141707..61458be2 100644 --- a/pkg/yqlib/doc/usage/lua.md +++ b/pkg/yqlib/doc/usage/lua.md @@ -42,7 +42,7 @@ cities: ``` then ```bash -yq -o=lua '.' sample.yml +yq -o=lua --lua-unquoted '.' sample.yml ``` will output ```lua @@ -72,7 +72,7 @@ cities: ``` then ```bash -yq -o=lua '.' sample.yml +yq -o=lua --lua-globals '.' sample.yml ``` will output ```lua diff --git a/release_notes.txt b/release_notes.txt index 4f8dfac5..46a6e692 100644 --- a/release_notes.txt +++ b/release_notes.txt @@ -1,3 +1,8 @@ +4.35.1: + - Added Lua output support (Thanks @Zash)! + - Added BSD checksum format (Thanks @viq)! + - Bumped dependencies + 4.34.2: - Bumped dependencies From 6609ed76ecb69f9d8254345292d90ea72f641715 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 11 Aug 2023 13:53:14 +1000 Subject: [PATCH 14/43] Bumping version --- cmd/version.go | 2 +- snap/snapcraft.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/version.go b/cmd/version.go index c0898fec..6a75038e 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -11,7 +11,7 @@ var ( GitDescribe string // Version is main version number that is being run at the moment. - Version = "v4.34.2" + Version = "v4.35.1" // VersionPrerelease is a pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index 2ef9c1e7..b6a55343 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,5 +1,5 @@ name: yq -version: 'v4.34.2' +version: 'v4.35.1' summary: A lightweight and portable command-line YAML processor description: | The aim of the project is to be the jq or sed of yaml files. From 596dda8c53e945ba3b9125e1f84cc7a4e1e200af Mon Sep 17 00:00:00 2001 From: Nikolas Grottendieck Date: Mon, 28 Aug 2023 09:27:26 +0200 Subject: [PATCH 15/43] improve community supported installation options for Windows (#1761) - Add missing package manager (scoop) - Improve winget installationn example & add link to reference documentation - Add link to missing reference documentation for chocolatey --- README.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 637bd2ad..1486fe6f 100644 --- a/README.md +++ b/README.md @@ -257,6 +257,9 @@ pacman -S go-yq ``` ### Windows: + +Using [Chocolatey](https://chocolatey.org) + [![Chocolatey](https://img.shields.io/chocolatey/v/yq.svg)](https://chocolatey.org/packages/yq) [![Chocolatey](https://img.shields.io/chocolatey/dt/yq.svg)](https://chocolatey.org/packages/yq) ``` @@ -264,12 +267,15 @@ choco install yq ``` Supported by @chillum (https://chocolatey.org/packages/yq) -and +Using [scoop](https://scoop.sh/) +``` +scoop install main/yq +``` -### Winget -winget install yq - -https://winget.run/pkg/MikeFarah/yq +Using [winget](https://learn.microsoft.com/en-us/windows/package-manager/) +``` +winget install --id MikeFarah.yq +``` ### Mac: Using [MacPorts](https://www.macports.org/) From d18a5e3adcb3304dba5902a1611c8c56709c1514 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Mon, 28 Aug 2023 17:39:46 +1000 Subject: [PATCH 16/43] Fixed new linting issue CWE-118 --- pkg/yqlib/decoder_json.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/yqlib/decoder_json.go b/pkg/yqlib/decoder_json.go index cd8cf5cc..c6b88eca 100644 --- a/pkg/yqlib/decoder_json.go +++ b/pkg/yqlib/decoder_json.go @@ -66,8 +66,8 @@ func (dec *jsonDecoder) convertToYamlNode(data *orderedMap) (*yaml.Node, error) } var yamlMap = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"} - for _, keyValuePair := range data.kv { - yamlValue, err := dec.convertToYamlNode(&keyValuePair.V) + for i, keyValuePair := range data.kv { + yamlValue, err := dec.convertToYamlNode(&data.kv[i].V) if err != nil { return nil, err } From 1e84d7ef1f7ca84276230e47baeeb91a5b858d9d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Aug 2023 09:27:31 +1000 Subject: [PATCH 17/43] Bump github.com/jinzhu/copier from 0.3.5 to 0.4.0 (#1763) Bumps [github.com/jinzhu/copier](https://github.com/jinzhu/copier) from 0.3.5 to 0.4.0. - [Commits](https://github.com/jinzhu/copier/compare/v0.3.5...v0.4.0) --- updated-dependencies: - dependency-name: github.com/jinzhu/copier dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 44dc7838..81dd79c6 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/fatih/color v1.15.0 github.com/goccy/go-json v0.10.2 github.com/goccy/go-yaml v1.11.0 - github.com/jinzhu/copier v0.3.5 + github.com/jinzhu/copier v0.4.0 github.com/magiconair/properties v1.8.7 github.com/pelletier/go-toml/v2 v2.0.9 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e diff --git a/go.sum b/go.sum index 2514ee94..ecaa6d37 100644 --- a/go.sum +++ b/go.sum @@ -22,8 +22,8 @@ github.com/goccy/go-yaml v1.11.0/go.mod h1:H+mJrWtjPTJAHvRbV09MCK9xYwODM+wRTVFFT github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= -github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= +github.com/jinzhu/copier v0.4.0 h1:w3ciUoD19shMCRargcpm0cm91ytaBhDvuRpz1ODO/U8= +github.com/jinzhu/copier v0.4.0/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= From 71f95ab87bba2028d0c52fe5bdc63da01187554b Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Wed, 30 Aug 2023 09:40:43 +1000 Subject: [PATCH 18/43] Fixed .a!= parsing issue #1711 --- pkg/yqlib/lexer_participle.go | 2 +- pkg/yqlib/lexer_participle_test.go | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go index 18025f44..3a3b7c10 100644 --- a/pkg/yqlib/lexer_participle.go +++ b/pkg/yqlib/lexer_participle.go @@ -201,7 +201,7 @@ var participleYqRules = []*participleYqRule{ {`whitespace`, `[ \t\n]+`, nil, 0}, {"WrappedPathElement", `\."[^ "]+"\??`, pathToken(true), 0}, - {"PathElement", `\.[^ ;\}\{\:\[\],\|\.\[\(\)=\n]+\??`, pathToken(false), 0}, + {"PathElement", `\.[^ ;\}\{\:\[\],\|\.\[\(\)=\n!]+\??`, pathToken(false), 0}, {"Pipe", `\|`, opToken(pipeOpType), 0}, {"Self", `\.`, opToken(selfReferenceOpType), 0}, diff --git a/pkg/yqlib/lexer_participle_test.go b/pkg/yqlib/lexer_participle_test.go index 2bce0ae2..1df7fec8 100644 --- a/pkg/yqlib/lexer_participle_test.go +++ b/pkg/yqlib/lexer_participle_test.go @@ -14,6 +14,29 @@ type participleLexerScenario struct { } var participleLexerScenarios = []participleLexerScenario{ + { + expression: ".a!=", + tokens: []*token{ + { + TokenType: operationToken, + Operation: &Operation{ + OperationType: traversePathOpType, + Value: "a", + StringValue: "a", + Preferences: traversePreferences{}, + }, + CheckForPostTraverse: true, + }, + { + TokenType: operationToken, + Operation: &Operation{ + OperationType: notEqualsOpType, + Value: "NOT_EQUALS", + StringValue: "!=", + }, + }, + }, + }, { expression: ".[:3]", tokens: []*token{ From 55c0d485bae7f3cf66468c0d3efae5303a179128 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Wed, 30 Aug 2023 09:46:45 +1000 Subject: [PATCH 19/43] Fixed string, null concact consistency #1712 --- pkg/yqlib/operator_add.go | 6 +++++- pkg/yqlib/operator_add_test.go | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/pkg/yqlib/operator_add.go b/pkg/yqlib/operator_add.go index f0281bdf..5b95c34b 100644 --- a/pkg/yqlib/operator_add.go +++ b/pkg/yqlib/operator_add.go @@ -107,7 +107,11 @@ func addScalars(context Context, target *CandidateNode, lhs *yaml.Node, rhs *yam } else if lhsTag == "!!str" { target.Node.Tag = lhs.Tag - target.Node.Value = lhs.Value + rhs.Value + if rhsTag == "!!null" { + target.Node.Value = lhs.Value + } else { + target.Node.Value = lhs.Value + rhs.Value + } } else if rhsTag == "!!str" { target.Node.Tag = rhs.Tag target.Node.Value = lhs.Value + rhs.Value diff --git a/pkg/yqlib/operator_add_test.go b/pkg/yqlib/operator_add_test.go index eeabffde..918d0d39 100644 --- a/pkg/yqlib/operator_add_test.go +++ b/pkg/yqlib/operator_add_test.go @@ -225,6 +225,20 @@ var addOperatorScenarios = []expressionScenario{ "D0, P[], (!cool)::3cat\n", }, }, + { + skipDoc: true, + expression: `null + "cat"`, + expected: []string{ + "D0, P[], (!!str)::cat\n", + }, + }, + { + skipDoc: true, + expression: `"cat" + null`, + expected: []string{ + "D0, P[], (!!str)::cat\n", + }, + }, { description: "Number addition - float", subdescription: "If the lhs or rhs are floats then the expression will be calculated with floats.", From e5564c18fe30af124a52ef642f2e7406d339b018 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 1 Sep 2023 11:52:58 +1000 Subject: [PATCH 20/43] Another xml example --- pkg/yqlib/doc/usage/xml.md | 44 +++++++++++++++++++++++++++----------- pkg/yqlib/xml_test.go | 17 ++++++++++----- 2 files changed, 43 insertions(+), 18 deletions(-) diff --git a/pkg/yqlib/doc/usage/xml.md b/pkg/yqlib/doc/usage/xml.md index 3d85a4ff..89bee794 100644 --- a/pkg/yqlib/doc/usage/xml.md +++ b/pkg/yqlib/doc/usage/xml.md @@ -53,7 +53,7 @@ Given a sample.xml file of: ``` then ```bash -yq -p=xml '.' sample.xml +yq -oy '.' sample.xml ``` will output ```yaml @@ -78,7 +78,7 @@ Given a sample.xml file of: ``` then ```bash -yq -p=xml ' (.. | select(tag == "!!str")) |= from_yaml' sample.xml +yq -oy ' (.. | select(tag == "!!str")) |= from_yaml' sample.xml ``` will output ```yaml @@ -100,7 +100,7 @@ Given a sample.xml file of: ``` then ```bash -yq -p=xml '.' sample.xml +yq -oy '.' sample.xml ``` will output ```yaml @@ -110,6 +110,24 @@ animal: - goat ``` +## Parse xml: force as an array +In XML, if your array has a single item, then yq doesn't know its an array. This is how you can consistently force it to be an array. This handles the 3 scenarios of having nothing in the array, having a single item and having multiple. + +Given a sample.xml file of: +```xml +cat +``` +then +```bash +yq -oy '.zoo.animal |= ([] + .)' sample.xml +``` +will output +```yaml +zoo: + animal: + - cat +``` + ## Parse xml: attributes Attributes are converted to fields, with the default attribute prefix '+'. Use '--xml-attribute-prefix` to set your own. @@ -122,7 +140,7 @@ Given a sample.xml file of: ``` then ```bash -yq -p=xml '.' sample.xml +yq -oy '.' sample.xml ``` will output ```yaml @@ -142,7 +160,7 @@ Given a sample.xml file of: ``` then ```bash -yq -p=xml '.' sample.xml +yq -oy '.' sample.xml ``` will output ```yaml @@ -161,7 +179,7 @@ Given a sample.xml file of: ``` then ```bash -yq -p=xml '.' sample.xml +yq -oy '.' sample.xml ``` will output ```yaml @@ -190,7 +208,7 @@ Given a sample.xml file of: ``` then ```bash -yq -p=xml -o=xml '.' sample.xml +yq '.' sample.xml ``` will output ```xml @@ -221,7 +239,7 @@ Given a sample.xml file of: ``` then ```bash -yq -p=xml -o=xml --xml-skip-directives '.' sample.xml +yq --xml-skip-directives '.' sample.xml ``` will output ```xml @@ -257,7 +275,7 @@ for x --> ``` then ```bash -yq -p=xml '.' sample.xml +yq -oy '.' sample.xml ``` will output ```yaml @@ -289,7 +307,7 @@ Given a sample.xml file of: ``` then ```bash -yq -p=xml -o=xml --xml-keep-namespace=false '.' sample.xml +yq --xml-keep-namespace=false '.' sample.xml ``` will output ```xml @@ -314,7 +332,7 @@ Given a sample.xml file of: ``` then ```bash -yq -p=xml -o=xml --xml-raw-token=false '.' sample.xml +yq --xml-raw-token=false '.' sample.xml ``` will output ```xml @@ -489,7 +507,7 @@ for x --> ``` then ```bash -yq -p=xml -o=xml '.' sample.xml +yq '.' sample.xml ``` will output ```xml @@ -522,7 +540,7 @@ Given a sample.xml file of: ``` then ```bash -yq -p=xml -o=xml '.' sample.xml +yq '.' sample.xml ``` will output ```xml diff --git a/pkg/yqlib/xml_test.go b/pkg/yqlib/xml_test.go index a86e25ab..e90e3654 100644 --- a/pkg/yqlib/xml_test.go +++ b/pkg/yqlib/xml_test.go @@ -356,6 +356,13 @@ var xmlScenarios = []formatScenario{ input: "\ncat\ngoat", expected: "+p_xml: version=\"1.0\" encoding=\"UTF-8\"\nanimal:\n - cat\n - goat\n", }, + { + description: "Parse xml: force as an array", + subdescription: "In XML, if your array has a single item, then yq doesn't know its an array. This is how you can consistently force it to be an array. This handles the 3 scenarios of having nothing in the array, having a single item and having multiple.", + input: "cat", + expression: ".zoo.animal |= ([] + .)", + expected: "zoo:\n animal:\n - cat\n", + }, { description: "Parse xml: attributes", subdescription: "Attributes are converted to fields, with the default attribute prefix '+'. Use '--xml-attribute-prefix` to set your own.", @@ -687,7 +694,7 @@ func documentXMLDecodeScenario(w *bufio.Writer, s formatScenario) { if expression == "" { expression = "." } - writeOrPanic(w, fmt.Sprintf("```bash\nyq -p=xml '%v' sample.xml\n```\n", expression)) + writeOrPanic(w, fmt.Sprintf("```bash\nyq -oy '%v' sample.xml\n```\n", expression)) writeOrPanic(w, "will output\n") writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences)))) @@ -705,7 +712,7 @@ func documentXMLDecodeKeepNsScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") - writeOrPanic(w, "```bash\nyq -p=xml -o=xml --xml-keep-namespace=false '.' sample.xml\n```\n") + writeOrPanic(w, "```bash\nyq --xml-keep-namespace=false '.' sample.xml\n```\n") writeOrPanic(w, "will output\n") prefs := NewDefaultXmlPreferences() prefs.KeepNamespace = false @@ -729,7 +736,7 @@ func documentXMLDecodeKeepNsRawTokenScenario(w *bufio.Writer, s formatScenario) writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") - writeOrPanic(w, "```bash\nyq -p=xml -o=xml --xml-raw-token=false '.' sample.xml\n```\n") + writeOrPanic(w, "```bash\nyq --xml-raw-token=false '.' sample.xml\n```\n") writeOrPanic(w, "will output\n") prefs := NewDefaultXmlPreferences() @@ -774,7 +781,7 @@ func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") - writeOrPanic(w, "```bash\nyq -p=xml -o=xml '.' sample.xml\n```\n") + writeOrPanic(w, "```bash\nyq '.' sample.xml\n```\n") writeOrPanic(w, "will output\n") writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)))) @@ -792,7 +799,7 @@ func documentXMLSkipDirectrivesScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("```xml\n%v\n```\n", s.input)) writeOrPanic(w, "then\n") - writeOrPanic(w, "```bash\nyq -p=xml -o=xml --xml-skip-directives '.' sample.xml\n```\n") + writeOrPanic(w, "```bash\nyq --xml-skip-directives '.' sample.xml\n```\n") writeOrPanic(w, "will output\n") prefs := NewDefaultXmlPreferences() prefs.SkipDirectives = true From cda69bff5ee2aa62aef16ebe81416214eb00ce46 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 1 Sep 2023 12:17:54 +1000 Subject: [PATCH 21/43] Adding recipes --- README.md | 5 ++++ pkg/yqlib/doc/usage/recipes.md | 52 ++++++++++++++++++++++++++++++++++ pkg/yqlib/operators_test.go | 9 ++++++ pkg/yqlib/recipes_test.go | 48 +++++++++++++++++++++++++++++++ 4 files changed, 114 insertions(+) create mode 100644 pkg/yqlib/doc/usage/recipes.md create mode 100644 pkg/yqlib/recipes_test.go diff --git a/README.md b/README.md index 1486fe6f..5c36aa73 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,11 @@ yq -i ' ' file.yaml ``` +Find and update an item in an array: +```bash +yq '(.[] | select(.name == "foo") | .address) = "12 cat st"' +``` + Convert JSON to YAML ```bash yq -Poy sample.json diff --git a/pkg/yqlib/doc/usage/recipes.md b/pkg/yqlib/doc/usage/recipes.md new file mode 100644 index 00000000..369689b8 --- /dev/null +++ b/pkg/yqlib/doc/usage/recipes.md @@ -0,0 +1,52 @@ + +## Find items in an array +Given a sample.yml file of: +```yaml +- name: Foo + numBuckets: 0 +- name: Bar + numBuckets: 0 +``` +then +```bash +yq '.[] | select(.name == "Foo")' sample.yml +``` +will output +```yaml +name: Foo +numBuckets: 0 +``` + +### Explanation: +- `.[]` splats the array, and puts all the items in the context. +- These items are then piped (`|`) into `select(.name == "Foo")` which will select all the nodes that have a name property set to 'Foo'. +- See the [select](https://mikefarah.gitbook.io/yq/operators/select) operator for more information. + +## Find and update items in an array +Given a sample.yml file of: +```yaml +- name: Foo + numBuckets: 0 +- name: Bar + numBuckets: 0 +``` +then +```bash +yq '(.[] | select(.name == "Foo") | .numBuckets) |= . + 1' sample.yml +``` +will output +```yaml +- name: Foo + numBuckets: 1 +- name: Bar + numBuckets: 0 +``` + +### Explanation: +- Following from the example above`.[]` splats the array, selects filters the items. +- We then pipe (`|`) that into `.numBuckets`, which will select that field from all the matching items +- Splat, select and the field are all in brackets, that whole expression is passed to the `|=` operator as the left hand side expression, with `. + 1` as the right hand side expression. +- `|=` is the operator that updates fields relative to their own value, which is referenced as dot (`.`). +- The expression `. + 1` increments the numBuckets counter. +- See the [assign](https://mikefarah.gitbook.io/yq/operators/assign-update) and [add](https://mikefarah.gitbook.io/yq/operators/add) operators for more information. + diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go index c040cbed..ff6e6f62 100644 --- a/pkg/yqlib/operators_test.go +++ b/pkg/yqlib/operators_test.go @@ -20,6 +20,7 @@ import ( type expressionScenario struct { description string subdescription string + explanation []string environmentVariables map[string]string document string document2 string @@ -256,6 +257,14 @@ func documentOperatorScenario(t *testing.T, w *bufio.Writer, i interface{}) { writeOrPanic(w, "will output\n") documentOutput(t, w, s, formattedDoc, formattedDoc2) + + if len(s.explanation) > 0 { + writeOrPanic(w, "### Explanation:\n") + for _, text := range s.explanation { + writeOrPanic(w, fmt.Sprintf("- %v\n", text)) + } + writeOrPanic(w, "\n") + } } func documentInput(w *bufio.Writer, s expressionScenario) (string, string) { diff --git a/pkg/yqlib/recipes_test.go b/pkg/yqlib/recipes_test.go new file mode 100644 index 00000000..6fab7d6b --- /dev/null +++ b/pkg/yqlib/recipes_test.go @@ -0,0 +1,48 @@ +package yqlib + +import ( + "testing" +) + +var recipes = []expressionScenario{ + { + description: "Find items in an array", + explanation: []string{ + "`.[]` splats the array, and puts all the items in the context.", + "These items are then piped (`|`) into `select(.name == \"Foo\")` which will select all the nodes that have a name property set to 'Foo'.", + "See the [select](https://mikefarah.gitbook.io/yq/operators/select) operator for more information.", + }, + document: `[{name: Foo, numBuckets: 0}, {name: Bar, numBuckets: 0}]`, + expression: `.[] | select(.name == "Foo")`, + expected: []string{ + "D0, P[0], (!!map)::{name: Foo, numBuckets: 0}\n", + }, + }, + { + description: "Find and update items in an array", + document: `[{name: Foo, numBuckets: 0}, {name: Bar, numBuckets: 0}]`, + expression: `(.[] | select(.name == "Foo") | .numBuckets) |= . + 1`, + explanation: []string{ + "Following from the example above`.[]` splats the array, selects filters the items.", + "We then pipe (`|`) that into `.numBuckets`, which will select that field from all the matching items", + "Splat, select and the field are all in brackets, that whole expression is passed to the `|=` operator as the left hand side expression, with `. + 1` as the right hand side expression.", + "`|=` is the operator that updates fields relative to their own value, which is referenced as dot (`.`).", + "The expression `. + 1` increments the numBuckets counter.", + "See the [assign](https://mikefarah.gitbook.io/yq/operators/assign-update) and [add](https://mikefarah.gitbook.io/yq/operators/add) operators for more information.", + }, + expected: []string{ + "D0, P[], (doc)::[{name: Foo, numBuckets: 1}, {name: Bar, numBuckets: 0}]\n", + }, + }, +} + +func TestRecipes(t *testing.T) { + for _, tt := range recipes { + testScenario(t, &tt) + } + genericScenarios := make([]interface{}, len(recipes)) + for i, s := range recipes { + genericScenarios[i] = s + } + documentScenarios(t, "usage", "recipes", genericScenarios, documentOperatorScenario) +} From c66480448d7ffb03c8d833bc0602fdd6eb0a2c45 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 1 Sep 2023 14:40:33 +1000 Subject: [PATCH 22/43] Added recipes documentation --- pkg/yqlib/doc/usage/headers/recipes.md | 5 ++ pkg/yqlib/doc/usage/recipes.md | 103 +++++++++++++++++++++++++ pkg/yqlib/recipes_test.go | 56 +++++++++++++- 3 files changed, 160 insertions(+), 4 deletions(-) create mode 100644 pkg/yqlib/doc/usage/headers/recipes.md diff --git a/pkg/yqlib/doc/usage/headers/recipes.md b/pkg/yqlib/doc/usage/headers/recipes.md new file mode 100644 index 00000000..79999606 --- /dev/null +++ b/pkg/yqlib/doc/usage/headers/recipes.md @@ -0,0 +1,5 @@ +# Recipes + +These examples are intended to show how you can use multiple operators together so you get an idea of how you can perform complex data manipulation. + +Please see the details [operator docs](https://mikefarah.gitbook.io/yq/operators) for details on each individual operator. diff --git a/pkg/yqlib/doc/usage/recipes.md b/pkg/yqlib/doc/usage/recipes.md index 369689b8..2a0cc17e 100644 --- a/pkg/yqlib/doc/usage/recipes.md +++ b/pkg/yqlib/doc/usage/recipes.md @@ -1,5 +1,12 @@ +# Recipes + +These examples are intended to show how you can use multiple operators together so you get an idea of how you can perform complex data manipulation. + +Please see the details [operator docs](https://mikefarah.gitbook.io/yq/operators) for details on each individual operator. ## Find items in an array +We have an array and we want to find the elements with a particular name. + Given a sample.yml file of: ```yaml - name: Foo @@ -23,6 +30,8 @@ numBuckets: 0 - See the [select](https://mikefarah.gitbook.io/yq/operators/select) operator for more information. ## Find and update items in an array +We have an array and we want to _update_ the elements with a particular name. + Given a sample.yml file of: ```yaml - name: Foo @@ -50,3 +59,97 @@ will output - The expression `. + 1` increments the numBuckets counter. - See the [assign](https://mikefarah.gitbook.io/yq/operators/assign-update) and [add](https://mikefarah.gitbook.io/yq/operators/add) operators for more information. +## Multiple or complex updates to items in an array +We have an array and we want to _update_ the elements with a particular name in reference to its type. + +Given a sample.yml file of: +```yaml +myArray: + - name: Foo + type: cat + - name: Bar + type: dog +``` +then +```bash +yq 'with(.myArray[]; .name = .name + " - " + .type)' sample.yml +``` +will output +```yaml +myArray: + - name: Foo - cat + type: cat + - name: Bar - dog + type: dog +``` + +### Explanation: +- The with operator will effectively loop through each given item in the first given expression, and run the second expression against it. +- `.myArray[]` splats the array in `myArray`. So `with` will run against each item in that array +- `.name = .name + " - " + .type` this expression is run against every item, updating the name to be a concatenation of the original name as well as the type. +- See the [with](https://mikefarah.gitbook.io/yq/operators/with) operator for more information and examples. + +## Sort an array by a field +Given a sample.yml file of: +```yaml +myArray: + - name: Foo + numBuckets: 1 + - name: Bar + numBuckets: 0 +``` +then +```bash +yq '.myArray |= sort_by(.numBuckets)' sample.yml +``` +will output +```yaml +myArray: + - name: Bar + numBuckets: 0 + - name: Foo + numBuckets: 1 +``` + +### Explanation: +- We want to resort `.myArray`. +- `sort_by` works by piping an array into it, and it pipes out a sorted array. +- So, we use `|=` to update `.myArray`. This is the same as doing `.myArray = (.myArray | sort_by(.numBuckets))` + +## Filter, flatten, sort and unique +Lets + +Given a sample.yml file of: +```yaml +- type: foo + names: + - Fred + - Catherine +- type: bar + names: + - Zelda +- type: foo + names: Fred +- type: foo + names: Ava +``` +then +```bash +yq '[.[] | select(.type == "foo") | .names] | flatten | sort | unique' sample.yml +``` +will output +```yaml +- Ava +- Catherine +- Fred +``` + +### Explanation: +- `.[] | select(.type == "foo") | .names` will select the array elements of type "foo" +- Splat `.[]` will unwrap the array and match all the items. We need to do this so we can work on the child items, for instance, filter items out using the `select` operator. +- But we still want the final results back into an array. So after we're doing working on the children, we wrap everything back into an array using square brackets around the expression. `[.[] | select(.type == "foo") | .names]` +- Now have have an array of all the 'names' values. Which includes arrays of strings as well as strings on their own. +- Pipe `|` this array through `flatten`. This will flatten nested arrays. So now we have a flat list of all the name value strings +- Next we pipe `|` that through `sort` and then `unique` to get a sorted, unique list of the names! +- See the [flatten](https://mikefarah.gitbook.io/yq/operators/flatten), [sort](https://mikefarah.gitbook.io/yq/operators/sort) and [unique](https://mikefarah.gitbook.io/yq/operators/unique) for more information and examples. + diff --git a/pkg/yqlib/recipes_test.go b/pkg/yqlib/recipes_test.go index 6fab7d6b..c04fa1de 100644 --- a/pkg/yqlib/recipes_test.go +++ b/pkg/yqlib/recipes_test.go @@ -6,7 +6,8 @@ import ( var recipes = []expressionScenario{ { - description: "Find items in an array", + description: "Find items in an array", + subdescription: "We have an array and we want to find the elements with a particular name.", explanation: []string{ "`.[]` splats the array, and puts all the items in the context.", "These items are then piped (`|`) into `select(.name == \"Foo\")` which will select all the nodes that have a name property set to 'Foo'.", @@ -19,9 +20,10 @@ var recipes = []expressionScenario{ }, }, { - description: "Find and update items in an array", - document: `[{name: Foo, numBuckets: 0}, {name: Bar, numBuckets: 0}]`, - expression: `(.[] | select(.name == "Foo") | .numBuckets) |= . + 1`, + description: "Find and update items in an array", + subdescription: "We have an array and we want to _update_ the elements with a particular name.", + document: `[{name: Foo, numBuckets: 0}, {name: Bar, numBuckets: 0}]`, + expression: `(.[] | select(.name == "Foo") | .numBuckets) |= . + 1`, explanation: []string{ "Following from the example above`.[]` splats the array, selects filters the items.", "We then pipe (`|`) that into `.numBuckets`, which will select that field from all the matching items", @@ -34,6 +36,52 @@ var recipes = []expressionScenario{ "D0, P[], (doc)::[{name: Foo, numBuckets: 1}, {name: Bar, numBuckets: 0}]\n", }, }, + { + description: "Multiple or complex updates to items in an array", + subdescription: "We have an array and we want to _update_ the elements with a particular name in reference to its type.", + document: `myArray: [{name: Foo, type: cat}, {name: Bar, type: dog}]`, + expression: `with(.myArray[]; .name = .name + " - " + .type)`, + explanation: []string{ + "The with operator will effectively loop through each given item in the first given expression, and run the second expression against it.", + "`.myArray[]` splats the array in `myArray`. So `with` will run against each item in that array", + "`.name = .name + \" - \" + .type` this expression is run against every item, updating the name to be a concatenation of the original name as well as the type.", + "See the [with](https://mikefarah.gitbook.io/yq/operators/with) operator for more information and examples.", + }, + expected: []string{ + "D0, P[], (doc)::myArray: [{name: Foo - cat, type: cat}, {name: Bar - dog, type: dog}]\n", + }, + }, + { + description: "Sort an array by a field", + document: `myArray: [{name: Foo, numBuckets: 1}, {name: Bar, numBuckets: 0}]`, + expression: `.myArray |= sort_by(.numBuckets)`, + explanation: []string{ + "We want to resort `.myArray`.", + "`sort_by` works by piping an array into it, and it pipes out a sorted array.", + "So, we use `|=` to update `.myArray`. This is the same as doing `.myArray = (.myArray | sort_by(.numBuckets))`", + }, + expected: []string{ + "D0, P[], (doc)::myArray: [{name: Bar, numBuckets: 0}, {name: Foo, numBuckets: 1}]\n", + }, + }, + { + description: "Filter, flatten, sort and unique", + subdescription: "Lets", + document: `[{type: foo, names: [Fred, Catherine]}, {type: bar, names: [Zelda]}, {type: foo, names: Fred}, {type: foo, names: Ava}]`, + expression: `[.[] | select(.type == "foo") | .names] | flatten | sort | unique`, + explanation: []string{ + "`.[] | select(.type == \"foo\") | .names` will select the array elements of type \"foo\"", + "Splat `.[]` will unwrap the array and match all the items. We need to do this so we can work on the child items, for instance, filter items out using the `select` operator.", + "But we still want the final results back into an array. So after we're doing working on the children, we wrap everything back into an array using square brackets around the expression. `[.[] | select(.type == \"foo\") | .names]`", + "Now have have an array of all the 'names' values. Which includes arrays of strings as well as strings on their own.", + "Pipe `|` this array through `flatten`. This will flatten nested arrays. So now we have a flat list of all the name value strings", + "Next we pipe `|` that through `sort` and then `unique` to get a sorted, unique list of the names!", + "See the [flatten](https://mikefarah.gitbook.io/yq/operators/flatten), [sort](https://mikefarah.gitbook.io/yq/operators/sort) and [unique](https://mikefarah.gitbook.io/yq/operators/unique) for more information and examples.", + }, + expected: []string{ + "D0, P[], (!!seq)::- Ava\n- Catherine\n- Fred\n", + }, + }, } func TestRecipes(t *testing.T) { From 62b25b1bf311b7a4072564eda418624976219437 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 1 Sep 2023 14:43:38 +1000 Subject: [PATCH 23/43] Added recipes documentation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5c36aa73..7bed4fbb 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Convert JSON to YAML yq -Poy sample.json ``` -See the [documentation](https://mikefarah.gitbook.io/yq/) for more examples. +See [recipes](https://mikefarah.gitbook.io/yq/recipes) for more examples and the [documentation](https://mikefarah.gitbook.io/yq/) for more information. Take a look at the discussions for [common questions](https://github.com/mikefarah/yq/discussions/categories/q-a), and [cool ideas](https://github.com/mikefarah/yq/discussions/categories/show-and-tell) From de55fb398864245f1487a3b66f79c87fd4ba3de3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Sep 2023 09:26:08 +1000 Subject: [PATCH 24/43] Bump github.com/pelletier/go-toml/v2 from 2.0.9 to 2.1.0 (#1768) Bumps [github.com/pelletier/go-toml/v2](https://github.com/pelletier/go-toml) from 2.0.9 to 2.1.0. - [Release notes](https://github.com/pelletier/go-toml/releases) - [Changelog](https://github.com/pelletier/go-toml/blob/v2/.goreleaser.yaml) - [Commits](https://github.com/pelletier/go-toml/compare/v2.0.9...v2.1.0) --- updated-dependencies: - dependency-name: github.com/pelletier/go-toml/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 81dd79c6..e115d90d 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/goccy/go-yaml v1.11.0 github.com/jinzhu/copier v0.4.0 github.com/magiconair/properties v1.8.7 - github.com/pelletier/go-toml/v2 v2.0.9 + github.com/pelletier/go-toml/v2 v2.1.0 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index ecaa6d37..6a7bfc16 100644 --- a/go.sum +++ b/go.sum @@ -31,8 +31,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0= -github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= +github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= From 95f61c6f69140dc1c0f5a18a5fa555ad4c739010 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Sep 2023 07:57:39 +1000 Subject: [PATCH 25/43] Bump golang from 1.21.0 to 1.21.1 (#1773) Bumps golang from 1.21.0 to 1.21.1. --- updated-dependencies: - dependency-name: golang dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- Dockerfile.dev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8ba25356..e5a9ae48 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.21.0 as builder +FROM golang:1.21.1 as builder WORKDIR /go/src/mikefarah/yq diff --git a/Dockerfile.dev b/Dockerfile.dev index 55d70477..d64eccb7 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM golang:1.21.0 +FROM golang:1.21.1 COPY scripts/devtools.sh /opt/devtools.sh From 594a0be8562327d49fb58cf588c02df27edf5d13 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Sep 2023 07:57:49 +1000 Subject: [PATCH 26/43] Bump golang.org/x/text from 0.12.0 to 0.13.0 (#1774) Bumps [golang.org/x/text](https://github.com/golang/text) from 0.12.0 to 0.13.0. - [Release notes](https://github.com/golang/text/releases) - [Commits](https://github.com/golang/text/compare/v0.12.0...v0.13.0) --- updated-dependencies: - dependency-name: golang.org/x/text dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e115d90d..b976b019 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 golang.org/x/net v0.14.0 - golang.org/x/text v0.12.0 + golang.org/x/text v0.13.0 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 6a7bfc16..5b452a77 100644 --- a/go.sum +++ b/go.sum @@ -55,8 +55,8 @@ golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= From 0a0182da13a11616212535fa3f03006845606250 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Sep 2023 07:58:30 +1000 Subject: [PATCH 27/43] Bump actions/checkout from 3 to 4 (#1776) Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/codeql.yml | 2 +- .github/workflows/docker-release.yml | 2 +- .github/workflows/go.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/snap-release.yml | 2 +- .github/workflows/test-yq.yml | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 128f89d7..033fb535 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -38,7 +38,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 17890b27..c3a180a4 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -14,7 +14,7 @@ jobs: IMAGE_NAME: mikefarah/yq runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up QEMU uses: docker/setup-qemu-action@v2 diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index a4eba0ad..ab4e62af 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -17,7 +17,7 @@ jobs: id: go - name: Check out code into the Go module directory - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Get dependencies run: | diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 99b4aad4..e1901562 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,7 +9,7 @@ jobs: publishGitRelease: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-go@v4 with: go-version: '^1.20' diff --git a/.github/workflows/snap-release.yml b/.github/workflows/snap-release.yml index 4fbfec03..bb5050f8 100644 --- a/.github/workflows/snap-release.yml +++ b/.github/workflows/snap-release.yml @@ -12,7 +12,7 @@ jobs: environment: snap runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: snapcore/action-build@v1 id: build - uses: snapcore/action-publish@v1 diff --git a/.github/workflows/test-yq.yml b/.github/workflows/test-yq.yml index ef34d144..921c3aa3 100644 --- a/.github/workflows/test-yq.yml +++ b/.github/workflows/test-yq.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Get test id: get_value uses: mikefarah/yq@master From e09779e0042f1aec21a416212664e60518017002 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Tue, 19 Sep 2023 09:42:07 +1000 Subject: [PATCH 28/43] Fixed number parsing as float bug in JSON #1756 --- pkg/yqlib/decoder_json.go | 12 ++++++++++-- pkg/yqlib/json_test.go | 4 ++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/pkg/yqlib/decoder_json.go b/pkg/yqlib/decoder_json.go index c6b88eca..ea712bfc 100644 --- a/pkg/yqlib/decoder_json.go +++ b/pkg/yqlib/decoder_json.go @@ -54,8 +54,16 @@ func (dec *jsonDecoder) convertToYamlNode(data *orderedMap) (*yaml.Node, error) case nil: return createScalarNode(nil, "null"), nil case float64, float32: - // json decoder returns ints as float. - return parseSnippet(fmt.Sprintf("%v", rawData)) + // json decoder returns ints as float.' + intNum := int(rawData.(float64)) + + // if the integer representation is the same as the original + // then its an int. + if float64(intNum) == rawData.(float64) { + return createScalarNode(intNum, fmt.Sprintf("%v", intNum)), nil + } + + return createScalarNode(rawData, fmt.Sprintf("%v", rawData)), nil case int, int64, int32, string, bool: return createScalarNode(rawData, fmt.Sprintf("%v", rawData)), nil case []*orderedMap: diff --git a/pkg/yqlib/json_test.go b/pkg/yqlib/json_test.go index 919fce9f..cda716ba 100644 --- a/pkg/yqlib/json_test.go +++ b/pkg/yqlib/json_test.go @@ -198,8 +198,8 @@ var jsonScenarios = []formatScenario{ { description: "numbers", skipDoc: true, - input: "[3, 3.0, 3.1, -1]", - expected: "- 3\n- 3\n- 3.1\n- -1\n", + input: "[3, 3.0, 3.1, -1, 999999, 1000000, 1000001, 1.1]", + expected: "- 3\n- 3\n- 3.1\n- -1\n- 999999\n- 1000000\n- 1000001\n- 1.1\n", scenarioType: "decode-ndjson", }, { From e126344c00ed8c5712ccaa14a9f976adeae987c1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 09:44:03 +1000 Subject: [PATCH 29/43] Bump github.com/goccy/go-yaml from 1.11.0 to 1.11.1 (#1788) Bumps [github.com/goccy/go-yaml](https://github.com/goccy/go-yaml) from 1.11.0 to 1.11.1. - [Release notes](https://github.com/goccy/go-yaml/releases) - [Changelog](https://github.com/goccy/go-yaml/blob/master/CHANGELOG.md) - [Commits](https://github.com/goccy/go-yaml/compare/v1.11.0...v1.11.1) --- updated-dependencies: - dependency-name: github.com/goccy/go-yaml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b976b019..682d7740 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/elliotchance/orderedmap v1.5.0 github.com/fatih/color v1.15.0 github.com/goccy/go-json v0.10.2 - github.com/goccy/go-yaml v1.11.0 + github.com/goccy/go-yaml v1.11.1 github.com/jinzhu/copier v0.4.0 github.com/magiconair/properties v1.8.7 github.com/pelletier/go-toml/v2 v2.1.0 diff --git a/go.sum b/go.sum index 5b452a77..342de037 100644 --- a/go.sum +++ b/go.sum @@ -17,8 +17,8 @@ github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/goccy/go-yaml v1.11.0 h1:n7Z+zx8S9f9KgzG6KtQKf+kwqXZlLNR2F6018Dgau54= -github.com/goccy/go-yaml v1.11.0/go.mod h1:H+mJrWtjPTJAHvRbV09MCK9xYwODM+wRTVFFTWckfng= +github.com/goccy/go-yaml v1.11.1 h1:LEoI21G2M4za/y6Ik7x7FYHIBwI2K7579x8zxvPBcD4= +github.com/goccy/go-yaml v1.11.1/go.mod h1:H+mJrWtjPTJAHvRbV09MCK9xYwODM+wRTVFFTWckfng= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= From 974ded5add5a82330ac4c389674b186199e1fca4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 09:44:23 +1000 Subject: [PATCH 30/43] Bump github.com/alecthomas/participle/v2 from 2.0.0 to 2.1.0 (#1787) Bumps [github.com/alecthomas/participle/v2](https://github.com/alecthomas/participle) from 2.0.0 to 2.1.0. - [Release notes](https://github.com/alecthomas/participle/releases) - [Changelog](https://github.com/alecthomas/participle/blob/master/CHANGES.md) - [Commits](https://github.com/alecthomas/participle/compare/v2.0.0...v2.1.0) --- updated-dependencies: - dependency-name: github.com/alecthomas/participle/v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 682d7740..f1d11f95 100644 --- a/go.mod +++ b/go.mod @@ -2,7 +2,7 @@ module github.com/mikefarah/yq/v4 require ( github.com/a8m/envsubst v1.4.2 - github.com/alecthomas/participle/v2 v2.0.0 + github.com/alecthomas/participle/v2 v2.1.0 github.com/alecthomas/repr v0.2.0 github.com/dimchansky/utfbom v1.1.1 github.com/elliotchance/orderedmap v1.5.0 diff --git a/go.sum b/go.sum index 342de037..0ea769f9 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,8 @@ github.com/a8m/envsubst v1.4.2 h1:4yWIHXOLEJHQEFd4UjrWDrYeYlV7ncFWJOCBRLOZHQg= github.com/a8m/envsubst v1.4.2/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY= -github.com/alecthomas/assert/v2 v2.2.2 h1:Z/iVC0xZfWTaFNE6bA3z07T86hd45Xe2eLt6WVy2bbk= -github.com/alecthomas/participle/v2 v2.0.0 h1:Fgrq+MbuSsJwIkw3fEj9h75vDP0Er5JzepJ0/HNHv0g= -github.com/alecthomas/participle/v2 v2.0.0/go.mod h1:rAKZdJldHu8084ojcWevWAL8KmEU+AT+Olodb+WoN2Y= +github.com/alecthomas/assert/v2 v2.3.0 h1:mAsH2wmvjsuvyBvAmCtm7zFsBlb8mIHx5ySLVdDZXL0= +github.com/alecthomas/participle/v2 v2.1.0 h1:z7dElHRrOEEq45F2TG5cbQihMtNTv8vwldytDj7Wrz4= +github.com/alecthomas/participle/v2 v2.1.0/go.mod h1:Y1+hAs8DHPmc3YUFzqllV+eSQ9ljPTk0ZkPMtEdAx2c= github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk= github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= From fa8cfd0400fdf83fa729b648628ef2bb0e6de29d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 09:44:36 +1000 Subject: [PATCH 31/43] Bump docker/setup-buildx-action from 2 to 3 (#1786) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 2 to 3. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index c3a180a4..90161467 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -23,7 +23,7 @@ jobs: - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: version: latest From 0667500b1b47140d60943cf708f6e4a9a76c93d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 09:45:31 +1000 Subject: [PATCH 32/43] Bump docker/setup-qemu-action from 2 to 3 (#1785) Bumps [docker/setup-qemu-action](https://github.com/docker/setup-qemu-action) from 2 to 3. - [Release notes](https://github.com/docker/setup-qemu-action/releases) - [Commits](https://github.com/docker/setup-qemu-action/compare/v2...v3) --- updated-dependencies: - dependency-name: docker/setup-qemu-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/docker-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 90161467..9840d2bc 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v4 - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 with: platforms: all From 980b0e6daa6f5726a57e453d879d54bdd148e189 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 09:45:46 +1000 Subject: [PATCH 33/43] Bump golang.org/x/net from 0.14.0 to 0.15.0 (#1775) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.14.0 to 0.15.0. - [Commits](https://github.com/golang/net/compare/v0.14.0...v0.15.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index f1d11f95..dd783cdc 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 - golang.org/x/net v0.14.0 + golang.org/x/net v0.15.0 golang.org/x/text v0.13.0 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 gopkg.in/yaml.v3 v3.0.1 @@ -25,7 +25,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.17 // indirect - golang.org/x/sys v0.11.0 // indirect + golang.org/x/sys v0.12.0 // indirect golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect ) diff --git a/go.sum b/go.sum index 0ea769f9..27fe8041 100644 --- a/go.sum +++ b/go.sum @@ -50,11 +50,11 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0= From cdb9b56066e698de711e8284b9abf624760051a3 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Tue, 19 Sep 2023 09:52:36 +1000 Subject: [PATCH 34/43] Spellig with cspell --- Makefile | 6 +- README.md | 8 +- acceptance_tests/bad_args.sh | 8 +- acceptance_tests/basic.sh | 6 +- acceptance_tests/leading-seperator.sh | 50 ++--- cmd/evaluate_all_command.go | 2 +- cmd/evalute_sequence_command.go | 2 +- cmd/root.go | 2 +- cmd/unwrap_flag.go | 12 +- cmd/utils.go | 6 +- cspell.config.yaml | 14 ++ pkg/yqlib/decoder_yaml.go | 2 +- pkg/yqlib/doc/operators/group-by.md | 2 +- pkg/yqlib/doc/operators/headers/Main.md | 2 +- pkg/yqlib/doc/usage/shellvariables.md | 4 +- pkg/yqlib/encoder_lua.go | 2 +- pkg/yqlib/encoder_properties.go | 2 +- pkg/yqlib/encoder_yaml.go | 2 +- pkg/yqlib/lexer_participle.go | 4 +- pkg/yqlib/operator_comments.go | 8 +- pkg/yqlib/operator_datetime.go | 8 +- pkg/yqlib/operator_group_by_test.go | 2 +- pkg/yqlib/operator_keys.go | 4 +- pkg/yqlib/operator_pick.go | 2 +- pkg/yqlib/operator_traverse_path.go | 2 +- pkg/yqlib/printer.go | 2 +- pkg/yqlib/printer_test.go | 20 +- pkg/yqlib/shellvariables_test.go | 8 +- pkg/yqlib/write_in_place_handler.go | 2 +- pkg/yqlib/xml_test.go | 4 +- project-words.txt | 253 ++++++++++++++++++++++++ scripts/extract-checksum.sh | 4 +- scripts/generate-man-page-md.sh | 2 +- scripts/generate-man-page.sh | 2 +- scripts/release-deb.sh | 2 +- scripts/spelling.sh | 3 + 36 files changed, 369 insertions(+), 95 deletions(-) create mode 100644 cspell.config.yaml create mode 100644 project-words.txt create mode 100755 scripts/spelling.sh diff --git a/Makefile b/Makefile index a186d5a8..42c11fb2 100644 --- a/Makefile +++ b/Makefile @@ -84,8 +84,12 @@ format: vendor ${ENGINERUN} bash ./scripts/format.sh +.PHONY: spelling +spelling: format + ${ENGINERUN} bash ./scripts/spelling.sh + .PHONY: secure -secure: format +secure: spelling ${ENGINERUN} bash ./scripts/secure.sh .PHONY: check diff --git a/README.md b/README.md index 7bed4fbb..06096704 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ Pipe from STDIN: yq '.a.b[0].c' < file.yaml ``` -Update a yaml file, inplace +Update a yaml file, in place ```bash yq -i '.a.b[0].c = "cool"' file.yaml ``` @@ -310,7 +310,7 @@ https://pkgs.alpinelinux.org/package/edge/community/x86/yq - [Deeply data structures](https://mikefarah.gitbook.io/yq/operators/traverse-read) - [Sort keys](https://mikefarah.gitbook.io/yq/operators/sort-keys) - Manipulate yaml [comments](https://mikefarah.gitbook.io/yq/operators/comment-operators), [styling](https://mikefarah.gitbook.io/yq/operators/style), [tags](https://mikefarah.gitbook.io/yq/operators/tag) and [anchors and aliases](https://mikefarah.gitbook.io/yq/operators/anchor-and-alias-operators). -- [Update inplace](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags) +- [Update in place](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags) - [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/operators/select#select-and-update-matching-values-in-map) - Keeps yaml formatting and comments when updating (though there are issues with whitespace) - [Decode/Encode base64 data](https://mikefarah.gitbook.io/yq/operators/encode-decode) @@ -337,7 +337,7 @@ Examples: # yq defaults to 'eval' command if no command is specified. See "yq eval --help" for more examples. yq '.stuff' < myfile.yml # outputs the data at the "stuff" node from "myfile.yml" -yq -i '.stuff = "foo"' myfile.yml # update myfile.yml inplace +yq -i '.stuff = "foo"' myfile.yml # update myfile.yml in place Available Commands: @@ -354,7 +354,7 @@ Flags: --header-preprocess Slurp any header comments and separators before processing expression. (default true) -h, --help help for yq -I, --indent int sets indent level for output (default 2) - -i, --inplace update the file inplace of first file given. + -i, --inplace update the file in place of first file given. -p, --input-format string [yaml|y|xml|x] parse format for input. Note that json is a subset of yaml. (default "yaml") -M, --no-colors force print with no colors -N, --no-doc Don't print document separators (---) diff --git a/acceptance_tests/bad_args.sh b/acceptance_tests/bad_args.sh index 34d43b6e..c5ac76db 100755 --- a/acceptance_tests/bad_args.sh +++ b/acceptance_tests/bad_args.sh @@ -3,25 +3,25 @@ testWriteInPlacePipeIn() { result=$(./yq e -i -n '.a' 2>&1) assertEquals 1 $? - assertEquals "Error: write inplace flag only applicable when giving an expression and at least one file" "$result" + assertEquals "Error: write in place flag only applicable when giving an expression and at least one file" "$result" } testWriteInPlacePipeInEvalall() { result=$(./yq ea -i -n '.a' 2>&1) assertEquals 1 $? - assertEquals "Error: write inplace flag only applicable when giving an expression and at least one file" "$result" + assertEquals "Error: write in place flag only applicable when giving an expression and at least one file" "$result" } testWriteInPlaceWithSplit() { result=$(./yq e -s "cat" -i '.a = "thing"' test.yml 2>&1) assertEquals 1 $? - assertEquals "Error: write inplace cannot be used with split file" "$result" + assertEquals "Error: write in place cannot be used with split file" "$result" } testWriteInPlaceWithSplitEvalAll() { result=$(./yq ea -s "cat" -i '.a = "thing"' test.yml 2>&1) assertEquals 1 $? - assertEquals "Error: write inplace cannot be used with split file" "$result" + assertEquals "Error: write in place cannot be used with split file" "$result" } testNullWithFiles() { diff --git a/acceptance_tests/basic.sh b/acceptance_tests/basic.sh index 141122c4..5705ea53 100755 --- a/acceptance_tests/basic.sh +++ b/acceptance_tests/basic.sh @@ -143,7 +143,7 @@ testBasicCatWithFilesNoDash() { } # when the nullinput flag is used -# dont automatically read STDIN (this breaks github actions) +# don't automatically read STDIN (this breaks github actions) testBasicCreateFileGithubAction() { cat /dev/null | ./yq -n ".a = 123" > test.yml } @@ -302,7 +302,7 @@ testBasicExitStatusNoEval() { assertEquals 1 "$?" } -testBasicExtractFieldWithSeperator() { +testBasicExtractFieldWithSeparator() { cat >test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml <test.yml < 0 { @@ -104,7 +104,7 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) { outputFormatType == yqlib.PropsOutputFormat { unwrapScalar = true } - if unwrapScalarFlag.IsExplicitySet() { + if unwrapScalarFlag.IsExplicitlySet() { unwrapScalar = unwrapScalarFlag.IsSet() } diff --git a/cspell.config.yaml b/cspell.config.yaml new file mode 100644 index 00000000..99ef61e1 --- /dev/null +++ b/cspell.config.yaml @@ -0,0 +1,14 @@ +--- +$schema: https://raw.githubusercontent.com/streetsidesoftware/cspell/main/cspell.schema.json +version: '0.2' +language: en-GB +dictionaryDefinitions: + - name: project-words + path: './project-words.txt' + addWords: true +dictionaries: + - project-words +ignorePaths: + - 'vendor' + - 'bin' + - '/project-words.txt' diff --git a/pkg/yqlib/decoder_yaml.go b/pkg/yqlib/decoder_yaml.go index a87233bc..c2652744 100644 --- a/pkg/yqlib/decoder_yaml.go +++ b/pkg/yqlib/decoder_yaml.go @@ -49,7 +49,7 @@ func (dec *yamlDecoder) processReadStream(reader *bufio.Reader) (io.Reader, stri } } else if string(peekBytes) == "---" { _, err := reader.ReadString('\n') - sb.WriteString("$yqDocSeperator$\n") + sb.WriteString("$yqDocSeparator$\n") if errors.Is(err, io.EOF) { return reader, sb.String(), nil } else if err != nil { diff --git a/pkg/yqlib/doc/operators/group-by.md b/pkg/yqlib/doc/operators/group-by.md index f5169b72..c06b273e 100644 --- a/pkg/yqlib/doc/operators/group-by.md +++ b/pkg/yqlib/doc/operators/group-by.md @@ -26,7 +26,7 @@ will output bar: 100 ``` -## Group by field, with nuls +## Group by field, with nulls Given a sample.yml file of: ```yaml - cat: dog diff --git a/pkg/yqlib/doc/operators/headers/Main.md b/pkg/yqlib/doc/operators/headers/Main.md index 59284fdf..d6c4c1c7 100644 --- a/pkg/yqlib/doc/operators/headers/Main.md +++ b/pkg/yqlib/doc/operators/headers/Main.md @@ -26,7 +26,7 @@ yq '.a.b[0].c' file.yaml cat file.yaml | yq '.a.b[0].c' ``` -## Update a yaml file, inplace +## Update a yaml file, in place ```bash yq -i '.a.b[0].c = "cool"' file.yaml ``` diff --git a/pkg/yqlib/doc/usage/shellvariables.md b/pkg/yqlib/doc/usage/shellvariables.md index a2919715..ba50315a 100644 --- a/pkg/yqlib/doc/usage/shellvariables.md +++ b/pkg/yqlib/doc/usage/shellvariables.md @@ -34,7 +34,7 @@ Given a sample.yml file of: ascii_=_symbols: replaced with _ "ascii_ _controls": dropped (this example uses \t) nonascii_א_characters: dropped -effrot_expeñded_tò_preserve_accented_latin_letters: moderate (via unicode NFKD) +effort_expeñded_tò_preserve_accented_latin_letters: moderate (via unicode NFKD) ``` then @@ -46,7 +46,7 @@ will output ascii___symbols='replaced with _' ascii__controls='dropped (this example uses \t)' nonascii__characters=dropped -effrot_expended_to_preserve_accented_latin_letters='moderate (via unicode NFKD)' +effort_expended_to_preserve_accented_latin_letters='moderate (via unicode NFKD)' ``` ## Encode shell variables: empty values, arrays and maps diff --git a/pkg/yqlib/encoder_lua.go b/pkg/yqlib/encoder_lua.go index 898b3855..a74c68cb 100644 --- a/pkg/yqlib/encoder_lua.go +++ b/pkg/yqlib/encoder_lua.go @@ -100,7 +100,7 @@ func (le *luaEncoder) encodeString(writer io.Writer, node *yaml.Node) error { case yaml.SingleQuotedStyle: quote = "'" - // falltrough to regular ol' string + // fallthrough to regular ol' string } return writeString(writer, quote+le.escape.Replace(node.Value)+quote) } diff --git a/pkg/yqlib/encoder_properties.go b/pkg/yqlib/encoder_properties.go index d615fa52..575b4e89 100644 --- a/pkg/yqlib/encoder_properties.go +++ b/pkg/yqlib/encoder_properties.go @@ -37,7 +37,7 @@ func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content strin if errReading != nil && !errors.Is(errReading, io.EOF) { return errReading } - if strings.Contains(readline, "$yqDocSeperator$") { + if strings.Contains(readline, "$yqDocSeparator$") { if err := pe.PrintDocumentSeparator(writer); err != nil { return err diff --git a/pkg/yqlib/encoder_yaml.go b/pkg/yqlib/encoder_yaml.go index 315a4695..75b64c04 100644 --- a/pkg/yqlib/encoder_yaml.go +++ b/pkg/yqlib/encoder_yaml.go @@ -47,7 +47,7 @@ func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) err if errReading != nil && !errors.Is(errReading, io.EOF) { return errReading } - if strings.Contains(readline, "$yqDocSeperator$") { + if strings.Contains(readline, "$yqDocSeparator$") { if err := ye.PrintDocumentSeparator(writer); err != nil { return err diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go index 3a3b7c10..2352354a 100644 --- a/pkg/yqlib/lexer_participle.go +++ b/pkg/yqlib/lexer_participle.go @@ -26,8 +26,8 @@ var participleYqRules = []*participleYqRule{ {"RecursiveDecent", `\.\.`, recursiveDecentOpToken(false), 0}, {"GetVariable", `\$[a-zA-Z_\-0-9]+`, getVariableOpToken(), 0}, - {"AsignAsVariable", `as`, opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{}), 0}, - {"AsignRefVariable", `ref`, opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{IsReference: true}), 0}, + {"AssignAsVariable", `as`, opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{}), 0}, + {"AssignRefVariable", `ref`, opTokenWithPrefs(assignVariableOpType, nil, assignVarPreferences{IsReference: true}), 0}, {"CreateMap", `:\s*`, opToken(createMapOpType), 0}, simpleOp("length", lengthOpType), diff --git a/pkg/yqlib/operator_comments.go b/pkg/yqlib/operator_comments.go index 5bbe2e59..ecf882af 100644 --- a/pkg/yqlib/operator_comments.go +++ b/pkg/yqlib/operator_comments.go @@ -77,8 +77,8 @@ func assignCommentsOperator(d *dataTreeNavigator, context Context, expressionNod func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { preferences := expressionNode.Operation.Preferences.(commentOpPreferences) - var startCommentCharaterRegExp = regexp.MustCompile(`^# `) - var subsequentCommentCharaterRegExp = regexp.MustCompile(`\n# `) + var startCommentCharacterRegExp = regexp.MustCompile(`^# `) + var subsequentCommentCharacterRegExp = regexp.MustCompile(`\n# `) log.Debugf("GetComments operator!") var results = list.New() @@ -112,8 +112,8 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode * } else if preferences.FootComment { comment = candidate.Node.FootComment } - comment = startCommentCharaterRegExp.ReplaceAllString(comment, "") - comment = subsequentCommentCharaterRegExp.ReplaceAllString(comment, "\n") + comment = startCommentCharacterRegExp.ReplaceAllString(comment, "") + comment = subsequentCommentCharacterRegExp.ReplaceAllString(comment, "\n") node := &yaml.Node{Kind: yaml.ScalarNode, Value: comment, Tag: "!!str"} result := candidate.CreateReplacement(node) diff --git a/pkg/yqlib/operator_datetime.go b/pkg/yqlib/operator_datetime.go index 25bafa9e..03cdf95f 100644 --- a/pkg/yqlib/operator_datetime.go +++ b/pkg/yqlib/operator_datetime.go @@ -10,7 +10,7 @@ import ( "gopkg.in/yaml.v3" ) -func getStringParamter(parameterName string, d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (string, error) { +func getStringParameter(parameterName string, d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (string, error) { result, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode) if err != nil { @@ -24,7 +24,7 @@ func getStringParamter(parameterName string, d *dataTreeNavigator, context Conte func withDateTimeFormat(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { if expressionNode.RHS.Operation.OperationType == blockOpType || expressionNode.RHS.Operation.OperationType == unionOpType { - layout, err := getStringParamter("layout", d, context, expressionNode.RHS.LHS) + layout, err := getStringParameter("layout", d, context, expressionNode.RHS.LHS) if err != nil { return Context{}, fmt.Errorf("could not get date time format: %w", err) } @@ -63,7 +63,7 @@ func parseDateTime(layout string, datestring string) (time.Time, error) { } func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - format, err := getStringParamter("format", d, context, expressionNode.RHS) + format, err := getStringParameter("format", d, context, expressionNode.RHS) layout := context.GetDateTimeLayout() if err != nil { @@ -97,7 +97,7 @@ func formatDateTime(d *dataTreeNavigator, context Context, expressionNode *Expre } func tzOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { - timezoneStr, err := getStringParamter("timezone", d, context, expressionNode.RHS) + timezoneStr, err := getStringParameter("timezone", d, context, expressionNode.RHS) layout := context.GetDateTimeLayout() if err != nil { diff --git a/pkg/yqlib/operator_group_by_test.go b/pkg/yqlib/operator_group_by_test.go index 2c7803da..7332bd32 100644 --- a/pkg/yqlib/operator_group_by_test.go +++ b/pkg/yqlib/operator_group_by_test.go @@ -14,7 +14,7 @@ var groupByOperatorScenarios = []expressionScenario{ }, }, { - description: "Group by field, with nuls", + description: "Group by field, with nulls", document: `[{cat: dog}, {foo: 1, bar: 10}, {foo: 3, bar: 100}, {no: foo for you}, {foo: 1, bar: 1}]`, expression: `group_by(.foo)`, expected: []string{ diff --git a/pkg/yqlib/operator_keys.go b/pkg/yqlib/operator_keys.go index 0e77310f..3ea4b5da 100644 --- a/pkg/yqlib/operator_keys.go +++ b/pkg/yqlib/operator_keys.go @@ -50,7 +50,7 @@ func keysOperator(d *dataTreeNavigator, context Context, expressionNode *Express if node.Kind == yaml.MappingNode { targetNode = getMapKeys(node) } else if node.Kind == yaml.SequenceNode { - targetNode = getIndicies(node) + targetNode = getIndices(node) } else { return Context{}, fmt.Errorf("Cannot get keys of %v, keys only works for maps and arrays", node.Tag) } @@ -70,7 +70,7 @@ func getMapKeys(node *yaml.Node) *yaml.Node { return &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq", Content: contents} } -func getIndicies(node *yaml.Node) *yaml.Node { +func getIndices(node *yaml.Node) *yaml.Node { var contents = make([]*yaml.Node, len(node.Content)) for index := range node.Content { diff --git a/pkg/yqlib/operator_pick.go b/pkg/yqlib/operator_pick.go index 416d947d..5a6f7d2c 100644 --- a/pkg/yqlib/operator_pick.go +++ b/pkg/yqlib/operator_pick.go @@ -76,7 +76,7 @@ func pickOperator(d *dataTreeNavigator, context Context, expressionNode *Express } } else { - return Context{}, fmt.Errorf("cannot pick indicies from type %v (%v)", node.Tag, candidate.GetNicePath()) + return Context{}, fmt.Errorf("cannot pick indices from type %v (%v)", node.Tag, candidate.GetNicePath()) } results.PushBack(candidate.CreateReplacementWithDocWrappers(replacement)) diff --git a/pkg/yqlib/operator_traverse_path.go b/pkg/yqlib/operator_traverse_path.go index 4308545a..5af781db 100644 --- a/pkg/yqlib/operator_traverse_path.go +++ b/pkg/yqlib/operator_traverse_path.go @@ -202,7 +202,7 @@ func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node, pr contentLength := len(node.Content) for contentLength <= index { if contentLength == 0 { - // default to nice yaml formating + // default to nice yaml formatting node.Style = 0 } diff --git a/pkg/yqlib/printer.go b/pkg/yqlib/printer.go index c430fcfe..8388b0d9 100644 --- a/pkg/yqlib/printer.go +++ b/pkg/yqlib/printer.go @@ -148,7 +148,7 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error { return errorWriting } - commentsStartWithSepExp := regexp.MustCompile(`^\$yqDocSeperator\$`) + commentsStartWithSepExp := regexp.MustCompile(`^\$yqDocSeparator\$`) commentStartsWithSeparator := commentsStartWithSepExp.MatchString(mappedDoc.LeadingContent) if (p.previousDocIndex != mappedDoc.Document || p.previousFileIndex != mappedDoc.FileIndex) && !commentStartsWithSeparator { diff --git a/pkg/yqlib/printer_test.go b/pkg/yqlib/printer_test.go index 4b32f371..fd6068f7 100644 --- a/pkg/yqlib/printer_test.go +++ b/pkg/yqlib/printer_test.go @@ -82,15 +82,15 @@ func TestPrinterMultipleDocsInSequenceWithLeadingContent(t *testing.T) { } el := inputs.Front() - el.Value.(*CandidateNode).LeadingContent = "# go cats\n$yqDocSeperator$\n" + el.Value.(*CandidateNode).LeadingContent = "# go cats\n$yqDocSeparator$\n" sample1 := nodeToList(el.Value.(*CandidateNode)) el = el.Next() - el.Value.(*CandidateNode).LeadingContent = "$yqDocSeperator$\n" + el.Value.(*CandidateNode).LeadingContent = "$yqDocSeparator$\n" sample2 := nodeToList(el.Value.(*CandidateNode)) el = el.Next() - el.Value.(*CandidateNode).LeadingContent = "$yqDocSeperator$\n# cool\n" + el.Value.(*CandidateNode).LeadingContent = "$yqDocSeparator$\n# cool\n" sample3 := nodeToList(el.Value.(*CandidateNode)) err = printer.PrintResults(sample1) @@ -174,21 +174,21 @@ func TestPrinterMultipleFilesInSequenceWithLeadingContent(t *testing.T) { elNode := el.Value.(*CandidateNode) elNode.Document = 0 elNode.FileIndex = 0 - elNode.LeadingContent = "# go cats\n$yqDocSeperator$\n" + elNode.LeadingContent = "# go cats\n$yqDocSeparator$\n" sample1 := nodeToList(elNode) el = el.Next() elNode = el.Value.(*CandidateNode) elNode.Document = 0 elNode.FileIndex = 1 - elNode.LeadingContent = "$yqDocSeperator$\n" + elNode.LeadingContent = "$yqDocSeparator$\n" sample2 := nodeToList(elNode) el = el.Next() elNode = el.Value.(*CandidateNode) elNode.Document = 0 elNode.FileIndex = 2 - elNode.LeadingContent = "$yqDocSeperator$\n# cool\n" + elNode.LeadingContent = "$yqDocSeparator$\n# cool\n" sample3 := nodeToList(elNode) err = printer.PrintResults(sample1) @@ -239,7 +239,7 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDoc(t *testing.T) { panic(err) } - inputs.Front().Value.(*CandidateNode).LeadingContent = "# go cats\n$yqDocSeperator$\n" + inputs.Front().Value.(*CandidateNode).LeadingContent = "# go cats\n$yqDocSeparator$\n" err = printer.PrintResults(inputs) if err != nil { @@ -267,7 +267,7 @@ func TestPrinterMultipleDocsInSinglePrintWithLeadingDocTrailing(t *testing.T) { if err != nil { panic(err) } - inputs.Front().Value.(*CandidateNode).LeadingContent = "$yqDocSeperator$\n" + inputs.Front().Value.(*CandidateNode).LeadingContent = "$yqDocSeparator$\n" err = printer.PrintResults(inputs) if err != nil { panic(err) @@ -313,7 +313,7 @@ func TestPrinterMultipleDocsJson(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) // note printDocSeparators is true, it should still not print document separators - // when outputing JSON. + // when outputting JSON. encoder := NewJSONEncoder(0, false, false) if encoder == nil { t.Skipf("no support for %s output format", "json") @@ -365,7 +365,7 @@ func TestPrinterNulSeparatorWithJson(t *testing.T) { var output bytes.Buffer var writer = bufio.NewWriter(&output) // note printDocSeparators is true, it should still not print document separators - // when outputing JSON. + // when outputting JSON. encoder := NewJSONEncoder(0, false, false) if encoder == nil { t.Skipf("no support for %s output format", "json") diff --git a/pkg/yqlib/shellvariables_test.go b/pkg/yqlib/shellvariables_test.go index a922734e..5a7f56f4 100644 --- a/pkg/yqlib/shellvariables_test.go +++ b/pkg/yqlib/shellvariables_test.go @@ -35,12 +35,12 @@ var shellVariablesScenarios = []formatScenario{ "ascii_=_symbols: replaced with _" + "\n" + "\"ascii_\t_controls\": dropped (this example uses \\t)" + "\n" + "nonascii_\u05d0_characters: dropped" + "\n" + - "effrot_expe\u00f1ded_t\u00f2_preserve_accented_latin_letters: moderate (via unicode NFKD)" + "\n", + "effort_expe\u00f1ded_t\u00f2_preserve_accented_latin_letters: moderate (via unicode NFKD)" + "\n", expected: "" + "ascii___symbols='replaced with _'" + "\n" + "ascii__controls='dropped (this example uses \\t)'" + "\n" + "nonascii__characters=dropped" + "\n" + - "effrot_expended_to_preserve_accented_latin_letters='moderate (via unicode NFKD)'" + "\n", + "effort_expended_to_preserve_accented_latin_letters='moderate (via unicode NFKD)'" + "\n", }, { description: "Encode shell variables: empty values, arrays and maps", @@ -65,10 +65,10 @@ func TestShellVariableScenarios(t *testing.T) { for i, s := range shellVariablesScenarios { genericScenarios[i] = s } - documentScenarios(t, "usage", "shellvariables", genericScenarios, documentShellVaraibleScenario) + documentScenarios(t, "usage", "shellvariables", genericScenarios, documentShellVariableScenario) } -func documentShellVaraibleScenario(_ *testing.T, w *bufio.Writer, i interface{}) { +func documentShellVariableScenario(_ *testing.T, w *bufio.Writer, i interface{}) { s := i.(formatScenario) if s.skipDoc { return diff --git a/pkg/yqlib/write_in_place_handler.go b/pkg/yqlib/write_in_place_handler.go index d49b2963..83d42134 100644 --- a/pkg/yqlib/write_in_place_handler.go +++ b/pkg/yqlib/write_in_place_handler.go @@ -44,7 +44,7 @@ func (w *writeInPlaceHandlerImpl) CreateTempFile() (*os.File, error) { } func (w *writeInPlaceHandlerImpl) FinishWriteInPlace(evaluatedSuccessfully bool) error { - log.Debug("Going to write-inplace, evaluatedSuccessfully=%v, target=%v", evaluatedSuccessfully, w.inputFilename) + log.Debug("Going to write in place, evaluatedSuccessfully=%v, target=%v", evaluatedSuccessfully, w.inputFilename) safelyCloseFile(w.tempFile) if evaluatedSuccessfully { log.Debug("Moving temp file to target") diff --git a/pkg/yqlib/xml_test.go b/pkg/yqlib/xml_test.go index e90e3654..fdf50b98 100644 --- a/pkg/yqlib/xml_test.go +++ b/pkg/yqlib/xml_test.go @@ -671,7 +671,7 @@ func documentXMLScenario(t *testing.T, w *bufio.Writer, i interface{}) { case "decode-raw-token-off": documentXMLDecodeKeepNsRawTokenScenario(w, s) case "roundtrip-skip-directives": - documentXMLSkipDirectrivesScenario(w, s) + documentXMLSkipDirectivesScenario(w, s) default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) @@ -787,7 +787,7 @@ func documentXMLRoundTripScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("```xml\n%v```\n\n", mustProcessFormatScenario(s, NewXMLDecoder(ConfiguredXMLPreferences), NewXMLEncoder(2, ConfiguredXMLPreferences)))) } -func documentXMLSkipDirectrivesScenario(w *bufio.Writer, s formatScenario) { +func documentXMLSkipDirectivesScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) if s.subdescription != "" { diff --git a/project-words.txt b/project-words.txt new file mode 100644 index 00000000..218a49b7 --- /dev/null +++ b/project-words.txt @@ -0,0 +1,253 @@ +abxbbxdbxebxczzx +abxbbxdbxebxczzy +accum +Accum +adithyasunil +AEDT +água +ÁGUA +alecthomas +appleapple +Astuff +autocreating +autoparse +AWST +axbxcxdxe +axbxcxdxexxx +bananabanana +barp +bitnami +blarp +blddir +Bobo +BODMAS +bonapite +Brien +Bstuff +BUILDKIT +buildpackage +catmeow +CATYPE +CBVVE +chardata +chillum +choco +chomper +cleanup +cmlu +colorise +colors +compinit +coolioo +coverprofile +createmap +csvd +CSVUTF +currentlabel +cygpath +czvf +datestring +datetime +Datetime +datetimes +DEBEMAIL +debhelper +Debugf +debuild +delish +delpaths +DELPATHS +devorbitus +devscripts +dimchansky +Dont +dput +elliotchance +endhint +endofname +Entriesfrom +envsubst +errorlevel +Escandón +Evalall +fakefilename +fakeroot +Farah +fatih +Fifi +filebytes +Fileish +foobar +foobaz +foof +frood +fullpath +gitbook +githubactions +gnupg +goccy +gofmt +gogo +golangci +GOMODCACHE +GOPATH +gosec +gota +goversion +GOVERSION +haha +headcommentwas +hellno +herbygillot +hexdump +Hoang +hostpath +hotdog +howdy +incase +inlinetables +inplace +ints +ireduce +iwatch +jinzhu +jq's +jsond +keygrip +Keygrip +KEYGRIP +KEYID +keyvalue +kwak +lalilu +ldflags +LDFLAGS +lexer +Lexer +libdistro +lindex +linecomment +magiconair +mapvalues +Mier +mikefarah +minideb +minishift +mipsle +mitchellh +mktemp +multidoc +multimaint +multine +myenv +myenvnonexisting +myfile +myformat +ndjson +NDJSON +NFKD +nixpkgs +nojson +nonascii +nonempty +noninteractive +Nonquoting +nosec +notoml +noxml +nullinput +onea +Oneshot +opencollect +opstack +orderedmap +orignal +osarch +overridign +pacman +Padder +pandoc +parsechangelog +pcsv +pelletier +pflag +prechecking +Prerelease +proc +propsd +qylib +readline +realnames +realpath +repr +rhash +rindex +risentveber +rmescandon +Rosey +roundtrip +Roundtrip +roundtripping +runningvms +sadface +selfupdate +setpath +sharedfolder +Sharedfolder +shellvariables +shellvars +shortfunc +shortpipe +shunit +Sidenote +snapcraft +somevalue +splt +squeek +srcdir +stackoverflow +stiched +Strc +strenv +strload +stylig +subarray +subchild +subdescription +submatch +submatches +SUBSTR +tempfile +tfstate +Tfstate +thar +timezone +Timezone +timezones +Timezones +tojson +Tokenvalue +traver +tsvd +Tuan +tzdata +Uhoh +updateassign +urid +utfbom +Warningf +Wazowski +webi +Webi +whereever +winget +withdots +wizz +woop +workdir +Writable +xmld +xyzzy +yamld +yqlib +zabbix diff --git a/scripts/extract-checksum.sh b/scripts/extract-checksum.sh index 914dcfa2..3ba11104 100755 --- a/scripts/extract-checksum.sh +++ b/scripts/extract-checksum.sh @@ -26,7 +26,7 @@ if [ "$1" == "" ]; then fi if [ "$2" != "" ]; then - # so we dont match x.tar.gz when 'x' is given + # so we don't match x.tar.gz when 'x' is given file="$2\s" else file="" @@ -47,7 +47,7 @@ fi grepMatch=$(grep -m 1 -n "$1" checksums_hashes_order) if [ "$grepMatch" == "" ]; then - echo "Could not find hash algorith '$1' in checksums_hashes_order" + echo "Could not find hash algorithm '$1' in checksums_hashes_order" exit 1 fi diff --git a/scripts/generate-man-page-md.sh b/scripts/generate-man-page-md.sh index 34283fc1..b2e0494c 100755 --- a/scripts/generate-man-page-md.sh +++ b/scripts/generate-man-page-md.sh @@ -1,7 +1,7 @@ #! /bin/bash set -e -# note that this reqires pandoc to be installed. +# note that this requires pandoc to be installed. cat ./pkg/yqlib/doc/operators/headers/Main.md > man.md printf "\n# HOW IT WORKS\n" >> man.md diff --git a/scripts/generate-man-page.sh b/scripts/generate-man-page.sh index b39e21bf..2d658004 100755 --- a/scripts/generate-man-page.sh +++ b/scripts/generate-man-page.sh @@ -1,7 +1,7 @@ #! /bin/bash set -e -# note that this reqires pandoc to be installed. +# note that this requires pandoc to be installed. pandoc \ --variable=title:"YQ" \ diff --git a/scripts/release-deb.sh b/scripts/release-deb.sh index c3cf5353..fc2d9a16 100755 --- a/scripts/release-deb.sh +++ b/scripts/release-deb.sh @@ -30,7 +30,7 @@ show_help() { echo " distribution is considered" echo " --goversion VERSION The version of Golang to use. Default to $GOVERSION" echo " -k, --sign-key KEYID Sign the package sources with the provided gpg key id (long format). When not provided this" - echo " paramater, the generated sources are not signed" + echo " parameter, the generated sources are not signed" echo " -s, --sign Sign the package sources with a gpg key of the maintainer" echo " -m, --maintainer WHO The maintainer used as author of the changelog. git.name and git.email (see git config) is" echo " the considered format" diff --git a/scripts/spelling.sh b/scripts/spelling.sh new file mode 100755 index 00000000..00c39f74 --- /dev/null +++ b/scripts/spelling.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +npx cspell --no-progress "**/*.{sh,go}" \ No newline at end of file From c750f1a48cbe1e0fee1a16b55c993649be59f133 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Sep 2023 11:50:33 +1000 Subject: [PATCH 35/43] Bump github.com/goccy/go-yaml from 1.11.1 to 1.11.2 (#1794) Bumps [github.com/goccy/go-yaml](https://github.com/goccy/go-yaml) from 1.11.1 to 1.11.2. - [Release notes](https://github.com/goccy/go-yaml/releases) - [Changelog](https://github.com/goccy/go-yaml/blob/master/CHANGELOG.md) - [Commits](https://github.com/goccy/go-yaml/compare/v1.11.1...v1.11.2) --- updated-dependencies: - dependency-name: github.com/goccy/go-yaml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index dd783cdc..ef06c29b 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/elliotchance/orderedmap v1.5.0 github.com/fatih/color v1.15.0 github.com/goccy/go-json v0.10.2 - github.com/goccy/go-yaml v1.11.1 + github.com/goccy/go-yaml v1.11.2 github.com/jinzhu/copier v0.4.0 github.com/magiconair/properties v1.8.7 github.com/pelletier/go-toml/v2 v2.1.0 diff --git a/go.sum b/go.sum index 27fe8041..825e4134 100644 --- a/go.sum +++ b/go.sum @@ -17,8 +17,8 @@ github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/goccy/go-yaml v1.11.1 h1:LEoI21G2M4za/y6Ik7x7FYHIBwI2K7579x8zxvPBcD4= -github.com/goccy/go-yaml v1.11.1/go.mod h1:H+mJrWtjPTJAHvRbV09MCK9xYwODM+wRTVFFTWckfng= +github.com/goccy/go-yaml v1.11.2 h1:joq77SxuyIs9zzxEjgyLBugMQ9NEgTWxXfz2wVqwAaQ= +github.com/goccy/go-yaml v1.11.2/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= From 05ad1dad908a64de45a2e8727828615e5af559dd Mon Sep 17 00:00:00 2001 From: -pthread -Wall Date: Tue, 26 Sep 2023 06:18:18 +0200 Subject: [PATCH 36/43] Fix typos (#1798) --- .../{leading-seperator.sh => leading-separator.sh} | 2 +- ...alute_sequence_command.go => evaluate_sequence_command.go} | 0 examples/{leading-seperator.yaml => leading-separator.yaml} | 0 pkg/yqlib/candidate_node.go | 2 +- pkg/yqlib/doc/usage/csv-tsv.md | 2 +- pkg/yqlib/doc/usage/headers/csv-tsv.md | 2 +- project-words.txt | 4 ++-- release_notes.txt | 2 +- scripts/shunit2 | 2 +- 9 files changed, 8 insertions(+), 8 deletions(-) rename acceptance_tests/{leading-seperator.sh => leading-separator.sh} (99%) rename cmd/{evalute_sequence_command.go => evaluate_sequence_command.go} (100%) rename examples/{leading-seperator.yaml => leading-separator.yaml} (100%) diff --git a/acceptance_tests/leading-seperator.sh b/acceptance_tests/leading-separator.sh similarity index 99% rename from acceptance_tests/leading-seperator.sh rename to acceptance_tests/leading-separator.sh index 80b1a154..9e36085f 100755 --- a/acceptance_tests/leading-seperator.sh +++ b/acceptance_tests/leading-separator.sh @@ -167,7 +167,7 @@ testLeadingSeparatorExtractFieldMultiDocWithComments() { --- # there a: test -# whereever +# wherever --- # you are a: test2 diff --git a/cmd/evalute_sequence_command.go b/cmd/evaluate_sequence_command.go similarity index 100% rename from cmd/evalute_sequence_command.go rename to cmd/evaluate_sequence_command.go diff --git a/examples/leading-seperator.yaml b/examples/leading-separator.yaml similarity index 100% rename from examples/leading-seperator.yaml rename to examples/leading-separator.yaml diff --git a/pkg/yqlib/candidate_node.go b/pkg/yqlib/candidate_node.go index 61675c8a..24a5d43f 100644 --- a/pkg/yqlib/candidate_node.go +++ b/pkg/yqlib/candidate_node.go @@ -111,7 +111,7 @@ func (n *CandidateNode) createChildPath(path interface{}) []interface{} { return newPath } - //don't use append as they may actually modify the path of the orignal node! + //don't use append as they may actually modify the path of the original node! newPath := make([]interface{}, len(n.Path)+1) copy(newPath, n.Path) newPath[len(n.Path)] = path diff --git a/pkg/yqlib/doc/usage/csv-tsv.md b/pkg/yqlib/doc/usage/csv-tsv.md index c58e42cd..70331a2b 100644 --- a/pkg/yqlib/doc/usage/csv-tsv.md +++ b/pkg/yqlib/doc/usage/csv-tsv.md @@ -2,7 +2,7 @@ Encode/Decode/Roundtrip CSV and TSV files. ## Encode -Currently supports arrays of homogenous flat objects, that is: no nesting and it assumes the _first_ object has all the keys required: +Currently supports arrays of homogeneous flat objects, that is: no nesting and it assumes the _first_ object has all the keys required: ```yaml - name: Bobo diff --git a/pkg/yqlib/doc/usage/headers/csv-tsv.md b/pkg/yqlib/doc/usage/headers/csv-tsv.md index 0a7c7cac..228d3f5c 100644 --- a/pkg/yqlib/doc/usage/headers/csv-tsv.md +++ b/pkg/yqlib/doc/usage/headers/csv-tsv.md @@ -2,7 +2,7 @@ Encode/Decode/Roundtrip CSV and TSV files. ## Encode -Currently supports arrays of homogenous flat objects, that is: no nesting and it assumes the _first_ object has all the keys required: +Currently supports arrays of homogeneous flat objects, that is: no nesting and it assumes the _first_ object has all the keys required: ```yaml - name: Bobo diff --git a/project-words.txt b/project-words.txt index 218a49b7..00afd650 100644 --- a/project-words.txt +++ b/project-words.txt @@ -161,7 +161,7 @@ Oneshot opencollect opstack orderedmap -orignal +original osarch overridign pacman @@ -239,7 +239,7 @@ Warningf Wazowski webi Webi -whereever +wherever winget withdots wizz diff --git a/release_notes.txt b/release_notes.txt index 46a6e692..ff0f939c 100644 --- a/release_notes.txt +++ b/release_notes.txt @@ -104,7 +104,7 @@ - Fixed sorting by date #1412 - Added check to ensure only maps can be encoded to XML #1408 - Check merge alias is a map #1425 - - Explicity setting unwrap flag works for json output #437, #1409 + - Explicitly setting unwrap flag works for json output #437, #1409 - Bumped go version diff --git a/scripts/shunit2 b/scripts/shunit2 index 6239683a..f15ec1f1 100755 --- a/scripts/shunit2 +++ b/scripts/shunit2 @@ -1306,7 +1306,7 @@ if command [ "$#" -ge 2 ]; then # Argument $1 is either the filename of tests or '--'; either way, skip it. shift # Remaining arguments ($2 .. $#) are assumed to be test function names. - # Interate through all remaining args in "$@" in a POSIX (likely portable) way. + # Iterate through all remaining args in "$@" in a POSIX (likely portable) way. # Helpful tip: https://unix.stackexchange.com/questions/314032/how-to-use-arguments-like-1-2-in-a-for-loop for _shunit_arg_ do suite_addTest "${_shunit_arg_}" From 0cf3adf5dd69d7b0f7903bf8d58f21354790e519 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Tue, 26 Sep 2023 14:43:08 +1000 Subject: [PATCH 37/43] Adding another xml test; spelling fixes --- debian/changelog | 2 +- how-it-works.md | 2 +- pkg/yqlib/doc/operators/pick.md | 4 ++-- pkg/yqlib/doc/usage/xml.md | 19 +++++++++++++++++++ pkg/yqlib/encoder_xml.go | 2 +- pkg/yqlib/encoder_yaml.go | 1 - pkg/yqlib/lexer.go | 2 +- pkg/yqlib/operator_pick_test.go | 12 ++++++------ pkg/yqlib/xml_test.go | 7 +++++++ project-words.txt | 6 ------ 10 files changed, 38 insertions(+), 19 deletions(-) diff --git a/debian/changelog b/debian/changelog index f0a4d512..e9ce1202 100644 --- a/debian/changelog +++ b/debian/changelog @@ -127,7 +127,7 @@ yq (4.9.6) focal; urgency=medium * Added darwin/arm64 build, thanks @alecthomas * Incremented docker alpine base version, thanks @da6d6i7-bronga - * Bug fix: multine expression + * Bug fix: multiline expression * Bug fix: special character -- Roberto Mier Escandon Tue, 29 Jun 2021 21:32:14 +0200 diff --git a/how-it-works.md b/how-it-works.md index 51a41fcd..f77a4cd9 100644 --- a/how-it-works.md +++ b/how-it-works.md @@ -29,7 +29,7 @@ It pipes the current, lets call it 'root' context through the `lhs` expression o cat ``` -Sidenote: this node holds not only its value 'cat', but comments and metadata too, including path and parent information. +Side note: this node holds not only its value 'cat', but comments and metadata too, including path and parent information. The `=` operator then pipes the 'root' context through the `rhs` expression of `.b` to return the node diff --git a/pkg/yqlib/doc/operators/pick.md b/pkg/yqlib/doc/operators/pick.md index 3b477699..fbe9bd95 100644 --- a/pkg/yqlib/doc/operators/pick.md +++ b/pkg/yqlib/doc/operators/pick.md @@ -13,7 +13,7 @@ myMap: cat: meow dog: bark thing: hamster - hamster: squeek + hamster: squeak ``` then ```bash @@ -22,7 +22,7 @@ yq '.myMap |= pick(["hamster", "cat", "goat"])' sample.yml will output ```yaml myMap: - hamster: squeek + hamster: squeak cat: meow ``` diff --git a/pkg/yqlib/doc/usage/xml.md b/pkg/yqlib/doc/usage/xml.md index 89bee794..2d469450 100644 --- a/pkg/yqlib/doc/usage/xml.md +++ b/pkg/yqlib/doc/usage/xml.md @@ -128,6 +128,25 @@ zoo: - cat ``` +## Parse xml: force all as an array +Because of the way yq works, when updating everything you need to update the children before the parents. By default `..` will match parents first, so we reverse that before updating. + +Given a sample.xml file of: +```xml +boing +``` +then +```bash +yq -oy '([..] | reverse | .[]) |= [] + .' sample.xml +``` +will output +```yaml +- zoo: + - thing: + - frog: + - boing +``` + ## Parse xml: attributes Attributes are converted to fields, with the default attribute prefix '+'. Use '--xml-attribute-prefix` to set your own. diff --git a/pkg/yqlib/encoder_xml.go b/pkg/yqlib/encoder_xml.go index 345e825e..2e48bd99 100644 --- a/pkg/yqlib/encoder_xml.go +++ b/pkg/yqlib/encoder_xml.go @@ -233,7 +233,7 @@ func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) erro commentStr = chompRegexp.ReplaceAllString(commentStr, "") log.Debugf("chompRegexp [%v]", commentStr) commentStr = xmlEncodeMultilineCommentRegex.ReplaceAllString(commentStr, "$1$2") - log.Debugf("processed multine [%v]", commentStr) + log.Debugf("processed multiline [%v]", commentStr) // if the first line is non blank, add a space if commentStr[0] != '\n' && commentStr[0] != ' ' { commentStr = " " + commentStr diff --git a/pkg/yqlib/encoder_yaml.go b/pkg/yqlib/encoder_yaml.go index 75b64c04..e2a9943c 100644 --- a/pkg/yqlib/encoder_yaml.go +++ b/pkg/yqlib/encoder_yaml.go @@ -38,7 +38,6 @@ func (ye *yamlEncoder) PrintDocumentSeparator(writer io.Writer) error { } func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) error { - // log.Debug("headcommentwas [%v]", content) reader := bufio.NewReader(strings.NewReader(content)) for { diff --git a/pkg/yqlib/lexer.go b/pkg/yqlib/lexer.go index 259dbd33..d118fa0f 100644 --- a/pkg/yqlib/lexer.go +++ b/pkg/yqlib/lexer.go @@ -175,7 +175,7 @@ func handleToken(tokens []*token, index int, postProcessedTokens []*token) (toke if index != len(tokens)-1 && currentToken.CheckForPostTraverse && tokens[index+1].TokenType == openCollect { - log.Debug(" adding traverArray because next is opencollect") + log.Debug(" adding traverseArray because next is opencollect") op := &Operation{OperationType: traverseArrayOpType} postProcessedTokens = append(postProcessedTokens, &token{TokenType: operationToken, Operation: op}) } diff --git a/pkg/yqlib/operator_pick_test.go b/pkg/yqlib/operator_pick_test.go index 25542385..fe92f3ad 100644 --- a/pkg/yqlib/operator_pick_test.go +++ b/pkg/yqlib/operator_pick_test.go @@ -8,28 +8,28 @@ var pickOperatorScenarios = []expressionScenario{ { description: "Pick keys from map", subdescription: "Note that the order of the keys matches the pick order and non existent keys are skipped.", - document: "myMap: {cat: meow, dog: bark, thing: hamster, hamster: squeek}\n", + document: "myMap: {cat: meow, dog: bark, thing: hamster, hamster: squeak}\n", expression: `.myMap |= pick(["hamster", "cat", "goat"])`, expected: []string{ - "D0, P[], (doc)::myMap: {hamster: squeek, cat: meow}\n", + "D0, P[], (doc)::myMap: {hamster: squeak, cat: meow}\n", }, }, { description: "Pick keys from map", skipDoc: true, - document: "!things myMap: {cat: meow, dog: bark, thing: hamster, hamster: squeek}\n", + document: "!things myMap: {cat: meow, dog: bark, thing: hamster, hamster: squeak}\n", expression: `.myMap |= pick(["hamster", "cat", "goat"])`, expected: []string{ - "D0, P[], (doc)::!things myMap: {hamster: squeek, cat: meow}\n", + "D0, P[], (doc)::!things myMap: {hamster: squeak, cat: meow}\n", }, }, { description: "Pick keys from map with comments", skipDoc: true, - document: "# abc\nmyMap: {cat: meow, dog: bark, thing: hamster, hamster: squeek}\n# xyz\n", + document: "# abc\nmyMap: {cat: meow, dog: bark, thing: hamster, hamster: squeak}\n# xyz\n", expression: `.myMap |= pick(["hamster", "cat", "goat"])`, expected: []string{ - "D0, P[], (doc)::# abc\nmyMap: {hamster: squeek, cat: meow}\n# xyz\n", + "D0, P[], (doc)::# abc\nmyMap: {hamster: squeak, cat: meow}\n# xyz\n", }, }, { diff --git a/pkg/yqlib/xml_test.go b/pkg/yqlib/xml_test.go index fdf50b98..0f74ab10 100644 --- a/pkg/yqlib/xml_test.go +++ b/pkg/yqlib/xml_test.go @@ -363,6 +363,13 @@ var xmlScenarios = []formatScenario{ expression: ".zoo.animal |= ([] + .)", expected: "zoo:\n animal:\n - cat\n", }, + { + description: "Parse xml: force all as an array", + subdescription: "Because of the way yq works, when updating everything you need to update the children before the parents. By default `..` will match parents first, so we reverse that before updating.", + input: "boing", + expression: "([..] | reverse | .[]) |= [] + .", + expected: "- zoo:\n - thing:\n - frog:\n - boing\n", + }, { description: "Parse xml: attributes", subdescription: "Attributes are converted to fields, with the default attribute prefix '+'. Use '--xml-attribute-prefix` to set your own.", diff --git a/project-words.txt b/project-words.txt index 00afd650..cbbad841 100644 --- a/project-words.txt +++ b/project-words.txt @@ -96,7 +96,6 @@ gota goversion GOVERSION haha -headcommentwas hellno herbygillot hexdump @@ -138,7 +137,6 @@ mitchellh mktemp multidoc multimaint -multine myenv myenvnonexisting myfile @@ -161,7 +159,6 @@ Oneshot opencollect opstack orderedmap -original osarch overridign pacman @@ -199,11 +196,9 @@ shellvars shortfunc shortpipe shunit -Sidenote snapcraft somevalue splt -squeek srcdir stackoverflow stiched @@ -227,7 +222,6 @@ timezones Timezones tojson Tokenvalue -traver tsvd Tuan tzdata From 9f355b93a21128eb1df582b2cf5e651ac49a5999 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Tue, 26 Sep 2023 14:46:44 +1000 Subject: [PATCH 38/43] Release notes --- release_notes.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/release_notes.txt b/release_notes.txt index ff0f939c..114330be 100644 --- a/release_notes.txt +++ b/release_notes.txt @@ -1,3 +1,10 @@ +4.35.2: + - Fix various typos #1798 + - Fixed number parsing as float bug in JSON #1756 + - Fixed string, null concatenation consistency #1712 + - Fixed expression parsing issue #1711 + - Bumped dependencies + 4.35.1: - Added Lua output support (Thanks @Zash)! - Added BSD checksum format (Thanks @viq)! From a198f72367ce9da70b564a2cc25399de8e27bf37 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Tue, 26 Sep 2023 14:48:52 +1000 Subject: [PATCH 39/43] Bumping version --- cmd/version.go | 2 +- snap/snapcraft.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/version.go b/cmd/version.go index 6a75038e..b72e2f7f 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -11,7 +11,7 @@ var ( GitDescribe string // Version is main version number that is being run at the moment. - Version = "v4.35.1" + Version = "v4.35.2" // VersionPrerelease is a pre-release marker for the version. If this is "" (empty string) // then it means that it is a final release. Otherwise, this is a pre-release diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index b6a55343..9ac286f2 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -1,5 +1,5 @@ name: yq -version: 'v4.35.1' +version: 'v4.35.2' summary: A lightweight and portable command-line YAML processor description: | The aim of the project is to be the jq or sed of yaml files. From ee900ec9975bc315dd16c037b9cec7c9821f0b31 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Mon, 2 Oct 2023 14:43:12 +1100 Subject: [PATCH 40/43] Added kind operator --- pkg/yqlib/doc/operators/kind.md | 77 +++++++++++++++++++++++++++++++++ pkg/yqlib/lexer_participle.go | 1 + pkg/yqlib/lib.go | 1 + pkg/yqlib/operator_kind.go | 39 +++++++++++++++++ pkg/yqlib/operator_kind_test.go | 49 +++++++++++++++++++++ 5 files changed, 167 insertions(+) create mode 100644 pkg/yqlib/doc/operators/kind.md create mode 100644 pkg/yqlib/operator_kind.go create mode 100644 pkg/yqlib/operator_kind_test.go diff --git a/pkg/yqlib/doc/operators/kind.md b/pkg/yqlib/doc/operators/kind.md new file mode 100644 index 00000000..1f65dfbb --- /dev/null +++ b/pkg/yqlib/doc/operators/kind.md @@ -0,0 +1,77 @@ + +## Get kind +Given a sample.yml file of: +```yaml +a: cat +b: 5 +c: 3.2 +e: true +f: [] +g: {} +h: null +``` +then +```bash +yq '.. | kind' sample.yml +``` +will output +```yaml +map +scalar +scalar +scalar +scalar +seq +map +scalar +``` + +## Get kind, ignores custom tags +Unlike tag, kind is not affected by custom tags. + +Given a sample.yml file of: +```yaml +a: !!thing cat +b: !!foo {} +c: !!bar [] +``` +then +```bash +yq '.. | kind' sample.yml +``` +will output +```yaml +map +scalar +map +seq +``` + +## Add comments only to scalars +An example of how you can use kind + +Given a sample.yml file of: +```yaml +a: + b: 5 + c: 3.2 +e: true +f: [] +g: {} +h: null +``` +then +```bash +yq '(.. | select(kind == "scalar")) line_comment = "this is a scalar"' sample.yml +``` +will output +```yaml +a: + b: 5 # this is a scalar + c: 3.2 # this is a scalar +e: true # this is a scalar +f: [] +g: {} +h: null # this is a scalar +``` + diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go index 2352354a..4388752b 100644 --- a/pkg/yqlib/lexer_participle.go +++ b/pkg/yqlib/lexer_participle.go @@ -152,6 +152,7 @@ var participleYqRules = []*participleYqRule{ assignableOp("style", getStyleOpType, assignStyleOpType), assignableOp("tag|type", getTagOpType, assignTagOpType), + simpleOp("kind", getKindOpType), assignableOp("anchor", getAnchorOpType, assignAnchorOpType), assignableOp("alias", getAliasOpType, assignAliasOpType), diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 5fc503ae..756c161e 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -120,6 +120,7 @@ var splitDocumentOpType = &operationType{Type: "SPLIT_DOC", NumArgs: 0, Preceden var getVariableOpType = &operationType{Type: "GET_VARIABLE", NumArgs: 0, Precedence: 55, Handler: getVariableOperator} var getStyleOpType = &operationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: getStyleOperator} var getTagOpType = &operationType{Type: "GET_TAG", NumArgs: 0, Precedence: 50, Handler: getTagOperator} +var getKindOpType = &operationType{Type: "GET_KIND", NumArgs: 0, Precedence: 50, Handler: getKindOperator} var getKeyOpType = &operationType{Type: "GET_KEY", NumArgs: 0, Precedence: 50, Handler: getKeyOperator} var isKeyOpType = &operationType{Type: "IS_KEY", NumArgs: 0, Precedence: 50, Handler: isKeyOperator} diff --git a/pkg/yqlib/operator_kind.go b/pkg/yqlib/operator_kind.go new file mode 100644 index 00000000..da0892c7 --- /dev/null +++ b/pkg/yqlib/operator_kind.go @@ -0,0 +1,39 @@ +package yqlib + +import ( + "container/list" + + yaml "gopkg.in/yaml.v3" +) + +func kindToText(kind yaml.Kind) string { + switch kind { + case yaml.MappingNode: + return "map" + case yaml.SequenceNode: + return "seq" + case yaml.DocumentNode: + return "doc" + case yaml.ScalarNode: + return "scalar" + case yaml.AliasNode: + return "alias" + default: + return "unknown" + } +} + +func getKindOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { + log.Debugf("GetKindOperator") + + var results = list.New() + + for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + node := &yaml.Node{Kind: yaml.ScalarNode, Value: kindToText(candidate.Node.Kind), Tag: "!!str"} + result := candidate.CreateReplacement(node) + results.PushBack(result) + } + + return context.ChildContext(results), nil +} diff --git a/pkg/yqlib/operator_kind_test.go b/pkg/yqlib/operator_kind_test.go new file mode 100644 index 00000000..fadf69d8 --- /dev/null +++ b/pkg/yqlib/operator_kind_test.go @@ -0,0 +1,49 @@ +package yqlib + +import ( + "testing" +) + +var kindOperatorScenarios = []expressionScenario{ + { + description: "Get kind", + document: `{a: cat, b: 5, c: 3.2, e: true, f: [], g: {}, h: null}`, + expression: `.. | kind`, + expected: []string{ + "D0, P[], (!!str)::map\n", + "D0, P[a], (!!str)::scalar\n", + "D0, P[b], (!!str)::scalar\n", + "D0, P[c], (!!str)::scalar\n", + "D0, P[e], (!!str)::scalar\n", + "D0, P[f], (!!str)::seq\n", + "D0, P[g], (!!str)::map\n", + "D0, P[h], (!!str)::scalar\n", + }, + }, + { + description: "Get kind, ignores custom tags", + subdescription: "Unlike tag, kind is not affected by custom tags.", + document: `{a: !!thing cat, b: !!foo {}, c: !!bar []}`, + expression: `.. | kind`, + expected: []string{ + "D0, P[], (!!str)::map\n", + "D0, P[a], (!!str)::scalar\n", + "D0, P[b], (!!str)::map\n", + "D0, P[c], (!!str)::seq\n", + }, + }, + { + description: "Add comments only to scalars", + subdescription: "An example of how you can use kind", + document: "a:\n b: 5\n c: 3.2\ne: true\nf: []\ng: {}\nh: null", + expression: `(.. | select(kind == "scalar")) line_comment = "this is a scalar"`, + expected: []string{"D0, P[], (!!map)::a:\n b: 5 # this is a scalar\n c: 3.2 # this is a scalar\ne: true # this is a scalar\nf: []\ng: {}\nh: null # this is a scalar\n"}, + }, +} + +func TestKindOperatorScenarios(t *testing.T) { + for _, tt := range kindOperatorScenarios { + testScenario(t, &tt) + } + documentOperatorScenarios(t, "kind", kindOperatorScenarios) +} From 5fa41624c9fed5f2ae302838e3f065c4126c374a Mon Sep 17 00:00:00 2001 From: Kim Alvefur Date: Tue, 3 Oct 2023 07:00:51 +0200 Subject: [PATCH 41/43] Add support for Lua input (#1810) --- cmd/utils.go | 2 + go.mod | 1 + go.sum | 2 + pkg/yqlib/decoder.go | 3 + pkg/yqlib/decoder_lua.go | 167 +++++++++++++++++++++++++++++++++++++ pkg/yqlib/doc/usage/lua.md | 30 ++++++- pkg/yqlib/lua_test.go | 95 ++++++++++++++++++++- project-words.txt | 3 + 8 files changed, 301 insertions(+), 2 deletions(-) create mode 100644 pkg/yqlib/decoder_lua.go diff --git a/cmd/utils.go b/cmd/utils.go index d1d813fc..331a65e5 100644 --- a/cmd/utils.go +++ b/cmd/utils.go @@ -130,6 +130,8 @@ func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) { func createDecoder(format yqlib.InputFormat, evaluateTogether bool) (yqlib.Decoder, error) { switch format { + case yqlib.LuaInputFormat: + return yqlib.NewLuaDecoder(yqlib.ConfiguredLuaPreferences), nil case yqlib.XMLInputFormat: return yqlib.NewXMLDecoder(yqlib.ConfiguredXMLPreferences), nil case yqlib.PropertiesInputFormat: diff --git a/go.mod b/go.mod index ef06c29b..cd5a5ac2 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 + github.com/yuin/gopher-lua v1.1.0 golang.org/x/net v0.15.0 golang.org/x/text v0.13.0 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 diff --git a/go.sum b/go.sum index 825e4134..061c4f2c 100644 --- a/go.sum +++ b/go.sum @@ -50,6 +50,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE= +github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/pkg/yqlib/decoder.go b/pkg/yqlib/decoder.go index af5d3ecf..284b4316 100644 --- a/pkg/yqlib/decoder.go +++ b/pkg/yqlib/decoder.go @@ -18,6 +18,7 @@ const ( TSVObjectInputFormat TomlInputFormat UriInputFormat + LuaInputFormat ) type Decoder interface { @@ -41,6 +42,8 @@ func InputFormatFromString(format string) (InputFormat, error) { return TSVObjectInputFormat, nil case "toml": return TomlInputFormat, nil + case "lua", "l": + return LuaInputFormat, nil default: return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml|toml]", format) } diff --git a/pkg/yqlib/decoder_lua.go b/pkg/yqlib/decoder_lua.go new file mode 100644 index 00000000..5db879f4 --- /dev/null +++ b/pkg/yqlib/decoder_lua.go @@ -0,0 +1,167 @@ +package yqlib + +import ( + "fmt" + "io" + "math" + + lua "github.com/yuin/gopher-lua" + yaml "gopkg.in/yaml.v3" +) + +type luaDecoder struct { + reader io.Reader + finished bool + prefs LuaPreferences +} + +func NewLuaDecoder(prefs LuaPreferences) Decoder { + return &luaDecoder{ + prefs: prefs, + } +} + +func (dec *luaDecoder) Init(reader io.Reader) error { + dec.reader = reader + return nil +} + +func (dec *luaDecoder) convertToYamlNode(ls *lua.LState, lv lua.LValue) *yaml.Node { + switch lv.Type() { + case lua.LTNil: + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!null", + Value: "", + } + case lua.LTBool: + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!bool", + Value: lv.String(), + } + case lua.LTNumber: + n := float64(lua.LVAsNumber(lv)) + // various special case floats + if math.IsNaN(n) { + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!float", + Value: ".nan", + } + } + if math.IsInf(n, 1) { + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!float", + Value: ".inf", + } + } + if math.IsInf(n, -1) { + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!float", + Value: "-.inf", + } + } + + // does it look like an integer? + if n == float64(int(n)) { + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!int", + Value: lv.String(), + } + } + + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!float", + Value: lv.String(), + } + case lua.LTString: + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: lv.String(), + } + case lua.LTFunction: + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "tag:lua.org,2006,function", + Value: lv.String(), + } + case lua.LTTable: + // Simultaneously create a sequence and a map, pick which one to return + // based on whether all keys were consecutive integers + i := 1 + yaml_sequence := &yaml.Node{ + Kind: yaml.SequenceNode, + Tag: "!!seq", + } + yaml_map := &yaml.Node{ + Kind: yaml.MappingNode, + Tag: "!!map", + } + t := lv.(*lua.LTable) + k, v := ls.Next(t, lua.LNil) + for k != lua.LNil { + if ki, ok := k.(lua.LNumber); i != 0 && ok && math.Mod(float64(ki), 1) == 0 && int(ki) == i { + i++ + } else { + i = 0 + } + yaml_map.Content = append(yaml_map.Content, dec.convertToYamlNode(ls, k)) + yv := dec.convertToYamlNode(ls, v) + yaml_map.Content = append(yaml_map.Content, yv) + if i != 0 { + yaml_sequence.Content = append(yaml_sequence.Content, yv) + } + k, v = ls.Next(t, k) + } + if i != 0 { + return yaml_sequence + } + return yaml_map + default: + return &yaml.Node{ + Kind: yaml.ScalarNode, + LineComment: fmt.Sprintf("Unhandled Lua type: %s", lv.Type().String()), + Tag: "!!null", + Value: lv.String(), + } + } +} + +func (dec *luaDecoder) decideTopLevelNode(ls *lua.LState) *yaml.Node { + if ls.GetTop() == 0 { + // no items were explicitly returned, encode the globals table instead + return dec.convertToYamlNode(ls, ls.Get(lua.GlobalsIndex)) + } + return dec.convertToYamlNode(ls, ls.Get(1)) +} + +func (dec *luaDecoder) Decode() (*CandidateNode, error) { + if dec.finished { + return nil, io.EOF + } + ls := lua.NewState(lua.Options{SkipOpenLibs: true}) + defer ls.Close() + fn, err := ls.Load(dec.reader, "@input") + if err != nil { + return nil, err + } + ls.Push(fn) + err = ls.PCall(0, lua.MultRet, nil) + if err != nil { + return nil, err + } + firstNode := dec.decideTopLevelNode(ls) + dec.finished = true + return &CandidateNode{ + Node: &yaml.Node{ + Kind: yaml.DocumentNode, + Content: []*yaml.Node{firstNode}, + }, + }, nil +} diff --git a/pkg/yqlib/doc/usage/lua.md b/pkg/yqlib/doc/usage/lua.md index 61458be2..d6f067e4 100644 --- a/pkg/yqlib/doc/usage/lua.md +++ b/pkg/yqlib/doc/usage/lua.md @@ -1,5 +1,33 @@ -## Basic example +## Basic input example +Given a sample.lua file of: +```lua +return { + ["country"] = "Australia"; -- this place + ["cities"] = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", + }; +}; + +``` +then +```bash +yq -oy '.' sample.lua +``` +will output +```yaml +country: Australia +cities: + - Sydney + - Melbourne + - Brisbane + - Perth +``` + +## Basic output example Given a sample.yml file of: ```yaml --- diff --git a/pkg/yqlib/lua_test.go b/pkg/yqlib/lua_test.go index 3f8133d6..4089308f 100644 --- a/pkg/yqlib/lua_test.go +++ b/pkg/yqlib/lua_test.go @@ -10,7 +10,27 @@ import ( var luaScenarios = []formatScenario{ { - description: "Basic example", + description: "Basic input example", + input: `return { + ["country"] = "Australia"; -- this place + ["cities"] = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", + }; +}; +`, + expected: `country: Australia +cities: + - Sydney + - Melbourne + - Brisbane + - Perth +`, + }, + { + description: "Basic output example", scenarioType: "encode", input: `--- country: Australia # this place @@ -28,6 +48,31 @@ cities: "Perth", }; }; +`, + }, + { + description: "Basic roundtrip", + skipDoc: true, + scenarioType: "roundtrip", + input: `return { + ["country"] = "Australia"; -- this place + ["cities"] = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", + }; +}; +`, + expected: `return { + ["country"] = "Australia"; + ["cities"] = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", + }; +}; `, }, { @@ -168,8 +213,12 @@ numbers: func testLuaScenario(t *testing.T, s formatScenario) { switch s.scenarioType { + case "", "decode": + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewYamlEncoder(4, false, ConfiguredYamlPreferences)), s.description) case "encode": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(ConfiguredLuaPreferences)), s.description) + case "roundtrip": + test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewLuaEncoder(ConfiguredLuaPreferences)), s.description) case "unquoted-encode": test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(LuaPreferences{ DocPrefix: "return ", @@ -196,13 +245,39 @@ func documentLuaScenario(t *testing.T, w *bufio.Writer, i interface{}) { return } switch s.scenarioType { + case "", "decode": + documentLuaDecodeScenario(w, s) case "encode", "unquoted-encode", "globals-encode": documentLuaEncodeScenario(w, s) + case "roundtrip": + documentLuaRoundTripScenario(w, s) default: panic(fmt.Sprintf("unhandled scenario type %q", s.scenarioType)) } } +func documentLuaDecodeScenario(w *bufio.Writer, s formatScenario) { + writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) + + if s.subdescription != "" { + writeOrPanic(w, s.subdescription) + writeOrPanic(w, "\n\n") + } + + writeOrPanic(w, "Given a sample.lua file of:\n") + writeOrPanic(w, fmt.Sprintf("```lua\n%v\n```\n", s.input)) + + writeOrPanic(w, "then\n") + expression := s.expression + if expression == "" { + expression = "." + } + writeOrPanic(w, fmt.Sprintf("```bash\nyq -oy '%v' sample.lua\n```\n", expression)) + writeOrPanic(w, "will output\n") + + writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewYamlEncoder(2, false, ConfiguredYamlPreferences)))) +} + func documentLuaEncodeScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) @@ -245,6 +320,24 @@ func documentLuaEncodeScenario(w *bufio.Writer, s formatScenario) { writeOrPanic(w, fmt.Sprintf("```lua\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(prefs)))) } +func documentLuaRoundTripScenario(w *bufio.Writer, s formatScenario) { + writeOrPanic(w, fmt.Sprintf("## %v\n", s.description)) + + if s.subdescription != "" { + writeOrPanic(w, s.subdescription) + writeOrPanic(w, "\n\n") + } + + writeOrPanic(w, "Given a sample.lua file of:\n") + writeOrPanic(w, fmt.Sprintf("```lua\n%v\n```\n", s.input)) + + writeOrPanic(w, "then\n") + writeOrPanic(w, "```bash\nyq '.' sample.lua\n```\n") + writeOrPanic(w, "will output\n") + + writeOrPanic(w, fmt.Sprintf("```lua\n%v```\n\n", mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewLuaEncoder(ConfiguredLuaPreferences)))) +} + func TestLuaScenarios(t *testing.T) { for _, tt := range luaScenarios { testLuaScenario(t, tt) diff --git a/project-words.txt b/project-words.txt index cbbad841..cbea0767 100644 --- a/project-words.txt +++ b/project-words.txt @@ -126,6 +126,7 @@ Lexer libdistro lindex linecomment +LVAs magiconair mapvalues Mier @@ -135,6 +136,7 @@ minishift mipsle mitchellh mktemp +Mult multidoc multimaint myenv @@ -244,4 +246,5 @@ xmld xyzzy yamld yqlib +yuin zabbix From 6e65d44a980bbf791c5164e22dc4b290262689a9 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Tue, 3 Oct 2023 16:22:23 +1100 Subject: [PATCH 42/43] Can build yq with no lua --- examples/data.lua | 9 +++++++++ pkg/yqlib/decoder_lua.go | 2 ++ pkg/yqlib/encoder_lua.go | 2 ++ pkg/yqlib/lua_test.go | 5 ++++- pkg/yqlib/no_lua.go | 11 +++++++++++ project-words.txt | 1 + scripts/build-small-yq.sh | 2 +- 7 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 examples/data.lua create mode 100644 pkg/yqlib/no_lua.go diff --git a/examples/data.lua b/examples/data.lua new file mode 100644 index 00000000..c4051e9c --- /dev/null +++ b/examples/data.lua @@ -0,0 +1,9 @@ +return { + ["country"] = "Australia"; -- this place + ["cities"] = { + "Sydney", + "Melbourne", + "Brisbane", + "Perth", + }; +}; \ No newline at end of file diff --git a/pkg/yqlib/decoder_lua.go b/pkg/yqlib/decoder_lua.go index 5db879f4..efc04eac 100644 --- a/pkg/yqlib/decoder_lua.go +++ b/pkg/yqlib/decoder_lua.go @@ -1,3 +1,5 @@ +//go:build !yq_nolua + package yqlib import ( diff --git a/pkg/yqlib/encoder_lua.go b/pkg/yqlib/encoder_lua.go index a74c68cb..d1bb81aa 100644 --- a/pkg/yqlib/encoder_lua.go +++ b/pkg/yqlib/encoder_lua.go @@ -1,3 +1,5 @@ +//go:build !yq_nolua + package yqlib import ( diff --git a/pkg/yqlib/lua_test.go b/pkg/yqlib/lua_test.go index 4089308f..ddcbd023 100644 --- a/pkg/yqlib/lua_test.go +++ b/pkg/yqlib/lua_test.go @@ -1,3 +1,5 @@ +//go:build !yq_nolua + package yqlib import ( @@ -54,6 +56,7 @@ cities: description: "Basic roundtrip", skipDoc: true, scenarioType: "roundtrip", + expression: `.cities[0] = "Adelaide"`, input: `return { ["country"] = "Australia"; -- this place ["cities"] = { @@ -67,7 +70,7 @@ cities: expected: `return { ["country"] = "Australia"; ["cities"] = { - "Sydney", + "Adelaide", "Melbourne", "Brisbane", "Perth", diff --git a/pkg/yqlib/no_lua.go b/pkg/yqlib/no_lua.go new file mode 100644 index 00000000..05368eff --- /dev/null +++ b/pkg/yqlib/no_lua.go @@ -0,0 +1,11 @@ +//go:build yq_nolua + +package yqlib + +func NewLuaEncoder(prefs LuaPreferences) Encoder { + return nil +} + +func NewLuaDecoder(prefs LuaPreferences) Decoder { + return nil +} diff --git a/project-words.txt b/project-words.txt index cbea0767..ab50a416 100644 --- a/project-words.txt +++ b/project-words.txt @@ -155,6 +155,7 @@ Nonquoting nosec notoml noxml +nolua nullinput onea Oneshot diff --git a/scripts/build-small-yq.sh b/scripts/build-small-yq.sh index 81c78c11..9ecfddab 100755 --- a/scripts/build-small-yq.sh +++ b/scripts/build-small-yq.sh @@ -1,2 +1,2 @@ #!/bin/bash -go build -tags yq_notoml -tags yq_noxml -tags yq_nojson -ldflags "-s -w" . \ No newline at end of file +go build -tags "yq_nolua yq_notoml yq_noxml yq_nojson" -ldflags "-s -w" . \ No newline at end of file From d113344abfea8c08447111f9c362eba49a55c53a Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Thu, 5 Oct 2023 15:13:46 +1100 Subject: [PATCH 43/43] Added tonumber support #71 --- pkg/yqlib/doc/operators/headers/to_number.md | 2 + pkg/yqlib/doc/operators/to_number.md | 49 +++++++++++++++++ pkg/yqlib/lexer_participle.go | 1 + pkg/yqlib/lib.go | 1 + pkg/yqlib/operator_to_number.go | 56 ++++++++++++++++++++ pkg/yqlib/operator_to_number_test.go | 51 ++++++++++++++++++ project-words.txt | 1 + 7 files changed, 161 insertions(+) create mode 100644 pkg/yqlib/doc/operators/headers/to_number.md create mode 100644 pkg/yqlib/doc/operators/to_number.md create mode 100644 pkg/yqlib/operator_to_number.go create mode 100644 pkg/yqlib/operator_to_number_test.go diff --git a/pkg/yqlib/doc/operators/headers/to_number.md b/pkg/yqlib/doc/operators/headers/to_number.md new file mode 100644 index 00000000..edcb9bd1 --- /dev/null +++ b/pkg/yqlib/doc/operators/headers/to_number.md @@ -0,0 +1,2 @@ +# To Number +Parses the input as a number. yq will try to parse values as an int first, failing that it will try float. Values that already ints or floats will be left alone. diff --git a/pkg/yqlib/doc/operators/to_number.md b/pkg/yqlib/doc/operators/to_number.md new file mode 100644 index 00000000..2e329033 --- /dev/null +++ b/pkg/yqlib/doc/operators/to_number.md @@ -0,0 +1,49 @@ +# To Number +Parses the input as a number. yq will try to parse values as an int first, failing that it will try float. Values that already ints or floats will be left alone. + +## Converts strings to numbers +Given a sample.yml file of: +```yaml +- "3" +- "3.1" +- "-1e3" +``` +then +```bash +yq '.[] | to_number' sample.yml +``` +will output +```yaml +3 +3.1 +-1e3 +``` + +## Doesn't change numbers +Given a sample.yml file of: +```yaml +- 3 +- 3.1 +- -1e3 +``` +then +```bash +yq '.[] | to_number' sample.yml +``` +will output +```yaml +3 +3.1 +-1e3 +``` + +## Cannot convert null +Running +```bash +yq --null-input '.a.b | to_number' +``` +will output +```bash +Error: cannot convert node value [null] at path a.b of tag !!null to number +``` + diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go index 4388752b..f2184a43 100644 --- a/pkg/yqlib/lexer_participle.go +++ b/pkg/yqlib/lexer_participle.go @@ -34,6 +34,7 @@ var participleYqRules = []*participleYqRule{ simpleOp("line", lineOpType), simpleOp("column", columnOpType), simpleOp("eval", evalOpType), + simpleOp("to_?number", toNumberOpType), {"MapValues", `map_?values`, opToken(mapValuesOpType), 0}, simpleOp("map", mapOpType), diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go index 756c161e..b88b9e2d 100644 --- a/pkg/yqlib/lib.go +++ b/pkg/yqlib/lib.go @@ -167,6 +167,7 @@ var valueOpType = &operationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Hand var referenceOpType = &operationType{Type: "REF", NumArgs: 0, Precedence: 50, Handler: referenceOperator} var envOpType = &operationType{Type: "ENV", NumArgs: 0, Precedence: 50, Handler: envOperator} var notOpType = &operationType{Type: "NOT", NumArgs: 0, Precedence: 50, Handler: notOperator} +var toNumberOpType = &operationType{Type: "TO_NUMBER", NumArgs: 0, Precedence: 50, Handler: toNumberOperator} var emptyOpType = &operationType{Type: "EMPTY", Precedence: 50, Handler: emptyOperator} var envsubstOpType = &operationType{Type: "ENVSUBST", NumArgs: 0, Precedence: 50, Handler: envsubstOperator} diff --git a/pkg/yqlib/operator_to_number.go b/pkg/yqlib/operator_to_number.go new file mode 100644 index 00000000..6c97d4ed --- /dev/null +++ b/pkg/yqlib/operator_to_number.go @@ -0,0 +1,56 @@ +package yqlib + +import ( + "container/list" + "fmt" + "strconv" + + yaml "gopkg.in/yaml.v3" +) + +func tryConvertToNumber(value string) (string, bool) { + // try a int first + _, _, err := parseInt64(value) + if err == nil { + return "!!int", true + } + // try float + _, floatErr := strconv.ParseFloat(value, 64) + + if floatErr == nil { + return "!!float", true + } + return "", false + +} + +func toNumberOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { + log.Debugf("ToNumberOperator") + + var results = list.New() + + for el := context.MatchingNodes.Front(); el != nil; el = el.Next() { + candidate := el.Value.(*CandidateNode) + if candidate.Node.Kind != yaml.ScalarNode { + return Context{}, fmt.Errorf("cannot convert node at path %v of tag %v to number", candidate.GetNicePath(), candidate.GetNiceTag()) + } + + if candidate.Node.Tag == "!!int" || candidate.Node.Tag == "!!float" { + // it already is a number! + results.PushBack(candidate) + } else { + tag, converted := tryConvertToNumber(candidate.Node.Value) + if converted { + node := &yaml.Node{Kind: yaml.ScalarNode, Value: candidate.Node.Value, Tag: tag} + + result := candidate.CreateReplacement(node) + results.PushBack(result) + } else { + return Context{}, fmt.Errorf("cannot convert node value [%v] at path %v of tag %v to number", candidate.Node.Value, candidate.GetNicePath(), candidate.GetNiceTag()) + } + + } + } + + return context.ChildContext(results), nil +} diff --git a/pkg/yqlib/operator_to_number_test.go b/pkg/yqlib/operator_to_number_test.go new file mode 100644 index 00000000..00fe576b --- /dev/null +++ b/pkg/yqlib/operator_to_number_test.go @@ -0,0 +1,51 @@ +package yqlib + +import ( + "testing" +) + +var toNumberScenarios = []expressionScenario{ + { + description: "Converts strings to numbers", + document: `["3", "3.1", "-1e3"]`, + expression: `.[] | to_number`, + expected: []string{ + "D0, P[0], (!!int)::3\n", + "D0, P[1], (!!float)::3.1\n", + "D0, P[2], (!!float)::-1e3\n", + }, + }, + { + skipDoc: true, + description: "Converts strings to numbers, with tonumber because jq", + document: `["3", "3.1", "-1e3"]`, + expression: `.[] | tonumber`, + expected: []string{ + "D0, P[0], (!!int)::3\n", + "D0, P[1], (!!float)::3.1\n", + "D0, P[2], (!!float)::-1e3\n", + }, + }, + { + description: "Doesn't change numbers", + document: `[3, 3.1, -1e3]`, + expression: `.[] | to_number`, + expected: []string{ + "D0, P[0], (!!int)::3\n", + "D0, P[1], (!!float)::3.1\n", + "D0, P[2], (!!float)::-1e3\n", + }, + }, + { + description: "Cannot convert null", + expression: `.a.b | to_number`, + expectedError: "cannot convert node value [null] at path a.b of tag !!null to number", + }, +} + +func TestToNumberOperatorScenarios(t *testing.T) { + for _, tt := range toNumberScenarios { + testScenario(t, &tt) + } + documentOperatorScenarios(t, "to_number", toNumberScenarios) +} diff --git a/project-words.txt b/project-words.txt index ab50a416..2371c1fb 100644 --- a/project-words.txt +++ b/project-words.txt @@ -249,3 +249,4 @@ yamld yqlib yuin zabbix +tonumber \ No newline at end of file