Merge branch 'master' into xml-proc-inst

.github/workflows/docker-release.yml

@@ -60,3 +60,23 @@ jobs:
             -t "${IMAGE_NAME}:4" \
             -t "${IMAGE_NAME}:latest" \
             .
+          
+          cd github-action
+          docker buildx build \
+            --label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
+            --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
+            --label "org.opencontainers.image.description=yq is a portable command-line YAML processor" \
+            --label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
+            --label "org.opencontainers.image.licenses=MIT" \
+            --label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \
+            --label "org.opencontainers.image.source=https://github.com/mikefarah/yq" \
+            --label "org.opencontainers.image.title=yq" \
+            --label "org.opencontainers.image.url=https://mikefarah.gitbook.io/yq/" \
+            --label "org.opencontainers.image.version=${IMAGE_VERSION}" \
+            --platform "${PLATFORMS}" \
+            --pull \
+            --push \
+            -t "${IMAGE_NAME}:${IMAGE_VERSION}-githubaction" \
+            -t "${IMAGE_NAME}:4-githubaction" \
+            -t "${IMAGE_NAME}:latest-githubaction" \
+            .

.github/workflows/release.yml

@@ -13,6 +13,7 @@ jobs:
       - uses: actions/setup-go@v3
         with:
           go-version: '^1.19'
+          check-latest: true
       - name: Compile man page markup
         id: gen-man-page-md
         run: |

.github/workflows/test-yq.yml

@@ -1,5 +1,8 @@
 name: Test Yq Action
-on: [push]
+on: 
+  push:
+  workflow_dispatch:
+
 permissions:
   contents: read
 
@@ -12,11 +15,14 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Get test
+        id: get_value
         uses: mikefarah/yq@master
         with:
-          cmd: yq eval '.a' examples/sample.yaml
+          cmd: yq '.test' examples/multiline-text.yaml
+      - name: Multiline test
+        run: echo "### It was [${{ steps.get_value.outputs.result }}]" >> $GITHUB_STEP_SUMMARY
       - name: Write inplace test
         id: lookupSdkVersion
         uses: mikefarah/yq@master
         with:
-          cmd: yq eval -i '.b.c = 5' examples/sample.yaml
+          cmd: yq -i '.b.c = 5' examples/sample.yaml

README.md

@@ -221,7 +221,17 @@ go install github.com/mikefarah/yq/v4@latest
 ## Community Supported Installation methods
 As these are supported by the community :heart: - however, they may be out of date with the officially supported releases.
 
-# Webi
+
+### Nix
+
+```
+nix profile install nixpkgs#yq-go
+```
+
+See [here](https://search.nixos.org/packages?channel=unstable&show=yq-go&from=0&size=50&sort=relevance&type=packages&query=yq-go)
+
+
+### Webi
 
 ```
 webi yq

action.yml

@@ -12,6 +12,6 @@ outputs:
     description: "The complete result from the yq command being run"
 runs:
   using: 'docker'
-  image: 'github-action/Dockerfile'
+  image: 'docker://mikefarah/yq:4-githubaction'
   args:
     - ${{ inputs.cmd }}

cmd/shell-completion.go

@@ -42,7 +42,7 @@ $ yq shell-completion fish > ~/.config/fish/completions/yq.fish
 `,
 	DisableFlagsInUseLine: true,
 	ValidArgs:             []string{"bash", "zsh", "fish", "powershell"},
-	Args:                  cobra.ExactValidArgs(1),
+	Args:                  cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs),
 	RunE: func(cmd *cobra.Command, args []string) error {
 		var err error = nil
 		switch args[0] {

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.28.1"
+	Version = "4.28.2"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

github-action/entrypoint.sh

@@ -2,8 +2,11 @@
 set -e
 echo "::debug::\$cmd: $1"
 RESULT=$(eval "$1")
-RESULT="${RESULT//'%'/'%25'}"
-RESULT="${RESULT//$'\n'/'%0A'}"
-RESULT="${RESULT//$'\r'/'%0D'}"
 echo "::debug::\$RESULT: $RESULT"
-echo ::set-output name=result::"$RESULT"
+# updating from 
+# https://github.com/orgs/community/discussions/26288#discussioncomment-3876281
+# https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-output-parameter
+delimiter=$(cat /proc/sys/kernel/random/uuid)
+echo "result<<${delimiter}" >> "${GITHUB_OUTPUT}"
+echo "${RESULT}" >> "${GITHUB_OUTPUT}"
+echo "${delimiter}" >> "${GITHUB_OUTPUT}"

go.mod

@@ -3,7 +3,7 @@ module github.com/mikefarah/yq/v4
 require (
 	github.com/a8m/envsubst v1.3.0
 	github.com/alecthomas/participle/v2 v2.0.0-beta.5
-	github.com/alecthomas/repr v0.1.0
+	github.com/alecthomas/repr v0.1.1
 	github.com/dimchansky/utfbom v1.1.1
 	github.com/elliotchance/orderedmap v1.5.0
 	github.com/fatih/color v1.13.0
@@ -12,14 +12,14 @@ require (
 	github.com/jinzhu/copier v0.3.5
 	github.com/magiconair/properties v1.8.6
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
-	github.com/spf13/cobra v1.5.0
+	github.com/spf13/cobra v1.6.0
 	golang.org/x/net v0.0.0-20220708220712-1185a9018129
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect

go.sum

@@ -3,8 +3,8 @@ github.com/a8m/envsubst v1.3.0/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGt
 github.com/alecthomas/assert/v2 v2.0.3 h1:WKqJODfOiQG0nEJKFKzDIG3E29CN2/4zR9XGJzKIkbg=
 github.com/alecthomas/participle/v2 v2.0.0-beta.5 h1:y6dsSYVb1G5eK6mgmy+BgI3Mw35a3WghArZ/Hbebrjo=
 github.com/alecthomas/participle/v2 v2.0.0-beta.5/go.mod h1:RC764t6n4L8D8ITAJv0qdokritYSNR3wV5cVwmIEaMM=
-github.com/alecthomas/repr v0.1.0 h1:ENn2e1+J3k09gyj2shc0dHr/yjaWSHRlrJ4DPMevDqE=
+github.com/alecthomas/repr v0.1.1 h1:87P60cSmareLAxMc4Hro0r2RBY4ROm0dYwkJNpS4pPs=
-github.com/alecthomas/repr v0.1.0/go.mod h1:2kn6fqh/zIyPLmm3ugklbEi5hg5wS435eygvNfaDQL8=
+github.com/alecthomas/repr v0.1.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -24,8 +24,8 @@ github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG
 github.com/goccy/go-yaml v1.9.5 h1:Eh/+3uk9kLxG4koCX6lRMAPS1OaMSAi+FJcya0INdB0=
 github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
-github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
-github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg=
 github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
@@ -43,8 +43,8 @@ github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsK
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/spf13/cobra v1.5.0 h1:X+jTBEBqF0bHN+9cSMgmfuvv2VHJ9ezmFNf9Y/XstYU=
+github.com/spf13/cobra v1.6.0 h1:42a0n6jwCot1pUmomAp4T7DeMD+20LFv4Q54pxLf2LI=
-github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
+github.com/spf13/cobra v1.6.0/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -77,7 +77,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473 h1:6D+BvnJ/j6e222UW8s2qTSe3wGBtvo0MbVQG/c5k8RE=
 gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473/go.mod h1:N1eN2tsCx0Ydtgjl4cqmbRCsY4/+z4cYDeqwZTk6zog=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/yqlib/doc/operators/datetime.md

@@ -13,11 +13,11 @@ To specify a custom parsing format, use the `with_dtf` operator. The first param
 yq 'with_dtf("myformat"; .a + "3h" | tz("Australia/Melbourne"))'
 ```
 
-See https://pkg.go.dev/time#pkg-constants for examples of formatting options.
+See the [library docs](https://pkg.go.dev/time#pkg-constants) for examples of formatting options.
 
 
 ## Timezones
-This uses golangs built in LoadLocation function to parse timezones strings. See https://pkg.go.dev/time#LoadLocation for more details.
+This uses golangs built in LoadLocation function to parse timezones strings. See the [library docs](https://pkg.go.dev/time#LoadLocation) for more details.
 
 
 ## Durations
@@ -150,7 +150,7 @@ a: 2021-01-01T03:10:00Z
 ```
 
 ## Date subtraction
-You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.
+You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/datetime#date-time-formattings) for more information.
 
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/operators/entries.md

@@ -101,6 +101,26 @@ KEY_a: 1
 KEY_b: 2
 ```
 
+## Custom sort map keys
+Use to_entries to convert to an array of key/value pairs, sort the array using sort/sort_by/etc, and convert it back.
+
+Given a sample.yml file of:
+```yaml
+a: 1
+c: 3
+b: 2
+```
+then
+```bash
+yq 'to_entries | sort_by(.key) | reverse | from_entries' sample.yml
+```
+will output
+```yaml
+c: 3
+b: 2
+a: 1
+```
+
 ## Use with_entries to filter the map
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/operators/headers/datetime.md

@@ -13,11 +13,11 @@ To specify a custom parsing format, use the `with_dtf` operator. The first param
 yq 'with_dtf("myformat"; .a + "3h" | tz("Australia/Melbourne"))'
 ```
 
-See https://pkg.go.dev/time#pkg-constants for examples of formatting options.
+See the [library docs](https://pkg.go.dev/time#pkg-constants) for examples of formatting options.
 
 
 ## Timezones
-This uses golangs built in LoadLocation function to parse timezones strings. See https://pkg.go.dev/time#LoadLocation for more details.
+This uses golangs built in LoadLocation function to parse timezones strings. See the [library docs](https://pkg.go.dev/time#LoadLocation) for more details.
 
 
 ## Durations

pkg/yqlib/doc/operators/headers/sort-keys.md

@@ -11,3 +11,6 @@ diff file1.yml file2.yml
 ```
 
 Note that `yq` does not yet consider anchors when sorting by keys - this may result in invalid yaml documents if your are using merge anchors.
+
+For more advanced sorting, using `to_entries` to convert the map to an array, then sort/process the array as you like (e.g. using `sort_by`) and convert back to a map using `from_entries`.
+See [here](https://mikefarah.gitbook.io/yq/operators/entries#custom-sort-map-keys) for an example. 

pkg/yqlib/doc/operators/sort-keys.md

@@ -12,6 +12,9 @@ diff file1.yml file2.yml
 
 Note that `yq` does not yet consider anchors when sorting by keys - this may result in invalid yaml documents if your are using merge anchors.
 
+For more advanced sorting, using `to_entries` to convert the map to an array, then sort/process the array as you like (e.g. using `sort_by`) and convert back to a map using `from_entries`.
+See [here](https://mikefarah.gitbook.io/yq/operators/entries#custom-sort-map-keys) for an example. 
+
 ## Sort keys of map
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/operator_datetime_test.go

@@ -77,7 +77,7 @@ var dateTimeOperatorScenarios = []expressionScenario{
 	},
 	{
 		description:    "Date subtraction",
-		subdescription: "You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/date-time-operators) for more information.",
+		subdescription: "You can subtract durations from dates. Assumes RFC3339 date time format, see [date-time operators](https://mikefarah.gitbook.io/yq/operators/datetime#date-time-formattings) for more information.",
 		document:       `a: 2021-01-01T03:10:00Z`,
 		expression:     `.a -= "3h10m"`,
 		expected: []string{

pkg/yqlib/operator_entries_test.go

@@ -52,6 +52,15 @@ var entriesOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!map)::KEY_a: 1\nKEY_b: 2\n",
 		},
 	},
+	{
+		description:    "Custom sort map keys",
+		subdescription: "Use to_entries to convert to an array of key/value pairs, sort the array using sort/sort_by/etc, and convert it back.",
+		document:       `{a: 1, c: 3, b: 2}`,
+		expression:     `to_entries | sort_by(.key) | reverse | from_entries`,
+		expected: []string{
+			"D0, P[], (!!map)::c: 3\nb: 2\na: 1\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `{a: 1, b: 2}`,

release_notes.txt

@@ -1,3 +1,11 @@
+4.28.2:
+  - Fixed Github Actions issues (thanks @mattphelps-8451)
+  - Fixed bug - can now delete documents #1377
+  - Fixed handling of UTF8 encoded CSVs #1373
+  - Detect and fail on missing closing brackets #1366
+  - yq Github actions now build docker image as part of release
+  - Bumped dependencies
+
 4.28.1:
   - Added `setpath` and `delpaths` operators, like jq (#1374)
   - Added `is_key` operator, to check if a match was a key when recursing

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.28.1'
+version: '4.28.2'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.