From 36e0194e42d9f7919e22f2fbe06d6e19df3fc5e0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sun, 25 Dec 2022 11:45:10 +1100
Subject: [PATCH 01/53] Bump github.com/goccy/go-yaml from 1.9.7 to 1.9.8
 (#1487)

Bumps [github.com/goccy/go-yaml](https://github.com/goccy/go-yaml) from 1.9.7 to 1.9.8.
- [Release notes](https://github.com/goccy/go-yaml/releases)
- [Changelog](https://github.com/goccy/go-yaml/blob/master/CHANGELOG.md)
- [Commits](https://github.com/goccy/go-yaml/compare/v1.9.7...v1.9.8)

---
updated-dependencies:
- dependency-name: github.com/goccy/go-yaml
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index c5fe6cc0..354b5fe8 100644
--- a/go.mod
+++ b/go.mod
@@ -8,7 +8,7 @@ require (
 	github.com/elliotchance/orderedmap v1.5.0
 	github.com/fatih/color v1.13.0
 	github.com/goccy/go-json v0.10.0
-	github.com/goccy/go-yaml v1.9.7
+	github.com/goccy/go-yaml v1.9.8
 	github.com/jinzhu/copier v0.3.5
 	github.com/magiconair/properties v1.8.7
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
diff --git a/go.sum b/go.sum
index c3c28eec..f9248d19 100644
--- a/go.sum
+++ b/go.sum
@@ -21,8 +21,8 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA=
 github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
-github.com/goccy/go-yaml v1.9.7 h1:D/Vx+JITklB1ugSkncB4BNR67M3X6AKs9+rqVeo3ddw=
-github.com/goccy/go-yaml v1.9.7/go.mod h1:JubOolP3gh0HpiBc4BLRD4YmjEjHAmIIB2aaXKkTfoE=
+github.com/goccy/go-yaml v1.9.8 h1:5gMyLUeU1/6zl+WFfR1hN7D2kf+1/eRGa7DFtToiBvQ=
+github.com/goccy/go-yaml v1.9.8/go.mod h1:JubOolP3gh0HpiBc4BLRD4YmjEjHAmIIB2aaXKkTfoE=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
 github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=

From 18cdea3f881712883382acbe8be68202a94828b8 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Tue, 3 Jan 2023 15:52:01 +1100
Subject: [PATCH 02/53] Build constraint not working for non linux (#1481)

* Build constraint not working for non linux

* Go Build Constraint: Fix Non-Linux Filename (#1494)

Correct the filename pkg/yqlib/chown_not_linux.go to escape the
default OS detection in filename format `*_GOOS[_test].go`.

Add an extra word after `linux` to resolve the issue.
pkg/yqlib/chown_not_linux_os.go

Signed-off-by: Bhargav Ravuri <vaguecoder0to.n@gmail.com>

Signed-off-by: Bhargav Ravuri <vaguecoder0to.n@gmail.com>

Signed-off-by: Bhargav Ravuri <vaguecoder0to.n@gmail.com>
Co-authored-by: Bhargav Ravuri <saibhargavravuri@gmail.com>
---
 pkg/yqlib/chown_linux.go            | 18 ++++++++++++++++++
 pkg/yqlib/chown_not_linux_os.go     | 12 ++++++++++++
 pkg/yqlib/write_in_place_handler.go |  4 ++++
 3 files changed, 34 insertions(+)
 create mode 100644 pkg/yqlib/chown_linux.go
 create mode 100644 pkg/yqlib/chown_not_linux_os.go

diff --git a/pkg/yqlib/chown_linux.go b/pkg/yqlib/chown_linux.go
new file mode 100644
index 00000000..3913662b
--- /dev/null
+++ b/pkg/yqlib/chown_linux.go
@@ -0,0 +1,18 @@
+//go:build linux
+
+package yqlib
+
+import (
+	"io/fs"
+	"os"
+	"syscall"
+)
+
+func changeOwner(info fs.FileInfo, file *os.File) error {
+	if stat, ok := info.Sys().(*syscall.Stat_t); ok {
+		uid := int(stat.Uid)
+		gid := int(stat.Gid)
+		return os.Chown(file.Name(), uid, gid)
+	}
+	return nil
+}
diff --git a/pkg/yqlib/chown_not_linux_os.go b/pkg/yqlib/chown_not_linux_os.go
new file mode 100644
index 00000000..3675791a
--- /dev/null
+++ b/pkg/yqlib/chown_not_linux_os.go
@@ -0,0 +1,12 @@
+//go:build !linux
+
+package yqlib
+
+import (
+	"io/fs"
+	"os"
+)
+
+func changeOwner(info fs.FileInfo, file *os.File) error {
+	return nil
+}
diff --git a/pkg/yqlib/write_in_place_handler.go b/pkg/yqlib/write_in_place_handler.go
index d3bbb487..d49b2963 100644
--- a/pkg/yqlib/write_in_place_handler.go
+++ b/pkg/yqlib/write_in_place_handler.go
@@ -34,6 +34,10 @@ func (w *writeInPlaceHandlerImpl) CreateTempFile() (*os.File, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	if err = changeOwner(info, file); err != nil {
+		return nil, err
+	}
 	log.Debug("WriteInPlaceHandler: writing to tempfile: %v", file.Name())
 	w.tempFile = file
 	return file, err

From ef2064c75388692e0c155e9449a7b5abac75b0da Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Tue, 3 Jan 2023 19:25:27 +1100
Subject: [PATCH 03/53] Updating release notes

---
 release_notes.txt | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/release_notes.txt b/release_notes.txt
index 32095b88..977e6cae 100644
--- a/release_notes.txt
+++ b/release_notes.txt
@@ -1,3 +1,7 @@
+(not yet released):
+  - Ownership of file now maintained in linux (thanks @vaguecoder) #1473
+  - Bumped dependency versions
+
 4.30.6:
   - Fixed xml comment in array of scalars #1465
   - Include blank new lines in leading header preprocessing #1462

From 9af55d555bf36b089db637c8da7971bdfaffc8f9 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Tue, 10 Jan 2023 13:48:57 +1100
Subject: [PATCH 04/53] Fixed handling of merging null #1501

---
 pkg/yqlib/doc/operators/multiply-merge.md | 40 +++++++++++++++++++++++
 pkg/yqlib/operator_multiply.go            | 10 ++++--
 pkg/yqlib/operator_multiply_test.go       | 35 ++++++++++++++++++++
 3 files changed, 83 insertions(+), 2 deletions(-)

diff --git a/pkg/yqlib/doc/operators/multiply-merge.md b/pkg/yqlib/doc/operators/multiply-merge.md
index 6db12660..60967413 100644
--- a/pkg/yqlib/doc/operators/multiply-merge.md
+++ b/pkg/yqlib/doc/operators/multiply-merge.md
@@ -499,3 +499,43 @@ b: !goat
   dog: woof
 ```
 
+## Merging a null with a map
+Running
+```bash
+yq --null-input 'null * {"some": "thing"}'
+```
+will output
+```yaml
+some: thing
+```
+
+## Merging a map with null
+Running
+```bash
+yq --null-input '{"some": "thing"} * null'
+```
+will output
+```yaml
+some: thing
+```
+
+## Merging an null with an array
+Running
+```bash
+yq --null-input 'null * ["some"]'
+```
+will output
+```yaml
+- some
+```
+
+## Merging an array with null
+Running
+```bash
+yq --null-input '["some"] * null'
+```
+will output
+```yaml
+- some
+```
+
diff --git a/pkg/yqlib/operator_multiply.go b/pkg/yqlib/operator_multiply.go
index ac27ad1f..defca156 100644
--- a/pkg/yqlib/operator_multiply.go
+++ b/pkg/yqlib/operator_multiply.go
@@ -60,8 +60,14 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
 		log.Debugf("Multiplying LHS: %v", lhs.Node.Tag)
 		log.Debugf("-          RHS: %v", rhs.Node.Tag)
 
-		if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode ||
-			(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) {
+		if rhs.Node.Tag == "!!null" {
+			return lhs.Copy()
+		}
+
+		if (lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode) ||
+			(lhs.Node.Tag == "!!null" && rhs.Node.Kind == yaml.MappingNode) ||
+			(lhs.Node.Kind == yaml.SequenceNode && rhs.Node.Kind == yaml.SequenceNode) ||
+			(lhs.Node.Tag == "!!null" && rhs.Node.Kind == yaml.SequenceNode) {
 			var newBlank = CandidateNode{}
 			err := copier.CopyWithOption(&newBlank, lhs, copier.Option{IgnoreEmpty: true, DeepCopy: true})
 			if err != nil {
diff --git a/pkg/yqlib/operator_multiply_test.go b/pkg/yqlib/operator_multiply_test.go
index d775986c..c20278e6 100644
--- a/pkg/yqlib/operator_multiply_test.go
+++ b/pkg/yqlib/operator_multiply_test.go
@@ -579,6 +579,41 @@ var multiplyOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: {a: apple is included, b: cool.}\n",
 		},
 	},
+	{
+		description: "Merging a null with a map",
+		expression:  `null * {"some": "thing"}`,
+		expected: []string{
+			"D0, P[], (!!map)::some: thing\n",
+		},
+	},
+	{
+		description: "Merging a map with null",
+		expression:  `{"some": "thing"} * null`,
+		expected: []string{
+			"D0, P[], (!!map)::some: thing\n",
+		},
+	},
+	{
+		description: "Merging an null with an array",
+		expression:  `null * ["some"]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- some\n",
+		},
+	},
+	{
+		description: "Merging an array with null",
+		expression:  `["some"] * null`,
+		expected: []string{
+			"D0, P[], (!!seq)::- some\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		expression: `null * null`,
+		expected: []string{
+			"D0, P[], (!!null)::null\n",
+		},
+	},
 }
 
 func TestMultiplyOperatorScenarios(t *testing.T) {

From 00c2be541d7db3ad4a837b7cfafd39576829649a Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Wed, 11 Jan 2023 12:19:46 +1100
Subject: [PATCH 05/53] Added prepend array example

---
 pkg/yqlib/doc/operators/add.md | 17 +++++++++++++++++
 pkg/yqlib/operator_add_test.go |  8 ++++++++
 2 files changed, 25 insertions(+)

diff --git a/pkg/yqlib/doc/operators/add.md b/pkg/yqlib/doc/operators/add.md
index ecadb84f..469e13d8 100644
--- a/pkg/yqlib/doc/operators/add.md
+++ b/pkg/yqlib/doc/operators/add.md
@@ -86,6 +86,23 @@ will output
 a: ['dog', 'cat']
 ```
 
+## Prepend to existing array
+Given a sample.yml file of:
+```yaml
+a:
+  - dog
+```
+then
+```bash
+yq '.a = ["cat"] + .a' sample.yml
+```
+will output
+```yaml
+a:
+  - cat
+  - dog
+```
+
 ## Add new object to array
 Given a sample.yml file of:
 ```yaml
diff --git a/pkg/yqlib/operator_add_test.go b/pkg/yqlib/operator_add_test.go
index b7452512..99a84845 100644
--- a/pkg/yqlib/operator_add_test.go
+++ b/pkg/yqlib/operator_add_test.go
@@ -101,6 +101,14 @@ var addOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: ['dog', 'cat']\n",
 		},
 	},
+	{
+		description: "Prepend to existing array",
+		document:    `a: [dog]`,
+		expression:  `.a = ["cat"] + .a`,
+		expected: []string{
+			"D0, P[], (doc)::a: [cat, dog]\n",
+		},
+	},
 	{
 		skipDoc:        true,
 		description:    "Concatenate to existing array",

From 95d14cb0af0fa4cb5157e8fab141e5b2c1ad9d30 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 12 Jan 2023 14:21:28 +1100
Subject: [PATCH 06/53] Bump golang from 1.19.4 to 1.19.5 (#1516)

Bumps golang from 1.19.4 to 1.19.5.

---
updated-dependencies:
- dependency-name: golang
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 Dockerfile     | 2 +-
 Dockerfile.dev | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 1b13254c..d3dd13a4 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.19.4 as builder
+FROM golang:1.19.5 as builder
 
 WORKDIR /go/src/mikefarah/yq
 
diff --git a/Dockerfile.dev b/Dockerfile.dev
index ddd124a0..a2c77979 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -1,4 +1,4 @@
-FROM golang:1.19.4
+FROM golang:1.19.5
 
 COPY scripts/devtools.sh /opt/devtools.sh
 

From e23c989b78d335d95e9804973853625525e589e2 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 12 Jan 2023 14:22:43 +1100
Subject: [PATCH 07/53] Bump github.com/alecthomas/repr from 0.1.1 to 0.2.0
 (#1517)

Bumps [github.com/alecthomas/repr](https://github.com/alecthomas/repr) from 0.1.1 to 0.2.0.
- [Release notes](https://github.com/alecthomas/repr/releases)
- [Commits](https://github.com/alecthomas/repr/compare/v0.1.1...v0.2.0)

---
updated-dependencies:
- dependency-name: github.com/alecthomas/repr
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 354b5fe8..942b92b0 100644
--- a/go.mod
+++ b/go.mod
@@ -3,7 +3,7 @@ module github.com/mikefarah/yq/v4
 require (
 	github.com/a8m/envsubst v1.3.0
 	github.com/alecthomas/participle/v2 v2.0.0-beta.5
-	github.com/alecthomas/repr v0.1.1
+	github.com/alecthomas/repr v0.2.0
 	github.com/dimchansky/utfbom v1.1.1
 	github.com/elliotchance/orderedmap v1.5.0
 	github.com/fatih/color v1.13.0
diff --git a/go.sum b/go.sum
index f9248d19..3a4b4813 100644
--- a/go.sum
+++ b/go.sum
@@ -3,8 +3,8 @@ github.com/a8m/envsubst v1.3.0/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGt
 github.com/alecthomas/assert/v2 v2.0.3 h1:WKqJODfOiQG0nEJKFKzDIG3E29CN2/4zR9XGJzKIkbg=
 github.com/alecthomas/participle/v2 v2.0.0-beta.5 h1:y6dsSYVb1G5eK6mgmy+BgI3Mw35a3WghArZ/Hbebrjo=
 github.com/alecthomas/participle/v2 v2.0.0-beta.5/go.mod h1:RC764t6n4L8D8ITAJv0qdokritYSNR3wV5cVwmIEaMM=
-github.com/alecthomas/repr v0.1.1 h1:87P60cSmareLAxMc4Hro0r2RBY4ROm0dYwkJNpS4pPs=
-github.com/alecthomas/repr v0.1.1/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/alecthomas/repr v0.2.0 h1:HAzS41CIzNW5syS8Mf9UwXhNH1J9aix/BvDRf1Ml2Yk=
+github.com/alecthomas/repr v0.2.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

From fcda053d7325d3ddf2c263b065aea3d520ac1610 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Thu, 12 Jan 2023 14:26:58 +1100
Subject: [PATCH 08/53] Updating tests from lexer version bump

---
 pkg/yqlib/lexer_participle_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pkg/yqlib/lexer_participle_test.go b/pkg/yqlib/lexer_participle_test.go
index 29d65adc..2bce0ae2 100644
--- a/pkg/yqlib/lexer_participle_test.go
+++ b/pkg/yqlib/lexer_participle_test.go
@@ -61,7 +61,7 @@ var participleLexerScenarios = []participleLexerScenario{
 				TokenType: operationToken,
 				Operation: &Operation{
 					OperationType: valueOpType,
-					Value:         3,
+					Value:         int64(3),
 					StringValue:   "3",
 					CandidateNode: &CandidateNode{
 						Node: &yaml.Node{
@@ -103,7 +103,7 @@ var participleLexerScenarios = []participleLexerScenario{
 				TokenType: operationToken,
 				Operation: &Operation{
 					OperationType: valueOpType,
-					Value:         -2,
+					Value:         int64(-2),
 					StringValue:   "-2",
 					CandidateNode: &CandidateNode{
 						Node: &yaml.Node{

From d7da0cca3c707b6ed9fa7ecf6a82bbe809aa8281 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Thu, 12 Jan 2023 15:11:45 +1100
Subject: [PATCH 09/53] Fixed value operator bug  #1515

---
 pkg/yqlib/decoder_properties.go  |  4 ++--
 pkg/yqlib/lib.go                 |  2 ++
 pkg/yqlib/operator_multiply.go   |  2 +-
 pkg/yqlib/operator_path.go       |  2 +-
 pkg/yqlib/operator_value.go      | 26 +++++++++++++++++++++++++-
 pkg/yqlib/operator_value_test.go | 22 ++++++++++++++++++++++
 pkg/yqlib/operators.go           |  4 ++--
 7 files changed, 55 insertions(+), 7 deletions(-)

diff --git a/pkg/yqlib/decoder_properties.go b/pkg/yqlib/decoder_properties.go
index f234f9c3..4e6541a9 100644
--- a/pkg/yqlib/decoder_properties.go
+++ b/pkg/yqlib/decoder_properties.go
@@ -63,7 +63,7 @@ func (dec *propertiesDecoder) applyPropertyComments(context Context, path []inte
 
 	rhsCandidateNode.Node.Tag = guessTagFromCustomType(rhsCandidateNode.Node)
 
-	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}
+	rhsOp := &Operation{OperationType: referenceOpType, CandidateNode: rhsCandidateNode}
 
 	assignmentOpNode := &ExpressionNode{
 		Operation: assignmentOp,
@@ -102,7 +102,7 @@ func (dec *propertiesDecoder) applyProperty(context Context, properties *propert
 
 	assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
 
-	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhsCandidateNode}
+	rhsOp := &Operation{OperationType: referenceOpType, CandidateNode: rhsCandidateNode}
 
 	assignmentOpNode := &ExpressionNode{
 		Operation: assignmentOp,
diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go
index 5cd2f652..1bb7f405 100644
--- a/pkg/yqlib/lib.go
+++ b/pkg/yqlib/lib.go
@@ -155,6 +155,7 @@ var traverseArrayOpType = &operationType{Type: "TRAVERSE_ARRAY", NumArgs: 2, Pre
 
 var selfReferenceOpType = &operationType{Type: "SELF", NumArgs: 0, Precedence: 55, Handler: selfOperator}
 var valueOpType = &operationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: valueOperator}
+var referenceOpType = &operationType{Type: "REF", NumArgs: 0, Precedence: 50, Handler: referenceOperator}
 var envOpType = &operationType{Type: "ENV", NumArgs: 0, Precedence: 50, Handler: envOperator}
 var notOpType = &operationType{Type: "NOT", NumArgs: 0, Precedence: 50, Handler: notOperator}
 var emptyOpType = &operationType{Type: "EMPTY", Precedence: 50, Handler: emptyOperator}
@@ -416,6 +417,7 @@ func footComment(node *yaml.Node) string {
 }
 
 func createValueOperation(value interface{}, stringValue string) *Operation {
+	log.Debug("creating value op for string %v", stringValue)
 	var node = createScalarNode(value, stringValue)
 
 	return &Operation{
diff --git a/pkg/yqlib/operator_multiply.go b/pkg/yqlib/operator_multiply.go
index defca156..cf638015 100644
--- a/pkg/yqlib/operator_multiply.go
+++ b/pkg/yqlib/operator_multiply.go
@@ -195,7 +195,7 @@ func applyAssignment(d *dataTreeNavigator, context Context, pathIndexToStartFrom
 	} else {
 		log.Debugf("merge - assignmentOp := &Operation{OperationType: assignAttributesOpType}")
 	}
-	rhsOp := &Operation{OperationType: valueOpType, CandidateNode: rhs}
+	rhsOp := &Operation{OperationType: referenceOpType, CandidateNode: rhs}
 
 	assignmentOpNode := &ExpressionNode{
 		Operation: assignmentOp,
diff --git a/pkg/yqlib/operator_path.go b/pkg/yqlib/operator_path.go
index 3905d372..cdad7f1b 100644
--- a/pkg/yqlib/operator_path.go
+++ b/pkg/yqlib/operator_path.go
@@ -81,7 +81,7 @@ func setPathOperator(d *dataTreeNavigator, context Context, expressionNode *Expr
 			return Context{}, fmt.Errorf("SETPATH: expected single value on RHS but found %v", targetContextValue.MatchingNodes.Len())
 		}
 
-		rhsOp := &Operation{OperationType: valueOpType, CandidateNode: targetContextValue.MatchingNodes.Front().Value.(*CandidateNode)}
+		rhsOp := &Operation{OperationType: referenceOpType, CandidateNode: targetContextValue.MatchingNodes.Front().Value.(*CandidateNode)}
 
 		assignmentOpNode := &ExpressionNode{
 			Operation: assignmentOp,
diff --git a/pkg/yqlib/operator_value.go b/pkg/yqlib/operator_value.go
index 6c70b1f7..b4296acc 100644
--- a/pkg/yqlib/operator_value.go
+++ b/pkg/yqlib/operator_value.go
@@ -1,6 +1,30 @@
 package yqlib
 
+import "container/list"
+
+func referenceOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	return context.SingleChildContext(expressionNode.Operation.CandidateNode), nil
+}
+
 func valueOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debug("value = %v", expressionNode.Operation.CandidateNode.Node.Value)
-	return context.SingleChildContext(expressionNode.Operation.CandidateNode), nil
+	if context.MatchingNodes.Len() == 0 {
+		clone, err := expressionNode.Operation.CandidateNode.Copy()
+		if err != nil {
+			return Context{}, err
+		}
+		return context.SingleChildContext(clone), nil
+	}
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		clone, err := expressionNode.Operation.CandidateNode.Copy()
+		if err != nil {
+			return Context{}, err
+		}
+		results.PushBack(clone)
+	}
+
+	return context.ChildContext(results), nil
 }
diff --git a/pkg/yqlib/operator_value_test.go b/pkg/yqlib/operator_value_test.go
index 81534804..b77a3c80 100644
--- a/pkg/yqlib/operator_value_test.go
+++ b/pkg/yqlib/operator_value_test.go
@@ -12,6 +12,28 @@ var valueOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!int)::1\n",
 		},
 	},
+	{
+		document:   `[1,2,3]`,
+		expression: `.[] | "foo"`,
+		expected: []string{
+			"D0, P[], (!!str)::foo\n",
+			"D0, P[], (!!str)::foo\n",
+			"D0, P[], (!!str)::foo\n",
+		},
+	},
+	{
+		document:   `[1,2,3]`,
+		expression: `[.[] | "foo"] | .[0] = "cat"`,
+		expected: []string{
+			"D0, P[], (!!seq)::- cat\n- foo\n- foo\n",
+		},
+	},
+	{
+		expression: `"foo"`,
+		expected: []string{
+			"D0, P[], (!!str)::foo\n",
+		},
+	},
 	{
 		document:   ``,
 		expression: `0x9f`,
diff --git a/pkg/yqlib/operators.go b/pkg/yqlib/operators.go
index 00f45d49..ea0574d6 100644
--- a/pkg/yqlib/operators.go
+++ b/pkg/yqlib/operators.go
@@ -41,9 +41,9 @@ func compoundAssignFunction(d *dataTreeNavigator, context Context, expressionNod
 		if err != nil {
 			return Context{}, err
 		}
-		valueCopyExp := &ExpressionNode{Operation: &Operation{OperationType: valueOpType, CandidateNode: clone}}
+		valueCopyExp := &ExpressionNode{Operation: &Operation{OperationType: referenceOpType, CandidateNode: clone}}
 
-		valueExpression := &ExpressionNode{Operation: &Operation{OperationType: valueOpType, CandidateNode: candidate}}
+		valueExpression := &ExpressionNode{Operation: &Operation{OperationType: referenceOpType, CandidateNode: candidate}}
 
 		assignmentOpNode := &ExpressionNode{Operation: assignmentOp, LHS: valueExpression, RHS: calculation(valueCopyExp, expressionNode.RHS)}
 

From 3b84c031314b554db9401f9bdbc8cb954a8f8522 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Fri, 13 Jan 2023 13:45:19 +1100
Subject: [PATCH 10/53] Fixed bug in splice operator #1511

---
 pkg/yqlib/operator_slice.go      |  2 ++
 pkg/yqlib/operator_slice_test.go | 18 ++++++++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/pkg/yqlib/operator_slice.go b/pkg/yqlib/operator_slice.go
index e20311df..e496cf93 100644
--- a/pkg/yqlib/operator_slice.go
+++ b/pkg/yqlib/operator_slice.go
@@ -48,6 +48,8 @@ func sliceArrayOperator(d *dataTreeNavigator, context Context, expressionNode *E
 		relativeSecondNumber := secondNumber
 		if relativeSecondNumber < 0 {
 			relativeSecondNumber = len(original.Content) + secondNumber
+		} else if relativeSecondNumber > len(original.Content) {
+			relativeSecondNumber = len(original.Content)
 		}
 
 		log.Debug("calculateIndicesToTraverse: slice from %v to %v", relativeFirstNumber, relativeSecondNumber)
diff --git a/pkg/yqlib/operator_slice_test.go b/pkg/yqlib/operator_slice_test.go
index 9785410b..29a821eb 100644
--- a/pkg/yqlib/operator_slice_test.go
+++ b/pkg/yqlib/operator_slice_test.go
@@ -55,6 +55,24 @@ var sliceArrayScenarios = []expressionScenario{
 			"D0, P[1], (!!seq)::- banana\n- grape\n",
 		},
 	},
+	{
+		skipDoc:     true,
+		description: "second index beyond array clamps",
+		document:    `[cat]`,
+		expression:  `.[:3]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- cat\n",
+		},
+	},
+	{
+		skipDoc:     true,
+		description: "first index beyond array returns nothing",
+		document:    `[cat]`,
+		expression:  `.[3:]`,
+		expected: []string{
+			"D0, P[], (!!seq)::[]\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `[[cat, dog, frog, cow], [apple, banana, grape, mango]]`,

From 729ac285daf9b84c75694a3a3badda679fb6b4ed Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Sat, 14 Jan 2023 14:04:48 +1100
Subject: [PATCH 11/53] updating release notes

---
 release_notes.txt | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/release_notes.txt b/release_notes.txt
index 977e6cae..0b697eee 100644
--- a/release_notes.txt
+++ b/release_notes.txt
@@ -1,6 +1,9 @@
-(not yet released):
-  - Ownership of file now maintained in linux (thanks @vaguecoder) #1473
-  - Bumped dependency versions
+4.30.7:
+ - Fixed bug in splice operator #1511
+ - Fixed value operator bug  #1515
+ - Fixed handling of merging null #1501
+ - Ownership of file now maintained in linux (thanks @vaguecoder) #1473
+ - Bumped dependency versions
 
 4.30.6:
   - Fixed xml comment in array of scalars #1465

From ec40a1a08ac3f998164dd173625a606c53a5529a Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Sat, 14 Jan 2023 14:05:19 +1100
Subject: [PATCH 12/53] Bumping version

---
 cmd/version.go      | 2 +-
 snap/snapcraft.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd/version.go b/cmd/version.go
index 678b3373..74f061ac 100644
--- a/cmd/version.go
+++ b/cmd/version.go
@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "v4.30.6"
+	Version = "v4.30.7"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release
diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml
index 0beed65a..4d02ad62 100644
--- a/snap/snapcraft.yaml
+++ b/snap/snapcraft.yaml
@@ -1,5 +1,5 @@
 name: yq
-version: 'v4.30.6'
+version: 'v4.30.7'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.

From 473be23153aed6e04ec4d687991ce8998b2e8d5b Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Sun, 15 Jan 2023 11:36:52 +1100
Subject: [PATCH 13/53] Log info message when unable to chown file #1521

---
 pkg/yqlib/chown_linux.go | 9 ++++++++-
 release_notes.txt        | 4 ++++
 2 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/pkg/yqlib/chown_linux.go b/pkg/yqlib/chown_linux.go
index 3913662b..174a7948 100644
--- a/pkg/yqlib/chown_linux.go
+++ b/pkg/yqlib/chown_linux.go
@@ -12,7 +12,14 @@ func changeOwner(info fs.FileInfo, file *os.File) error {
 	if stat, ok := info.Sys().(*syscall.Stat_t); ok {
 		uid := int(stat.Uid)
 		gid := int(stat.Gid)
-		return os.Chown(file.Name(), uid, gid)
+
+		err := os.Chown(file.Name(), uid, gid)
+		if err != nil {
+			// this happens with snap confinement
+			// not really a big issue as users can chown
+			// the file themselves if required.
+			log.Info("Skipping chown: %v", err)
+		}
 	}
 	return nil
 }
diff --git a/release_notes.txt b/release_notes.txt
index 0b697eee..3cc78097 100644
--- a/release_notes.txt
+++ b/release_notes.txt
@@ -1,3 +1,7 @@
+4.30.8:
+ - Log info message when unable to chown file in linux (e.g. snap confinement) #1521
+
+
 4.30.7:
  - Fixed bug in splice operator #1511
  - Fixed value operator bug  #1515

From dd6cf3df146f3e2c0f8c765a6ef9e35780ad8cc1 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Sun, 15 Jan 2023 11:37:07 +1100
Subject: [PATCH 14/53] Bumping version

---
 cmd/version.go      | 2 +-
 snap/snapcraft.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd/version.go b/cmd/version.go
index 74f061ac..05977638 100644
--- a/cmd/version.go
+++ b/cmd/version.go
@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "v4.30.7"
+	Version = "v4.30.8"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release
diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml
index 4d02ad62..cd8355bb 100644
--- a/snap/snapcraft.yaml
+++ b/snap/snapcraft.yaml
@@ -1,5 +1,5 @@
 name: yq
-version: 'v4.30.7'
+version: 'v4.30.8'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.

From 6d7d76a3f100e54f9ab69d2d83eb54978325a688 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Mon, 23 Jan 2023 11:37:18 +1100
Subject: [PATCH 15/53] Adds @uri/@urid #1529

---
 pkg/yqlib/decoder.go                          |  1 +
 pkg/yqlib/decoder_uri.go                      | 60 +++++++++++++++++++
 pkg/yqlib/doc/operators/encode-decode.md      | 29 +++++++++
 .../doc/operators/headers/encode-decode.md    |  1 +
 pkg/yqlib/encoder_uri.go                      | 37 ++++++++++++
 pkg/yqlib/lexer_participle.go                 |  3 +
 pkg/yqlib/operator_encoder_decoder.go         |  4 ++
 pkg/yqlib/operator_encoder_decoder_test.go    | 16 +++++
 pkg/yqlib/printer.go                          |  1 +
 9 files changed, 152 insertions(+)
 create mode 100644 pkg/yqlib/decoder_uri.go
 create mode 100644 pkg/yqlib/encoder_uri.go

diff --git a/pkg/yqlib/decoder.go b/pkg/yqlib/decoder.go
index 904b65c3..ff7ab2db 100644
--- a/pkg/yqlib/decoder.go
+++ b/pkg/yqlib/decoder.go
@@ -15,6 +15,7 @@ const (
 	JsonInputFormat
 	CSVObjectInputFormat
 	TSVObjectInputFormat
+	UriInputFormat
 )
 
 type Decoder interface {
diff --git a/pkg/yqlib/decoder_uri.go b/pkg/yqlib/decoder_uri.go
new file mode 100644
index 00000000..eabf26be
--- /dev/null
+++ b/pkg/yqlib/decoder_uri.go
@@ -0,0 +1,60 @@
+package yqlib
+
+import (
+	"bytes"
+	"io"
+	"net/url"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+type uriDecoder struct {
+	reader       io.Reader
+	finished     bool
+	readAnything bool
+}
+
+func NewUriDecoder() Decoder {
+	return &uriDecoder{finished: false}
+}
+
+func (dec *uriDecoder) Init(reader io.Reader) error {
+	dec.reader = reader
+	dec.readAnything = false
+	dec.finished = false
+	return nil
+}
+
+func (dec *uriDecoder) Decode() (*CandidateNode, error) {
+	if dec.finished {
+		return nil, io.EOF
+	}
+
+	buf := new(bytes.Buffer)
+
+	if _, err := buf.ReadFrom(dec.reader); err != nil {
+		return nil, err
+	}
+	if buf.Len() == 0 {
+		dec.finished = true
+
+		// if we've read _only_ an empty string, lets return that
+		// otherwise if we've already read some bytes, and now we get
+		// an empty string, then we are done.
+		if dec.readAnything {
+			return nil, io.EOF
+		}
+	}
+	newValue, err := url.QueryUnescape(buf.String())
+	if err != nil {
+		return nil, err
+	}
+	dec.readAnything = true
+	return &CandidateNode{
+		Node: &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!str",
+			Value: newValue,
+		},
+	}, nil
+}
diff --git a/pkg/yqlib/doc/operators/encode-decode.md b/pkg/yqlib/doc/operators/encode-decode.md
index f9e5097d..f6be772c 100644
--- a/pkg/yqlib/doc/operators/encode-decode.md
+++ b/pkg/yqlib/doc/operators/encode-decode.md
@@ -16,6 +16,7 @@ These operators are useful to process yaml documents that have stringified embed
 | TSV | from_tsv/@tsvd | to_tsv/@tsv |
 | XML | from_xml/@xmld | to_xml(i)/@xml |
 | Base64 | @base64d | @base64 |
+| URI | @urid | @uri |
 
 
 See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) for accepted formats.
@@ -435,6 +436,34 @@ will output
 YTogYXBwbGUK
 ```
 
+## Encode a string to uri
+Given a sample.yml file of:
+```yaml
+coolData: this has & special () characters *
+```
+then
+```bash
+yq '.coolData | @uri' sample.yml
+```
+will output
+```yaml
+this+has+%26+special+%28%29+characters+%2A
+```
+
+## Decode a URI to a string
+Given a sample.yml file of:
+```yaml
+this+has+%26+special+%28%29+characters+%2A
+```
+then
+```bash
+yq '@urid' sample.yml
+```
+will output
+```yaml
+this has & special () characters *
+```
+
 ## Decode a base64 encoded string
 Decoded data is assumed to be a string.
 
diff --git a/pkg/yqlib/doc/operators/headers/encode-decode.md b/pkg/yqlib/doc/operators/headers/encode-decode.md
index f83b45df..31c1c10b 100644
--- a/pkg/yqlib/doc/operators/headers/encode-decode.md
+++ b/pkg/yqlib/doc/operators/headers/encode-decode.md
@@ -16,6 +16,7 @@ These operators are useful to process yaml documents that have stringified embed
 | TSV | from_tsv/@tsvd | to_tsv/@tsv |
 | XML | from_xml/@xmld | to_xml(i)/@xml |
 | Base64 | @base64d | @base64 |
+| URI | @urid | @uri |
 
 
 See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) for accepted formats.
diff --git a/pkg/yqlib/encoder_uri.go b/pkg/yqlib/encoder_uri.go
new file mode 100644
index 00000000..e4141cb8
--- /dev/null
+++ b/pkg/yqlib/encoder_uri.go
@@ -0,0 +1,37 @@
+package yqlib
+
+import (
+	"fmt"
+	"io"
+	"net/url"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+type uriEncoder struct {
+}
+
+func NewUriEncoder() Encoder {
+	return &uriEncoder{}
+}
+
+func (e *uriEncoder) CanHandleAliases() bool {
+	return false
+}
+
+func (e *uriEncoder) PrintDocumentSeparator(writer io.Writer) error {
+	return nil
+}
+
+func (e *uriEncoder) PrintLeadingContent(writer io.Writer, content string) error {
+	return nil
+}
+
+func (e *uriEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error {
+	node := unwrapDoc(originalNode)
+	if guessTagFromCustomType(node) != "!!str" {
+		return fmt.Errorf("cannot encode %v as url, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag)
+	}
+	_, err := writer.Write([]byte(url.QueryEscape(originalNode.Value)))
+	return err
+}
diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go
index 14ea14e8..547c1810 100644
--- a/pkg/yqlib/lexer_participle.go
+++ b/pkg/yqlib/lexer_participle.go
@@ -78,6 +78,9 @@ var participleYqRules = []*participleYqRule{
 	{"Base64d", `@base64d`, decodeOp(Base64InputFormat), 0},
 	{"Base64", `@base64`, encodeWithIndent(Base64OutputFormat, 0), 0},
 
+	{"Urld", `@urid`, decodeOp(UriInputFormat), 0},
+	{"Url", `@uri`, encodeWithIndent(UriOutputFormat, 0), 0},
+
 	{"LoadXML", `load_?xml|xml_?load`, loadOp(NewXMLDecoder(ConfiguredXMLPreferences), false), 0},
 
 	{"LoadBase64", `load_?base64`, loadOp(NewBase64Decoder(), false), 0},
diff --git a/pkg/yqlib/operator_encoder_decoder.go b/pkg/yqlib/operator_encoder_decoder.go
index 8f41389b..2fa0cb05 100644
--- a/pkg/yqlib/operator_encoder_decoder.go
+++ b/pkg/yqlib/operator_encoder_decoder.go
@@ -26,6 +26,8 @@ func configureEncoder(format PrinterOutputFormat, indent int) Encoder {
 		return NewXMLEncoder(indent, ConfiguredXMLPreferences)
 	case Base64OutputFormat:
 		return NewBase64Encoder()
+	case UriOutputFormat:
+		return NewUriEncoder()
 	}
 	panic("invalid encoder")
 }
@@ -113,6 +115,8 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 		decoder = NewCSVObjectDecoder(',')
 	case TSVObjectInputFormat:
 		decoder = NewCSVObjectDecoder('\t')
+	case UriInputFormat:
+		decoder = NewUriDecoder()
 	}
 
 	var results = list.New()
diff --git a/pkg/yqlib/operator_encoder_decoder_test.go b/pkg/yqlib/operator_encoder_decoder_test.go
index e3961fbe..ee54868e 100644
--- a/pkg/yqlib/operator_encoder_decoder_test.go
+++ b/pkg/yqlib/operator_encoder_decoder_test.go
@@ -241,6 +241,22 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!str)::YTogYXBwbGUK\n",
 		},
 	},
+	{
+		description: "Encode a string to uri",
+		document:    "coolData: this has & special () characters *",
+		expression:  ".coolData | @uri",
+		expected: []string{
+			"D0, P[coolData], (!!str)::this+has+%26+special+%28%29+characters+%2A\n",
+		},
+	},
+	{
+		description: "Decode a URI to a string",
+		document:    "this+has+%26+special+%28%29+characters+%2A",
+		expression:  "@urid",
+		expected: []string{
+			"D0, P[], (!!str)::this has & special () characters *\n",
+		},
+	},
 	{
 		description:    "Decode a base64 encoded string",
 		subdescription: "Decoded data is assumed to be a string.",
diff --git a/pkg/yqlib/printer.go b/pkg/yqlib/printer.go
index aef082d9..14f96cc3 100644
--- a/pkg/yqlib/printer.go
+++ b/pkg/yqlib/printer.go
@@ -27,6 +27,7 @@ const (
 	TSVOutputFormat
 	XMLOutputFormat
 	Base64OutputFormat
+	UriOutputFormat
 )
 
 func OutputFormatFromString(format string) (PrinterOutputFormat, error) {

From 3b1bcac5b30765f11bc09bac7cc7a26b8e532fed Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Wed, 25 Jan 2023 11:18:48 +1100
Subject: [PATCH 16/53] review feed back on URI

---
 pkg/yqlib/encoder_uri.go      | 2 +-
 pkg/yqlib/lexer_participle.go | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkg/yqlib/encoder_uri.go b/pkg/yqlib/encoder_uri.go
index e4141cb8..fdf58665 100644
--- a/pkg/yqlib/encoder_uri.go
+++ b/pkg/yqlib/encoder_uri.go
@@ -30,7 +30,7 @@ func (e *uriEncoder) PrintLeadingContent(writer io.Writer, content string) error
 func (e *uriEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error {
 	node := unwrapDoc(originalNode)
 	if guessTagFromCustomType(node) != "!!str" {
-		return fmt.Errorf("cannot encode %v as url, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag)
+		return fmt.Errorf("cannot encode %v as URI, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag)
 	}
 	_, err := writer.Write([]byte(url.QueryEscape(originalNode.Value)))
 	return err
diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go
index 547c1810..ff6b203e 100644
--- a/pkg/yqlib/lexer_participle.go
+++ b/pkg/yqlib/lexer_participle.go
@@ -78,8 +78,8 @@ var participleYqRules = []*participleYqRule{
 	{"Base64d", `@base64d`, decodeOp(Base64InputFormat), 0},
 	{"Base64", `@base64`, encodeWithIndent(Base64OutputFormat, 0), 0},
 
-	{"Urld", `@urid`, decodeOp(UriInputFormat), 0},
-	{"Url", `@uri`, encodeWithIndent(UriOutputFormat, 0), 0},
+	{"Urid", `@urid`, decodeOp(UriInputFormat), 0},
+	{"Uri", `@uri`, encodeWithIndent(UriOutputFormat, 0), 0},
 
 	{"LoadXML", `load_?xml|xml_?load`, loadOp(NewXMLDecoder(ConfiguredXMLPreferences), false), 0},
 

From 7eda4a5100df87770ddb429606058939905fcd5b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 30 Jan 2023 10:22:42 +1100
Subject: [PATCH 17/53] Bump github.com/fatih/color from 1.13.0 to 1.14.1
 (#1531)

Bumps [github.com/fatih/color](https://github.com/fatih/color) from 1.13.0 to 1.14.1.
- [Release notes](https://github.com/fatih/color/releases)
- [Commits](https://github.com/fatih/color/compare/v1.13.0...v1.14.1)

---
updated-dependencies:
- dependency-name: github.com/fatih/color
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod |  8 ++++----
 go.sum | 21 ++++++++++-----------
 2 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/go.mod b/go.mod
index 942b92b0..df72d60f 100644
--- a/go.mod
+++ b/go.mod
@@ -6,7 +6,7 @@ require (
 	github.com/alecthomas/repr v0.2.0
 	github.com/dimchansky/utfbom v1.1.1
 	github.com/elliotchance/orderedmap v1.5.0
-	github.com/fatih/color v1.13.0
+	github.com/fatih/color v1.14.1
 	github.com/goccy/go-json v0.10.0
 	github.com/goccy/go-yaml v1.9.8
 	github.com/jinzhu/copier v0.3.5
@@ -21,9 +21,9 @@ require (
 
 require (
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
-	github.com/mattn/go-colorable v0.1.12 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
-	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
+	golang.org/x/sys v0.3.0 // indirect
 	golang.org/x/text v0.3.8 // indirect
 	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 )
diff --git a/go.sum b/go.sum
index 3a4b4813..4b205c9e 100644
--- a/go.sum
+++ b/go.sum
@@ -13,8 +13,8 @@ github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/
 github.com/elliotchance/orderedmap v1.5.0 h1:1IsExUsjv5XNBD3ZdC7jkAAqLWOOKdbPTmkHx63OsBg=
 github.com/elliotchance/orderedmap v1.5.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
-github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
-github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
+github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w=
+github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
@@ -32,12 +32,12 @@ github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgx
 github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
 github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
-github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
-github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
+github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -60,11 +60,10 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg=
-golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
+golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=

From d119dbc2396d345f5a5ddcc13553db7cba65cf50 Mon Sep 17 00:00:00 2001
From: Arturo Fernandez <134515+bsnux@users.noreply.github.com>
Date: Wed, 1 Feb 2023 16:56:36 -0800
Subject: [PATCH 18/53] Upgrading golang.org/x/net package (#1540)

---
 go.mod | 4 ++--
 go.sum | 8 ++++----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/go.mod b/go.mod
index df72d60f..b8261b85 100644
--- a/go.mod
+++ b/go.mod
@@ -14,7 +14,7 @@ require (
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/pflag v1.0.5
-	golang.org/x/net v0.0.0-20220906165146-f3363e06e74c
+	golang.org/x/net v0.1.1-0.20221104162952-702349b0e862
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -24,7 +24,7 @@ require (
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
 	golang.org/x/sys v0.3.0 // indirect
-	golang.org/x/text v0.3.8 // indirect
+	golang.org/x/text v0.4.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 )
 
diff --git a/go.sum b/go.sum
index 4b205c9e..e457357d 100644
--- a/go.sum
+++ b/go.sum
@@ -54,8 +54,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20220906165146-f3363e06e74c h1:yKufUcDwucU5urd+50/Opbt4AYpqthk7wHpHok8f1lo=
-golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.1.1-0.20221104162952-702349b0e862 h1:KrLJ+iz8J6j6VVr/OCfULAcK+xozUmWE43fKpMR4MlI=
+golang.org/x/net v0.1.1-0.20221104162952-702349b0e862/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -66,8 +66,8 @@ golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
-golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0=

From d21bb920d6c566f39d7a9a2c8e73d8f097f7451a Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Thu, 2 Feb 2023 12:22:52 +1100
Subject: [PATCH 19/53] Added shell string encoder (@sh) #1526

---
 pkg/yqlib/doc/operators/encode-decode.md      | 17 +++++++
 .../doc/operators/headers/encode-decode.md    |  1 +
 pkg/yqlib/encoder_sh.go                       | 44 +++++++++++++++++++
 pkg/yqlib/lexer_participle.go                 |  1 +
 pkg/yqlib/operator_encoder_decoder.go         |  2 +
 pkg/yqlib/operator_encoder_decoder_test.go    |  9 ++++
 pkg/yqlib/printer.go                          |  1 +
 7 files changed, 75 insertions(+)
 create mode 100644 pkg/yqlib/encoder_sh.go

diff --git a/pkg/yqlib/doc/operators/encode-decode.md b/pkg/yqlib/doc/operators/encode-decode.md
index f6be772c..93b7abb4 100644
--- a/pkg/yqlib/doc/operators/encode-decode.md
+++ b/pkg/yqlib/doc/operators/encode-decode.md
@@ -17,6 +17,7 @@ These operators are useful to process yaml documents that have stringified embed
 | XML | from_xml/@xmld | to_xml(i)/@xml |
 | Base64 | @base64d | @base64 |
 | URI | @urid | @uri |
+| Shell |  | @sh |
 
 
 See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) for accepted formats.
@@ -464,6 +465,22 @@ will output
 this has & special () characters *
 ```
 
+## Encode a string to sh
+Sh/Bash friendly string
+
+Given a sample.yml file of:
+```yaml
+coolData: strings with spaces and a 'quote'
+```
+then
+```bash
+yq '.coolData | @sh' sample.yml
+```
+will output
+```yaml
+'strings with spaces and a \'quote\''
+```
+
 ## Decode a base64 encoded string
 Decoded data is assumed to be a string.
 
diff --git a/pkg/yqlib/doc/operators/headers/encode-decode.md b/pkg/yqlib/doc/operators/headers/encode-decode.md
index 31c1c10b..b5108de4 100644
--- a/pkg/yqlib/doc/operators/headers/encode-decode.md
+++ b/pkg/yqlib/doc/operators/headers/encode-decode.md
@@ -17,6 +17,7 @@ These operators are useful to process yaml documents that have stringified embed
 | XML | from_xml/@xmld | to_xml(i)/@xml |
 | Base64 | @base64d | @base64 |
 | URI | @urid | @uri |
+| Shell |  | @sh |
 
 
 See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) for accepted formats.
diff --git a/pkg/yqlib/encoder_sh.go b/pkg/yqlib/encoder_sh.go
new file mode 100644
index 00000000..53663158
--- /dev/null
+++ b/pkg/yqlib/encoder_sh.go
@@ -0,0 +1,44 @@
+package yqlib
+
+import (
+	"fmt"
+	"io"
+	"regexp"
+	"strings"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+var pattern = regexp.MustCompile(`[^\w@%+=:,./-]`)
+
+type shEncoder struct {
+}
+
+func NewShEncoder() Encoder {
+	return &shEncoder{}
+}
+
+func (e *shEncoder) CanHandleAliases() bool {
+	return false
+}
+
+func (e *shEncoder) PrintDocumentSeparator(writer io.Writer) error {
+	return nil
+}
+
+func (e *shEncoder) PrintLeadingContent(writer io.Writer, content string) error {
+	return nil
+}
+
+func (e *shEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error {
+	node := unwrapDoc(originalNode)
+	if guessTagFromCustomType(node) != "!!str" {
+		return fmt.Errorf("cannot encode %v as URI, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag)
+	}
+
+	value := originalNode.Value
+	if pattern.MatchString(value) {
+		value = "'" + strings.ReplaceAll(value, "'", "\\'") + "'"
+	}
+	return writeString(writer, value)
+}
diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go
index ff6b203e..09e9dd75 100644
--- a/pkg/yqlib/lexer_participle.go
+++ b/pkg/yqlib/lexer_participle.go
@@ -80,6 +80,7 @@ var participleYqRules = []*participleYqRule{
 
 	{"Urid", `@urid`, decodeOp(UriInputFormat), 0},
 	{"Uri", `@uri`, encodeWithIndent(UriOutputFormat, 0), 0},
+	{"SH", `@sh`, encodeWithIndent(ShOutputFormat, 0), 0},
 
 	{"LoadXML", `load_?xml|xml_?load`, loadOp(NewXMLDecoder(ConfiguredXMLPreferences), false), 0},
 
diff --git a/pkg/yqlib/operator_encoder_decoder.go b/pkg/yqlib/operator_encoder_decoder.go
index 2fa0cb05..255b0664 100644
--- a/pkg/yqlib/operator_encoder_decoder.go
+++ b/pkg/yqlib/operator_encoder_decoder.go
@@ -28,6 +28,8 @@ func configureEncoder(format PrinterOutputFormat, indent int) Encoder {
 		return NewBase64Encoder()
 	case UriOutputFormat:
 		return NewUriEncoder()
+	case ShOutputFormat:
+		return NewShEncoder()
 	}
 	panic("invalid encoder")
 }
diff --git a/pkg/yqlib/operator_encoder_decoder_test.go b/pkg/yqlib/operator_encoder_decoder_test.go
index ee54868e..7801e6cf 100644
--- a/pkg/yqlib/operator_encoder_decoder_test.go
+++ b/pkg/yqlib/operator_encoder_decoder_test.go
@@ -257,6 +257,15 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!str)::this has & special () characters *\n",
 		},
 	},
+	{
+		description:    "Encode a string to sh",
+		subdescription: "Sh/Bash friendly string",
+		document:       "coolData: strings with spaces and a 'quote'",
+		expression:     ".coolData | @sh",
+		expected: []string{
+			"D0, P[coolData], (!!str)::'strings with spaces and a \\'quote\\''\n",
+		},
+	},
 	{
 		description:    "Decode a base64 encoded string",
 		subdescription: "Decoded data is assumed to be a string.",
diff --git a/pkg/yqlib/printer.go b/pkg/yqlib/printer.go
index 14f96cc3..802793a9 100644
--- a/pkg/yqlib/printer.go
+++ b/pkg/yqlib/printer.go
@@ -28,6 +28,7 @@ const (
 	XMLOutputFormat
 	Base64OutputFormat
 	UriOutputFormat
+	ShOutputFormat
 )
 
 func OutputFormatFromString(format string) (PrinterOutputFormat, error) {

From 75920481b1128a9715975d4ab0c631eeab37d6c4 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Thu, 2 Feb 2023 12:42:36 +1100
Subject: [PATCH 20/53] Added from_unix operator #1535

---
 pkg/yqlib/doc/operators/datetime.md | 12 +++++++++
 pkg/yqlib/lexer_participle.go       |  1 +
 pkg/yqlib/lib.go                    |  1 +
 pkg/yqlib/operator_datetime.go      | 41 +++++++++++++++++++++++++++++
 pkg/yqlib/operator_datetime_test.go |  8 ++++++
 5 files changed, 63 insertions(+)

diff --git a/pkg/yqlib/doc/operators/datetime.md b/pkg/yqlib/doc/operators/datetime.md
index 0dc3ed5e..3a2fb72d 100644
--- a/pkg/yqlib/doc/operators/datetime.md
+++ b/pkg/yqlib/doc/operators/datetime.md
@@ -86,6 +86,18 @@ a: cool
 updated: 2021-05-19T01:02:03Z
 ```
 
+## From Unix
+Converts from unix time
+
+Running
+```bash
+yq --null-input '1675301929 | from_unix'
+```
+will output
+```yaml
+2023-02-02T12:38:49+11:00
+```
+
 ## Timezone: from standard RFC3339 format
 Returns a new datetime in the specified timezone. Specify standard IANA Time Zone format or 'utc', 'local'. When given a single parameter, this assumes the datetime is in RFC3339 format.
 
diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go
index 09e9dd75..cedbd110 100644
--- a/pkg/yqlib/lexer_participle.go
+++ b/pkg/yqlib/lexer_participle.go
@@ -45,6 +45,7 @@ var participleYqRules = []*participleYqRule{
 	simpleOp("format_datetime", formatDateTimeOpType),
 	simpleOp("now", nowOpType),
 	simpleOp("tz", tzOpType),
+	simpleOp("from_?unix", fromUnixOpType),
 	simpleOp("with_dtf", withDtFormatOpType),
 	simpleOp("error", errorOpType),
 	simpleOp("sortKeys", sortKeysOpType),
diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go
index 1bb7f405..12b299fd 100644
--- a/pkg/yqlib/lib.go
+++ b/pkg/yqlib/lib.go
@@ -93,6 +93,7 @@ var formatDateTimeOpType = &operationType{Type: "FORMAT_DATE_TIME", NumArgs: 1,
 var withDtFormatOpType = &operationType{Type: "WITH_DATE_TIME_FORMAT", NumArgs: 1, Precedence: 50, Handler: withDateTimeFormat}
 var nowOpType = &operationType{Type: "NOW", NumArgs: 0, Precedence: 50, Handler: nowOp}
 var tzOpType = &operationType{Type: "TIMEZONE", NumArgs: 1, Precedence: 50, Handler: tzOp}
+var fromUnixOpType = &operationType{Type: "FROM_UNIX", NumArgs: 0, Precedence: 50, Handler: fromUnixOp}
 
 var encodeOpType = &operationType{Type: "ENCODE", NumArgs: 0, Precedence: 50, Handler: encodeOperator}
 var decodeOpType = &operationType{Type: "DECODE", NumArgs: 0, Precedence: 50, Handler: decodeOperator}
diff --git a/pkg/yqlib/operator_datetime.go b/pkg/yqlib/operator_datetime.go
index c3ec05f6..f639c540 100644
--- a/pkg/yqlib/operator_datetime.go
+++ b/pkg/yqlib/operator_datetime.go
@@ -4,6 +4,7 @@ import (
 	"container/list"
 	"errors"
 	"fmt"
+	"strconv"
 	"time"
 
 	"gopkg.in/yaml.v3"
@@ -129,3 +130,43 @@ func tzOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode)
 
 	return context.ChildContext(results), nil
 }
+
+func parseUnixTime(unixTime string) (time.Time, error) {
+	seconds, err := strconv.ParseFloat(unixTime, 64)
+
+	if err != nil {
+		return time.Now(), err
+	}
+
+	return time.UnixMilli(int64(seconds * 1000)), nil
+}
+
+func fromUnixOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+
+		actualTag := guessTagFromCustomType(candidate.Node)
+
+		if actualTag != "!!int" && guessTagFromCustomType(candidate.Node) != "!!float" {
+			return Context{}, fmt.Errorf("from_unix only works on numbers, found %v instead", candidate.Node.Tag)
+		}
+
+		parsedTime, err := parseUnixTime(candidate.Node.Value)
+		if err != nil {
+			return Context{}, err
+		}
+
+		node := &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!timestamp",
+			Value: parsedTime.Format(time.RFC3339),
+		}
+
+		results.PushBack(candidate.CreateReplacement(node))
+	}
+
+	return context.ChildContext(results), nil
+}
diff --git a/pkg/yqlib/operator_datetime_test.go b/pkg/yqlib/operator_datetime_test.go
index a1d2692f..74b81390 100644
--- a/pkg/yqlib/operator_datetime_test.go
+++ b/pkg/yqlib/operator_datetime_test.go
@@ -39,6 +39,14 @@ var dateTimeOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: cool\nupdated: 2021-05-19T01:02:03Z\n",
 		},
 	},
+	{
+		description:    "From Unix",
+		subdescription: "Converts from unix time",
+		expression:     `1675301929 | from_unix`,
+		expected: []string{
+			"D0, P[], (!!timestamp)::2023-02-02T12:38:49+11:00\n",
+		},
+	},
 	{
 		description:    "Timezone: from standard RFC3339 format",
 		subdescription: "Returns a new datetime in the specified timezone. Specify standard IANA Time Zone format or 'utc', 'local'. When given a single parameter, this assumes the datetime is in RFC3339 format.",

From f9f340b6bfc514c29741ddb2f205d5a28ca43c92 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Thu, 2 Feb 2023 12:47:59 +1100
Subject: [PATCH 21/53] Github pipeline not in AU tz :D

---
 pkg/yqlib/doc/operators/datetime.md | 4 ++--
 pkg/yqlib/operator_datetime_test.go | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/pkg/yqlib/doc/operators/datetime.md b/pkg/yqlib/doc/operators/datetime.md
index 3a2fb72d..7bf9c24d 100644
--- a/pkg/yqlib/doc/operators/datetime.md
+++ b/pkg/yqlib/doc/operators/datetime.md
@@ -91,11 +91,11 @@ Converts from unix time
 
 Running
 ```bash
-yq --null-input '1675301929 | from_unix'
+yq --null-input '1675301929 | from_unix | tz("UTC")'
 ```
 will output
 ```yaml
-2023-02-02T12:38:49+11:00
+2023-02-02T01:38:49Z
 ```
 
 ## Timezone: from standard RFC3339 format
diff --git a/pkg/yqlib/operator_datetime_test.go b/pkg/yqlib/operator_datetime_test.go
index 74b81390..b326a254 100644
--- a/pkg/yqlib/operator_datetime_test.go
+++ b/pkg/yqlib/operator_datetime_test.go
@@ -42,9 +42,9 @@ var dateTimeOperatorScenarios = []expressionScenario{
 	{
 		description:    "From Unix",
 		subdescription: "Converts from unix time",
-		expression:     `1675301929 | from_unix`,
+		expression:     `1675301929 | from_unix | tz("UTC")`,
 		expected: []string{
-			"D0, P[], (!!timestamp)::2023-02-02T12:38:49+11:00\n",
+			"D0, P[], (!!timestamp)::2023-02-02T01:38:49Z\n",
 		},
 	},
 	{

From 915ab699220522c540fd34a8753d233d5de77841 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Thu, 2 Feb 2023 12:56:16 +1100
Subject: [PATCH 22/53] Added to_unix operator

---
 pkg/yqlib/doc/operators/datetime.md | 14 +++++++++++++-
 pkg/yqlib/lexer_participle.go       |  1 +
 pkg/yqlib/lib.go                    |  1 +
 pkg/yqlib/operator_datetime.go      | 26 ++++++++++++++++++++++++++
 pkg/yqlib/operator_datetime_test.go | 10 +++++++++-
 5 files changed, 50 insertions(+), 2 deletions(-)

diff --git a/pkg/yqlib/doc/operators/datetime.md b/pkg/yqlib/doc/operators/datetime.md
index 7bf9c24d..bd40053e 100644
--- a/pkg/yqlib/doc/operators/datetime.md
+++ b/pkg/yqlib/doc/operators/datetime.md
@@ -87,7 +87,7 @@ updated: 2021-05-19T01:02:03Z
 ```
 
 ## From Unix
-Converts from unix time
+Converts from unix time. Note, you don't have to pipe through the tz operator :)
 
 Running
 ```bash
@@ -98,6 +98,18 @@ will output
 2023-02-02T01:38:49Z
 ```
 
+## To Unix
+Converts to unix time
+
+Running
+```bash
+yq --null-input 'now | to_unix'
+```
+will output
+```yaml
+1621386123
+```
+
 ## Timezone: from standard RFC3339 format
 Returns a new datetime in the specified timezone. Specify standard IANA Time Zone format or 'utc', 'local'. When given a single parameter, this assumes the datetime is in RFC3339 format.
 
diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go
index cedbd110..7e2f2284 100644
--- a/pkg/yqlib/lexer_participle.go
+++ b/pkg/yqlib/lexer_participle.go
@@ -46,6 +46,7 @@ var participleYqRules = []*participleYqRule{
 	simpleOp("now", nowOpType),
 	simpleOp("tz", tzOpType),
 	simpleOp("from_?unix", fromUnixOpType),
+	simpleOp("to_?unix", toUnixOpType),
 	simpleOp("with_dtf", withDtFormatOpType),
 	simpleOp("error", errorOpType),
 	simpleOp("sortKeys", sortKeysOpType),
diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go
index 12b299fd..0325f6af 100644
--- a/pkg/yqlib/lib.go
+++ b/pkg/yqlib/lib.go
@@ -94,6 +94,7 @@ var withDtFormatOpType = &operationType{Type: "WITH_DATE_TIME_FORMAT", NumArgs:
 var nowOpType = &operationType{Type: "NOW", NumArgs: 0, Precedence: 50, Handler: nowOp}
 var tzOpType = &operationType{Type: "TIMEZONE", NumArgs: 1, Precedence: 50, Handler: tzOp}
 var fromUnixOpType = &operationType{Type: "FROM_UNIX", NumArgs: 0, Precedence: 50, Handler: fromUnixOp}
+var toUnixOpType = &operationType{Type: "TO_UNIX", NumArgs: 0, Precedence: 50, Handler: toUnixOp}
 
 var encodeOpType = &operationType{Type: "ENCODE", NumArgs: 0, Precedence: 50, Handler: encodeOperator}
 var decodeOpType = &operationType{Type: "DECODE", NumArgs: 0, Precedence: 50, Handler: decodeOperator}
diff --git a/pkg/yqlib/operator_datetime.go b/pkg/yqlib/operator_datetime.go
index f639c540..25bafa9e 100644
--- a/pkg/yqlib/operator_datetime.go
+++ b/pkg/yqlib/operator_datetime.go
@@ -170,3 +170,29 @@ func fromUnixOp(d *dataTreeNavigator, context Context, expressionNode *Expressio
 
 	return context.ChildContext(results), nil
 }
+
+func toUnixOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	layout := context.GetDateTimeLayout()
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+
+		parsedTime, err := parseDateTime(layout, candidate.Node.Value)
+		if err != nil {
+			return Context{}, fmt.Errorf("could not parse datetime of [%v] using layout [%v]: %w", candidate.GetNicePath(), layout, err)
+		}
+
+		node := &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!int",
+			Value: fmt.Sprintf("%v", parsedTime.Unix()),
+		}
+
+		results.PushBack(candidate.CreateReplacement(node))
+	}
+
+	return context.ChildContext(results), nil
+}
diff --git a/pkg/yqlib/operator_datetime_test.go b/pkg/yqlib/operator_datetime_test.go
index b326a254..a2d3ca17 100644
--- a/pkg/yqlib/operator_datetime_test.go
+++ b/pkg/yqlib/operator_datetime_test.go
@@ -41,12 +41,20 @@ var dateTimeOperatorScenarios = []expressionScenario{
 	},
 	{
 		description:    "From Unix",
-		subdescription: "Converts from unix time",
+		subdescription: "Converts from unix time. Note, you don't have to pipe through the tz operator :)",
 		expression:     `1675301929 | from_unix | tz("UTC")`,
 		expected: []string{
 			"D0, P[], (!!timestamp)::2023-02-02T01:38:49Z\n",
 		},
 	},
+	{
+		description:    "To Unix",
+		subdescription: "Converts to unix time",
+		expression:     `now | to_unix`,
+		expected: []string{
+			"D0, P[], (!!int)::1621386123\n",
+		},
+	},
 	{
 		description:    "Timezone: from standard RFC3339 format",
 		subdescription: "Returns a new datetime in the specified timezone. Specify standard IANA Time Zone format or 'utc', 'local'. When given a single parameter, this assumes the datetime is in RFC3339 format.",

From 88a6b20ba5d8c265ae4e455444f2d76d4953834b Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Thu, 2 Feb 2023 13:30:48 +1100
Subject: [PATCH 23/53] Fixed date comparison with string date #1537

---
 pkg/yqlib/{compare_operators.go => operator_compare.go}    | 3 +--
 ...compare_operators_test.go => operators_compare_test.go} | 7 +++++++
 2 files changed, 8 insertions(+), 2 deletions(-)
 rename pkg/yqlib/{compare_operators.go => operator_compare.go} (98%)
 rename pkg/yqlib/{compare_operators_test.go => operators_compare_test.go} (97%)

diff --git a/pkg/yqlib/compare_operators.go b/pkg/yqlib/operator_compare.go
similarity index 98%
rename from pkg/yqlib/compare_operators.go
rename to pkg/yqlib/operator_compare.go
index 904e1202..a33cfecf 100644
--- a/pkg/yqlib/compare_operators.go
+++ b/pkg/yqlib/operator_compare.go
@@ -3,7 +3,6 @@ package yqlib
 import (
 	"fmt"
 	"strconv"
-	"time"
 
 	yaml "gopkg.in/yaml.v3"
 )
@@ -80,7 +79,7 @@ func compareScalars(context Context, prefs compareTypePref, lhs *yaml.Node, rhs
 
 	isDateTime := lhs.Tag == "!!timestamp"
 	// if the lhs is a string, it might be a timestamp in a custom format.
-	if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
+	if lhsTag == "!!str" {
 		_, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value)
 		isDateTime = err == nil
 	}
diff --git a/pkg/yqlib/compare_operators_test.go b/pkg/yqlib/operators_compare_test.go
similarity index 97%
rename from pkg/yqlib/compare_operators_test.go
rename to pkg/yqlib/operators_compare_test.go
index 0c13e241..cc7d5fb8 100644
--- a/pkg/yqlib/compare_operators_test.go
+++ b/pkg/yqlib/operators_compare_test.go
@@ -21,6 +21,13 @@ var compareOperatorScenarios = []expressionScenario{
 			"D0, P[k], (!!bool)::true\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		expression: `"2022-01-30T15:53:09Z" > "2020-01-30T15:53:09Z"`,
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   "a: 5\nb: 4",

From f64f73a0ce7ef3346893835f942c3bb0e5e663ca Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Thu, 9 Feb 2023 18:13:18 +1100
Subject: [PATCH 24/53] Updated gosec

---
 scripts/devtools.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/devtools.sh b/scripts/devtools.sh
index d9feeedd..98ff0507 100755
--- a/scripts/devtools.sh
+++ b/scripts/devtools.sh
@@ -2,4 +2,4 @@
 set -ex
 go mod download golang.org/x/tools@latest
 curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.49.0
-wget -O- -nv https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.13.1
+wget -O- -nv https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.15.0

From 93b7c999be110c668325c46f48de2d077d00fbd2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?V=C3=ADt=20Zikmund?=
 <75443448+vit-zikmund@users.noreply.github.com>
Date: Thu, 9 Feb 2023 08:15:07 +0100
Subject: [PATCH 25/53] Use a lazy-quoting @sh encoder (#1548)

* Use a lazy-quoting @sh encoder

* Add internal quoting style switch to @sh

* Add test for stray empty quotes in @sh
---
 pkg/yqlib/doc/operators/encode-decode.md   |  2 +-
 pkg/yqlib/encoder_sh.go                    | 49 ++++++++++++++++++----
 pkg/yqlib/operator_encoder_decoder_test.go | 12 +++++-
 3 files changed, 54 insertions(+), 9 deletions(-)

diff --git a/pkg/yqlib/doc/operators/encode-decode.md b/pkg/yqlib/doc/operators/encode-decode.md
index 93b7abb4..b8cc1f75 100644
--- a/pkg/yqlib/doc/operators/encode-decode.md
+++ b/pkg/yqlib/doc/operators/encode-decode.md
@@ -478,7 +478,7 @@ yq '.coolData | @sh' sample.yml
 ```
 will output
 ```yaml
-'strings with spaces and a \'quote\''
+strings' with spaces and a '\'quote\'
 ```
 
 ## Decode a base64 encoded string
diff --git a/pkg/yqlib/encoder_sh.go b/pkg/yqlib/encoder_sh.go
index 53663158..1cee7fa0 100644
--- a/pkg/yqlib/encoder_sh.go
+++ b/pkg/yqlib/encoder_sh.go
@@ -9,13 +9,14 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
-var pattern = regexp.MustCompile(`[^\w@%+=:,./-]`)
+var unsafeChars = regexp.MustCompile(`[^\w@%+=:,./-]`)
 
 type shEncoder struct {
+	quoteAll bool
 }
 
 func NewShEncoder() Encoder {
-	return &shEncoder{}
+	return &shEncoder{false}
 }
 
 func (e *shEncoder) CanHandleAliases() bool {
@@ -36,9 +37,43 @@ func (e *shEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error {
 		return fmt.Errorf("cannot encode %v as URI, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag)
 	}
 
-	value := originalNode.Value
-	if pattern.MatchString(value) {
-		value = "'" + strings.ReplaceAll(value, "'", "\\'") + "'"
-	}
-	return writeString(writer, value)
+	return writeString(writer, e.encode(originalNode.Value))
+}
+
+// put any (shell-unsafe) characters into a single-quoted block, close the block lazily
+func (e *shEncoder) encode(input string) string {
+	const quote = '\''
+	var inQuoteBlock = false
+	var encoded strings.Builder
+	encoded.Grow(len(input))
+
+	for _, ir := range input {
+		// open or close a single-quote block
+		if ir == quote {
+			if inQuoteBlock {
+				// get out of a quote block for an input quote
+				encoded.WriteRune(quote)
+				inQuoteBlock = !inQuoteBlock
+			}
+			// escape the quote with a backslash
+			encoded.WriteRune('\\')
+		} else {
+			if e.shouldQuote(ir) && !inQuoteBlock {
+				// start a quote block for any (unsafe) characters
+				encoded.WriteRune(quote)
+				inQuoteBlock = !inQuoteBlock
+			}
+		}
+		// pass on the input character
+		encoded.WriteRune(ir)
+	}
+	// close any pending quote block
+	if inQuoteBlock {
+		encoded.WriteRune(quote)
+	}
+	return encoded.String()
+}
+
+func (e *shEncoder) shouldQuote(ir rune) bool {
+	return e.quoteAll || unsafeChars.MatchString(string(ir))
 }
diff --git a/pkg/yqlib/operator_encoder_decoder_test.go b/pkg/yqlib/operator_encoder_decoder_test.go
index 7801e6cf..13448adf 100644
--- a/pkg/yqlib/operator_encoder_decoder_test.go
+++ b/pkg/yqlib/operator_encoder_decoder_test.go
@@ -263,9 +263,19 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 		document:       "coolData: strings with spaces and a 'quote'",
 		expression:     ".coolData | @sh",
 		expected: []string{
-			"D0, P[coolData], (!!str)::'strings with spaces and a \\'quote\\''\n",
+			"D0, P[coolData], (!!str)::strings' with spaces and a '\\'quote\\'\n",
 		},
 	},
+	{
+		description:    "Encode a string to sh",
+		subdescription: "Watch out for stray '' (empty strings)",
+		document:       "coolData: \"'starts, contains more '' and ends with a quote'\"",
+		expression:     ".coolData | @sh",
+		expected: []string{
+			"D0, P[coolData], (!!str)::\\'starts,' contains more '\\'\\'' and ends with a quote'\\'\n",
+		},
+		skipDoc: true,
+	},
 	{
 		description:    "Decode a base64 encoded string",
 		subdescription: "Decoded data is assumed to be a string.",

From 18a51ca5b8b746cd7dfc1513b3a5f85afcd5e34b Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 9 Feb 2023 18:15:20 +1100
Subject: [PATCH 26/53] Bump github.com/a8m/envsubst from 1.3.0 to 1.4.1
 (#1551)

Bumps [github.com/a8m/envsubst](https://github.com/a8m/envsubst) from 1.3.0 to 1.4.1.
- [Release notes](https://github.com/a8m/envsubst/releases)
- [Commits](https://github.com/a8m/envsubst/compare/v1.3.0...v1.4.1)

---
updated-dependencies:
- dependency-name: github.com/a8m/envsubst
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index b8261b85..eeffaa40 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/mikefarah/yq/v4
 
 require (
-	github.com/a8m/envsubst v1.3.0
+	github.com/a8m/envsubst v1.4.1
 	github.com/alecthomas/participle/v2 v2.0.0-beta.5
 	github.com/alecthomas/repr v0.2.0
 	github.com/dimchansky/utfbom v1.1.1
diff --git a/go.sum b/go.sum
index e457357d..cf7fa672 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-github.com/a8m/envsubst v1.3.0 h1:GmXKmVssap0YtlU3E230W98RWtWCyIZzjtf1apWWyAg=
-github.com/a8m/envsubst v1.3.0/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
+github.com/a8m/envsubst v1.4.1 h1:RShc2OKbQpIIp5qR2glVsBOJCImTVHFrDkJvmGh0Qi0=
+github.com/a8m/envsubst v1.4.1/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
 github.com/alecthomas/assert/v2 v2.0.3 h1:WKqJODfOiQG0nEJKFKzDIG3E29CN2/4zR9XGJzKIkbg=
 github.com/alecthomas/participle/v2 v2.0.0-beta.5 h1:y6dsSYVb1G5eK6mgmy+BgI3Mw35a3WghArZ/Hbebrjo=
 github.com/alecthomas/participle/v2 v2.0.0-beta.5/go.mod h1:RC764t6n4L8D8ITAJv0qdokritYSNR3wV5cVwmIEaMM=

From bbb149b31e57b2ea55c1893b6868e797b78a7123 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Thu, 9 Feb 2023 18:20:11 +1100
Subject: [PATCH 27/53] Bumped linter

---
 scripts/devtools.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/devtools.sh b/scripts/devtools.sh
index 98ff0507..983fed52 100755
--- a/scripts/devtools.sh
+++ b/scripts/devtools.sh
@@ -1,5 +1,5 @@
 #!/bin/sh
 set -ex
 go mod download golang.org/x/tools@latest
-curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.49.0
+curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.51.1
 wget -O- -nv https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.15.0

From 0f2a84d270d969231abae346f690533aff217181 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Thu, 9 Feb 2023 18:20:25 +1100
Subject: [PATCH 28/53] Bump golang from 1.19.5 to 1.20.0 (#1542)

Bumps golang from 1.19.5 to 1.20.0.

---
updated-dependencies:
- dependency-name: golang
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 Dockerfile     | 2 +-
 Dockerfile.dev | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index d3dd13a4..2d10528c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.19.5 as builder
+FROM golang:1.20.0 as builder
 
 WORKDIR /go/src/mikefarah/yq
 
diff --git a/Dockerfile.dev b/Dockerfile.dev
index a2c77979..ef1f17eb 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -1,4 +1,4 @@
-FROM golang:1.19.5
+FROM golang:1.20.0
 
 COPY scripts/devtools.sh /opt/devtools.sh
 

From 5cb3c876fcfab56442c0b0823e23d2964322dd6e Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Sat, 11 Feb 2023 04:06:16 +1100
Subject: [PATCH 29/53] bump to go 1.20

---
 .github/workflows/go.yml      | 4 ++--
 .github/workflows/release.yml | 2 +-
 go.mod                        | 2 +-
 snap/snapcraft.yaml           | 2 +-
 4 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
index 32a9ae3d..6819a221 100644
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -10,10 +10,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.19
+    - name: Set up Go
       uses: actions/setup-go@v3
       with:
-        go-version: 1.19
+        go-version: 1.20
       id: go
 
     - name: Check out code into the Go module directory
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 87491f60..6bbc83c8 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -12,7 +12,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: '^1.19'
+          go-version: '^1.20'
           check-latest: true
       - name: Compile man page markup
         id: gen-man-page-md
diff --git a/go.mod b/go.mod
index eeffaa40..1d3c9613 100644
--- a/go.mod
+++ b/go.mod
@@ -28,4 +28,4 @@ require (
 	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 )
 
-go 1.19
+go 1.20
diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml
index cd8355bb..d6c978ae 100644
--- a/snap/snapcraft.yaml
+++ b/snap/snapcraft.yaml
@@ -13,7 +13,7 @@ apps:
 parts:
   yq:
     plugin: go
-    go-channel: 1.19/stable
+    go-channel: 1.20/stable
     source: .
     source-type: git
     go-importpath: github.com/mikefarah/yq

From a1698b740a7730c338db500cf5a3a287912e91d6 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Sat, 11 Feb 2023 04:44:15 +1100
Subject: [PATCH 30/53] Added ability to sort by multiple fields #1541

---
 pkg/yqlib/doc/operators/sort.md | 22 +++++++++
 pkg/yqlib/operator_sort.go      | 80 +++++++++++++++++++++------------
 pkg/yqlib/operator_sort_test.go | 26 +++++++++++
 3 files changed, 100 insertions(+), 28 deletions(-)

diff --git a/pkg/yqlib/doc/operators/sort.md b/pkg/yqlib/doc/operators/sort.md
index 51c79e4a..72a17305 100644
--- a/pkg/yqlib/doc/operators/sort.md
+++ b/pkg/yqlib/doc/operators/sort.md
@@ -25,6 +25,28 @@ will output
 - a: cat
 ```
 
+## Sort by multiple fields
+Given a sample.yml file of:
+```yaml
+- a: dog
+- a: cat
+  b: banana
+- a: cat
+  b: apple
+```
+then
+```bash
+yq 'sort_by(.a, .b)' sample.yml
+```
+will output
+```yaml
+- a: cat
+  b: apple
+- a: cat
+  b: banana
+- a: dog
+```
+
 ## Sort descending by string field
 Use sort with reverse to sort in descending order.
 
diff --git a/pkg/yqlib/operator_sort.go b/pkg/yqlib/operator_sort.go
index 62c85e53..01070f2d 100644
--- a/pkg/yqlib/operator_sort.go
+++ b/pkg/yqlib/operator_sort.go
@@ -42,18 +42,7 @@ func sortByOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 				return Context{}, err
 			}
 
-			nodeToCompare := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!null"}
-			if compareContext.MatchingNodes.Len() > 0 {
-				nodeToCompare = compareContext.MatchingNodes.Front().Value.(*CandidateNode).Node
-			}
-
-			log.Debug("going to compare %v by %v", NodeToString(candidate.CreateReplacement(originalNode)), NodeToString(candidate.CreateReplacement(nodeToCompare)))
-
-			sortableArray[i] = sortableNode{Node: originalNode, NodeToCompare: nodeToCompare, dateTimeLayout: context.GetDateTimeLayout()}
-
-			if nodeToCompare.Kind != yaml.ScalarNode {
-				return Context{}, fmt.Errorf("sort only works for scalars, got %v", nodeToCompare.Tag)
-			}
+			sortableArray[i] = sortableNode{Node: originalNode, CompareContext: compareContext, dateTimeLayout: context.GetDateTimeLayout()}
 
 		}
 
@@ -72,7 +61,7 @@ func sortByOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 
 type sortableNode struct {
 	Node           *yaml.Node
-	NodeToCompare  *yaml.Node
+	CompareContext Context
 	dateTimeLayout string
 }
 
@@ -82,9 +71,28 @@ func (a sortableNodeArray) Len() int      { return len(a) }
 func (a sortableNodeArray) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
 
 func (a sortableNodeArray) Less(i, j int) bool {
-	lhs := a[i].NodeToCompare
-	rhs := a[j].NodeToCompare
+	lhsContext := a[i].CompareContext
+	rhsContext := a[j].CompareContext
 
+	rhsEl := rhsContext.MatchingNodes.Front()
+	for lhsEl := lhsContext.MatchingNodes.Front(); lhsEl != nil && rhsEl != nil; lhsEl = lhsEl.Next() {
+		lhs := lhsEl.Value.(*CandidateNode)
+		rhs := rhsEl.Value.(*CandidateNode)
+
+		result := a.compare(lhs.Node, rhs.Node, a[i].dateTimeLayout)
+
+		if result < 0 {
+			return true
+		} else if result > 0 {
+			return false
+		}
+
+		rhsEl = rhsEl.Next()
+	}
+	return false
+}
+
+func (a sortableNodeArray) compare(lhs *yaml.Node, rhs *yaml.Node, dateTimeLayout string) int {
 	lhsTag := lhs.Tag
 	rhsTag := rhs.Tag
 
@@ -99,7 +107,7 @@ func (a sortableNodeArray) Less(i, j int) bool {
 	}
 
 	isDateTime := lhsTag == "!!timestamp" && rhsTag == "!!timestamp"
-	layout := a[i].dateTimeLayout
+	layout := dateTimeLayout
 	// if the lhs is a string, it might be a timestamp in a custom format.
 	if lhsTag == "!!str" && layout != time.RFC3339 {
 		_, errLhs := parseDateTime(layout, lhs.Value)
@@ -108,13 +116,13 @@ func (a sortableNodeArray) Less(i, j int) bool {
 	}
 
 	if lhsTag == "!!null" && rhsTag != "!!null" {
-		return true
+		return -1
 	} else if lhsTag != "!!null" && rhsTag == "!!null" {
-		return false
+		return 1
 	} else if lhsTag == "!!bool" && rhsTag != "!!bool" {
-		return true
+		return -1
 	} else if lhsTag != "!!bool" && rhsTag == "!!bool" {
-		return false
+		return 1
 	} else if lhsTag == "!!bool" && rhsTag == "!!bool" {
 		lhsTruthy, err := isTruthyNode(lhs)
 		if err != nil {
@@ -125,20 +133,30 @@ func (a sortableNodeArray) Less(i, j int) bool {
 		if err != nil {
 			panic(fmt.Errorf("could not parse %v as boolean: %w", rhs.Value, err))
 		}
-
-		return !lhsTruthy && rhsTruthy
+		if lhsTruthy == rhsTruthy {
+			return 0
+		} else if lhsTruthy {
+			return 1
+		}
+		return -1
 	} else if isDateTime {
 		lhsTime, err := parseDateTime(layout, lhs.Value)
 		if err != nil {
 			log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %w", lhs.Value, layout, err)
-			return strings.Compare(lhs.Value, rhs.Value) < 0
+			return strings.Compare(lhs.Value, rhs.Value)
 		}
 		rhsTime, err := parseDateTime(layout, rhs.Value)
 		if err != nil {
 			log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %w", rhs.Value, layout, err)
-			return strings.Compare(lhs.Value, rhs.Value) < 0
+			return strings.Compare(lhs.Value, rhs.Value)
 		}
-		return lhsTime.Before(rhsTime)
+		if lhsTime.Equal(rhsTime) {
+			return 0
+		} else if lhsTime.Before(rhsTime) {
+			return -1
+		}
+
+		return 1
 	} else if lhsTag == "!!int" && rhsTag == "!!int" {
 		_, lhsNum, err := parseInt64(lhs.Value)
 		if err != nil {
@@ -148,7 +166,7 @@ func (a sortableNodeArray) Less(i, j int) bool {
 		if err != nil {
 			panic(err)
 		}
-		return lhsNum < rhsNum
+		return int(lhsNum - rhsNum)
 	} else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") {
 		lhsNum, err := strconv.ParseFloat(lhs.Value, 64)
 		if err != nil {
@@ -158,8 +176,14 @@ func (a sortableNodeArray) Less(i, j int) bool {
 		if err != nil {
 			panic(err)
 		}
-		return lhsNum < rhsNum
+		if lhsNum == rhsNum {
+			return 0
+		} else if lhsNum < rhsNum {
+			return -1
+		}
+
+		return 1
 	}
 
-	return strings.Compare(lhs.Value, rhs.Value) < 0
+	return strings.Compare(lhs.Value, rhs.Value)
 }
diff --git a/pkg/yqlib/operator_sort_test.go b/pkg/yqlib/operator_sort_test.go
index 21e5c990..9a8bc84a 100644
--- a/pkg/yqlib/operator_sort_test.go
+++ b/pkg/yqlib/operator_sort_test.go
@@ -11,6 +11,32 @@ var sortByOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::[{a: apple}, {a: banana}, {a: cat}]\n",
 		},
 	},
+	{
+		description: "Sort by multiple fields",
+		document:    "[{a: dog},{a: cat, b: banana},{a: cat, b: apple}]",
+		expression:  `sort_by(.a, .b)`,
+		expected: []string{
+			"D0, P[], (!!seq)::[{a: cat, b: apple}, {a: cat, b: banana}, {a: dog}]\n",
+		},
+	},
+	{
+		description: "Sort by multiple fields",
+		skipDoc:     true,
+		document:    "[{a: dog, b: good},{a: cat, c: things},{a: cat, b: apple}]",
+		expression:  `sort_by(.a, .b)`,
+		expected: []string{
+			"D0, P[], (!!seq)::[{a: cat, c: things}, {a: cat, b: apple}, {a: dog, b: good}]\n",
+		},
+	},
+	{
+		description: "Sort by multiple fields",
+		skipDoc:     true,
+		document:    "[{a: dog, b: 0.1},{a: cat, b: 0.01},{a: cat, b: 0.001}]",
+		expression:  `sort_by(.a, .b)`,
+		expected: []string{
+			"D0, P[], (!!seq)::[{a: cat, b: 0.001}, {a: cat, b: 0.01}, {a: dog, b: 0.1}]\n",
+		},
+	},
 	{
 		description:    "Sort descending by string field",
 		subdescription: "Use sort with reverse to sort in descending order.",

From d17fd9424e19ac70e5058a77855e55807ac0698e Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Sat, 11 Feb 2023 05:08:20 +1100
Subject: [PATCH 31/53] Added shuffle command #1503

---
 pkg/yqlib/doc/operators/headers/shuffle.md |  4 ++
 pkg/yqlib/doc/operators/shuffle.md         | 51 ++++++++++++++++++++++
 pkg/yqlib/lexer_participle.go              |  1 +
 pkg/yqlib/lib.go                           |  1 +
 pkg/yqlib/operator_shuffle.go              | 37 ++++++++++++++++
 pkg/yqlib/operator_shuffle_test.go         | 30 +++++++++++++
 6 files changed, 124 insertions(+)
 create mode 100644 pkg/yqlib/doc/operators/headers/shuffle.md
 create mode 100644 pkg/yqlib/doc/operators/shuffle.md
 create mode 100644 pkg/yqlib/operator_shuffle.go
 create mode 100644 pkg/yqlib/operator_shuffle_test.go

diff --git a/pkg/yqlib/doc/operators/headers/shuffle.md b/pkg/yqlib/doc/operators/headers/shuffle.md
new file mode 100644
index 00000000..3fc66481
--- /dev/null
+++ b/pkg/yqlib/doc/operators/headers/shuffle.md
@@ -0,0 +1,4 @@
+# Shuffle
+
+Shuffles an array. Note that this command does _not_ use a cryptographically secure random number generator to randomise the array order.
+
diff --git a/pkg/yqlib/doc/operators/shuffle.md b/pkg/yqlib/doc/operators/shuffle.md
new file mode 100644
index 00000000..953b513f
--- /dev/null
+++ b/pkg/yqlib/doc/operators/shuffle.md
@@ -0,0 +1,51 @@
+# Shuffle
+
+Shuffles an array. Note that this command does _not_ use a cryptographically secure random number generator to randomise the array order.
+
+
+## Shuffle array
+Given a sample.yml file of:
+```yaml
+- 1
+- 2
+- 3
+- 4
+- 5
+```
+then
+```bash
+yq 'shuffle' sample.yml
+```
+will output
+```yaml
+- 5
+- 2
+- 4
+- 1
+- 3
+```
+
+## Shuffle array in place
+Given a sample.yml file of:
+```yaml
+cool:
+  - 1
+  - 2
+  - 3
+  - 4
+  - 5
+```
+then
+```bash
+yq '.cool |= shuffle' sample.yml
+```
+will output
+```yaml
+cool:
+  - 5
+  - 2
+  - 4
+  - 1
+  - 3
+```
+
diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go
index 7e2f2284..33756fef 100644
--- a/pkg/yqlib/lexer_participle.go
+++ b/pkg/yqlib/lexer_participle.go
@@ -49,6 +49,7 @@ var participleYqRules = []*participleYqRule{
 	simpleOp("to_?unix", toUnixOpType),
 	simpleOp("with_dtf", withDtFormatOpType),
 	simpleOp("error", errorOpType),
+	simpleOp("shuffle", shuffleOpType),
 	simpleOp("sortKeys", sortKeysOpType),
 	simpleOp("sort_?keys", sortKeysOpType),
 
diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go
index 0325f6af..8b61eae3 100644
--- a/pkg/yqlib/lib.go
+++ b/pkg/yqlib/lib.go
@@ -135,6 +135,7 @@ var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50,
 var sortByOpType = &operationType{Type: "SORT_BY", NumArgs: 1, Precedence: 50, Handler: sortByOperator}
 var reverseOpType = &operationType{Type: "REVERSE", NumArgs: 0, Precedence: 50, Handler: reverseOperator}
 var sortOpType = &operationType{Type: "SORT", NumArgs: 0, Precedence: 50, Handler: sortOperator}
+var shuffleOpType = &operationType{Type: "SHUFFLE", NumArgs: 0, Precedence: 50, Handler: shuffleOperator}
 
 var sortKeysOpType = &operationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: sortKeysOperator}
 
diff --git a/pkg/yqlib/operator_shuffle.go b/pkg/yqlib/operator_shuffle.go
new file mode 100644
index 00000000..73a76965
--- /dev/null
+++ b/pkg/yqlib/operator_shuffle.go
@@ -0,0 +1,37 @@
+package yqlib
+
+import (
+	"container/list"
+	"fmt"
+	"math/rand"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+func shuffleOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	// ignore CWE-338 gosec issue of not using crypto/rand
+	// this is just to shuffle an array rather generating a
+	// secret or something that needs proper rand.
+	myRand := rand.New(rand.NewSource(Now().UnixNano())) // #nosec
+
+	results := list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+
+		candidateNode := unwrapDoc(candidate.Node)
+
+		if candidateNode.Kind != yaml.SequenceNode {
+			return context, fmt.Errorf("node at path [%v] is not an array (it's a %v)", candidate.GetNicePath(), candidate.GetNiceTag())
+		}
+
+		result := deepClone(candidateNode)
+
+		a := result.Content
+
+		myRand.Shuffle(len(a), func(i, j int) { a[i], a[j] = a[j], a[i] })
+		results.PushBack(candidate.CreateReplacement(result))
+	}
+	return context.ChildContext(results), nil
+}
diff --git a/pkg/yqlib/operator_shuffle_test.go b/pkg/yqlib/operator_shuffle_test.go
new file mode 100644
index 00000000..23a3748b
--- /dev/null
+++ b/pkg/yqlib/operator_shuffle_test.go
@@ -0,0 +1,30 @@
+package yqlib
+
+import "testing"
+
+var shuffleOperatorScenarios = []expressionScenario{
+	{
+		description: "Shuffle array",
+		document:    "[1, 2, 3, 4, 5]",
+		expression:  `shuffle`,
+		expected: []string{
+			"D0, P[], (!!seq)::[5, 2, 4, 1, 3]\n",
+		},
+	},
+
+	{
+		description: "Shuffle array in place",
+		document:    "cool: [1, 2, 3, 4, 5]",
+		expression:  `.cool |= shuffle`,
+		expected: []string{
+			"D0, P[], (doc)::cool: [5, 2, 4, 1, 3]\n",
+		},
+	},
+}
+
+func TestShuffleByOperatorScenarios(t *testing.T) {
+	for _, tt := range shuffleOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentOperatorScenarios(t, "shuffle", shuffleOperatorScenarios)
+}

From f4e7203a558befb7176c910a5504abc596127132 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Sat, 11 Feb 2023 23:50:47 +1100
Subject: [PATCH 32/53] Fix github action attempt

---
 .github/workflows/go.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
index 6819a221..0e4ab093 100644
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -13,7 +13,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v3
       with:
-        go-version: 1.20
+        go-version: '^1.20'
       id: go
 
     - name: Check out code into the Go module directory

From 75483fe908b997f46fc728b8d1ed39fe944f5782 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Sat, 11 Feb 2023 23:54:16 +1100
Subject: [PATCH 33/53] Fix github action attempt

---
 scripts/devtools.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/devtools.sh b/scripts/devtools.sh
index 983fed52..01d76711 100755
--- a/scripts/devtools.sh
+++ b/scripts/devtools.sh
@@ -2,4 +2,4 @@
 set -ex
 go mod download golang.org/x/tools@latest
 curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.51.1
-wget -O- -nv https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.15.0
+wget -O- -nv https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s

From b369de6662ddd785a783e7bea53ec28845cc2e57 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 20 Feb 2023 15:59:50 +1100
Subject: [PATCH 34/53] Bump golang from 1.20.0 to 1.20.1 (#1557)

Bumps golang from 1.20.0 to 1.20.1.

---
updated-dependencies:
- dependency-name: golang
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 Dockerfile     | 2 +-
 Dockerfile.dev | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 2d10528c..b5fd237f 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.20.0 as builder
+FROM golang:1.20.1 as builder
 
 WORKDIR /go/src/mikefarah/yq
 
diff --git a/Dockerfile.dev b/Dockerfile.dev
index ef1f17eb..1bd9517c 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -1,4 +1,4 @@
-FROM golang:1.20.0
+FROM golang:1.20.1
 
 COPY scripts/devtools.sh /opt/devtools.sh
 

From b86fb0aea032661bbc9e8d774d4fdd1350b9a76d Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Mon, 20 Feb 2023 16:09:07 +1100
Subject: [PATCH 35/53] Bumping version

---
 cmd/version.go      | 2 +-
 snap/snapcraft.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd/version.go b/cmd/version.go
index 05977638..14b17304 100644
--- a/cmd/version.go
+++ b/cmd/version.go
@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "v4.30.8"
+	Version = "v4.31.1"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release
diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml
index d6c978ae..5d0b528b 100644
--- a/snap/snapcraft.yaml
+++ b/snap/snapcraft.yaml
@@ -1,5 +1,5 @@
 name: yq
-version: 'v4.30.8'
+version: 'v4.31.1'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.

From 9949a237245595739c363f0c165e7d8df32b0d85 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Mon, 20 Feb 2023 16:09:25 +1100
Subject: [PATCH 36/53] v4.31.1

---
 release_notes.txt | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/release_notes.txt b/release_notes.txt
index 3cc78097..f4aab574 100644
--- a/release_notes.txt
+++ b/release_notes.txt
@@ -1,3 +1,12 @@
+4.31.1:
+  - Added shuffle command #1503
+  - Added ability to sort by multiple fields #1541
+  - Added @sh encoder #1526
+  - Added @uri/@urid encoder/decoder #1529
+  - Fixed date comparison with string date #1537
+  - Added from_unix/to_unix Operators
+  - Bumped dependency versions
+
 4.30.8:
  - Log info message when unable to chown file in linux (e.g. snap confinement) #1521
 

From ce3d8378e7cdc0bd85755ea9bdd3cd940bce6152 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 28 Feb 2023 13:28:21 +1100
Subject: [PATCH 37/53] Bump github.com/a8m/envsubst from 1.4.1 to 1.4.2
 (#1569)

Bumps [github.com/a8m/envsubst](https://github.com/a8m/envsubst) from 1.4.1 to 1.4.2.
- [Release notes](https://github.com/a8m/envsubst/releases)
- [Commits](https://github.com/a8m/envsubst/compare/v1.4.1...v1.4.2)

---
updated-dependencies:
- dependency-name: github.com/a8m/envsubst
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index 1d3c9613..33669405 100644
--- a/go.mod
+++ b/go.mod
@@ -1,7 +1,7 @@
 module github.com/mikefarah/yq/v4
 
 require (
-	github.com/a8m/envsubst v1.4.1
+	github.com/a8m/envsubst v1.4.2
 	github.com/alecthomas/participle/v2 v2.0.0-beta.5
 	github.com/alecthomas/repr v0.2.0
 	github.com/dimchansky/utfbom v1.1.1
diff --git a/go.sum b/go.sum
index cf7fa672..0dbf8342 100644
--- a/go.sum
+++ b/go.sum
@@ -1,5 +1,5 @@
-github.com/a8m/envsubst v1.4.1 h1:RShc2OKbQpIIp5qR2glVsBOJCImTVHFrDkJvmGh0Qi0=
-github.com/a8m/envsubst v1.4.1/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
+github.com/a8m/envsubst v1.4.2 h1:4yWIHXOLEJHQEFd4UjrWDrYeYlV7ncFWJOCBRLOZHQg=
+github.com/a8m/envsubst v1.4.2/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
 github.com/alecthomas/assert/v2 v2.0.3 h1:WKqJODfOiQG0nEJKFKzDIG3E29CN2/4zR9XGJzKIkbg=
 github.com/alecthomas/participle/v2 v2.0.0-beta.5 h1:y6dsSYVb1G5eK6mgmy+BgI3Mw35a3WghArZ/Hbebrjo=
 github.com/alecthomas/participle/v2 v2.0.0-beta.5/go.mod h1:RC764t6n4L8D8ITAJv0qdokritYSNR3wV5cVwmIEaMM=

From cb27444e54e8d970497281e69435633f074afd98 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 28 Feb 2023 13:38:09 +1100
Subject: [PATCH 38/53] Bump golang.org/x/net from
 0.1.1-0.20221104162952-702349b0e862 to 0.7.0 (#1576)

Bumps [golang.org/x/net](https://github.com/golang/net) from 0.1.1-0.20221104162952-702349b0e862 to 0.7.0.
- [Release notes](https://github.com/golang/net/releases)
- [Commits](https://github.com/golang/net/commits/v0.7.0)

---
updated-dependencies:
- dependency-name: golang.org/x/net
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod |  6 +++---
 go.sum | 12 ++++++------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/go.mod b/go.mod
index 33669405..d340e9f5 100644
--- a/go.mod
+++ b/go.mod
@@ -14,7 +14,7 @@ require (
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/pflag v1.0.5
-	golang.org/x/net v0.1.1-0.20221104162952-702349b0e862
+	golang.org/x/net v0.7.0
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -23,8 +23,8 @@ require (
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
-	golang.org/x/sys v0.3.0 // indirect
-	golang.org/x/text v0.4.0 // indirect
+	golang.org/x/sys v0.5.0 // indirect
+	golang.org/x/text v0.7.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 )
 
diff --git a/go.sum b/go.sum
index 0dbf8342..68287a2a 100644
--- a/go.sum
+++ b/go.sum
@@ -54,20 +54,20 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.1.1-0.20221104162952-702349b0e862 h1:KrLJ+iz8J6j6VVr/OCfULAcK+xozUmWE43fKpMR4MlI=
-golang.org/x/net v0.1.1-0.20221104162952-702349b0e862/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
+golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
-golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
-golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0=

From 62d167c1414616a87b8871c7a9f8310e03ac443a Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Tue, 28 Feb 2023 16:40:38 +1100
Subject: [PATCH 39/53] Variable loop - Fixes #1566 (#1577)

* Variable loop wip

* Variable loop wip

* Variable loop wip

* Variable loop wip

* Fixed variable operator to work like jq
---
 pkg/yqlib/lib.go                         |  2 +-
 pkg/yqlib/operator_add_test.go           |  2 +-
 pkg/yqlib/operator_alternative_test.go   |  2 +-
 pkg/yqlib/operator_booleans_test.go      | 18 ++++--
 pkg/yqlib/operator_equals_test.go        |  4 +-
 pkg/yqlib/operator_has_test.go           |  2 +-
 pkg/yqlib/operator_pipe.go               |  6 +-
 pkg/yqlib/operator_subtract_test.go      |  2 +-
 pkg/yqlib/operator_traverse_path_test.go |  2 +-
 pkg/yqlib/operator_union_test.go         |  2 +-
 pkg/yqlib/operator_variables.go          | 81 ++++++++++++++++++++----
 pkg/yqlib/operator_variables_test.go     | 20 +++++-
 12 files changed, 112 insertions(+), 31 deletions(-)

diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go
index 8b61eae3..65389263 100644
--- a/pkg/yqlib/lib.go
+++ b/pkg/yqlib/lib.go
@@ -53,7 +53,7 @@ var subtractAssignOpType = &operationType{Type: "SUBTRACT_ASSIGN", NumArgs: 2, P
 
 var assignAttributesOpType = &operationType{Type: "ASSIGN_ATTRIBUTES", NumArgs: 2, Precedence: 40, Handler: assignAttributesOperator}
 var assignStyleOpType = &operationType{Type: "ASSIGN_STYLE", NumArgs: 2, Precedence: 40, Handler: assignStyleOperator}
-var assignVariableOpType = &operationType{Type: "ASSIGN_VARIABLE", NumArgs: 2, Precedence: 40, Handler: assignVariableOperator}
+var assignVariableOpType = &operationType{Type: "ASSIGN_VARIABLE", NumArgs: 2, Precedence: 40, Handler: useWithPipe}
 var assignTagOpType = &operationType{Type: "ASSIGN_TAG", NumArgs: 2, Precedence: 40, Handler: assignTagOperator}
 var assignCommentOpType = &operationType{Type: "ASSIGN_COMMENT", NumArgs: 2, Precedence: 40, Handler: assignCommentsOperator}
 var assignAnchorOpType = &operationType{Type: "ASSIGN_ANCHOR", NumArgs: 2, Precedence: 40, Handler: assignAnchorOperator}
diff --git a/pkg/yqlib/operator_add_test.go b/pkg/yqlib/operator_add_test.go
index 99a84845..eeabffde 100644
--- a/pkg/yqlib/operator_add_test.go
+++ b/pkg/yqlib/operator_add_test.go
@@ -34,7 +34,7 @@ var addOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{}`,
-		expression: "(.a + .b) as $x",
+		expression: "(.a + .b) as $x | .",
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},
diff --git a/pkg/yqlib/operator_alternative_test.go b/pkg/yqlib/operator_alternative_test.go
index 74106639..e70b8b25 100644
--- a/pkg/yqlib/operator_alternative_test.go
+++ b/pkg/yqlib/operator_alternative_test.go
@@ -16,7 +16,7 @@ var alternativeOperatorScenarios = []expressionScenario{
 	},
 	{
 		skipDoc:    true,
-		expression: `(.b // "hello") as $x`,
+		expression: `(.b // "hello") as $x | .`,
 		document:   `a: bridge`,
 		expected: []string{
 			"D0, P[], (doc)::a: bridge\n",
diff --git a/pkg/yqlib/operator_booleans_test.go b/pkg/yqlib/operator_booleans_test.go
index f5735bb2..dc219ad1 100644
--- a/pkg/yqlib/operator_booleans_test.go
+++ b/pkg/yqlib/operator_booleans_test.go
@@ -102,7 +102,7 @@ var booleanOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `[{pet: cat}]`,
-		expression: `any_c(.name == "harry") as $c`,
+		expression: `any_c(.name == "harry") as $c | .`,
 		expected: []string{
 			"D0, P[], (doc)::[{pet: cat}]\n",
 		},
@@ -110,9 +110,17 @@ var booleanOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `[{pet: cat}]`,
-		expression: `all_c(.name == "harry") as $c`,
+		expression: `any_c(.name == "harry") as $c | $c`,
 		expected: []string{
-			"D0, P[], (doc)::[{pet: cat}]\n",
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{pet: cat}]`,
+		expression: `all_c(.name == "harry") as $c | $c`,
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
 		},
 	},
 	{
@@ -185,7 +193,7 @@ var booleanOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{}`,
-		expression: `(.a.b or .c) as $x`,
+		expression: `(.a.b or .c) as $x | .`,
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},
@@ -193,7 +201,7 @@ var booleanOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{}`,
-		expression: `(.a.b and .c) as $x`,
+		expression: `(.a.b and .c) as $x | .`,
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},
diff --git a/pkg/yqlib/operator_equals_test.go b/pkg/yqlib/operator_equals_test.go
index 37fe5005..7a821647 100644
--- a/pkg/yqlib/operator_equals_test.go
+++ b/pkg/yqlib/operator_equals_test.go
@@ -47,7 +47,7 @@ var equalsOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   "{}",
-		expression: "(.a == .b) as $x",
+		expression: "(.a == .b) as $x | .",
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},
@@ -63,7 +63,7 @@ var equalsOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   "{}",
-		expression: "(.a != .b) as $x",
+		expression: "(.a != .b) as $x | .",
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},
diff --git a/pkg/yqlib/operator_has_test.go b/pkg/yqlib/operator_has_test.go
index 3a631330..47504048 100644
--- a/pkg/yqlib/operator_has_test.go
+++ b/pkg/yqlib/operator_has_test.go
@@ -16,7 +16,7 @@ var hasOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `a: hello`,
-		expression: `has(.b) as $c`,
+		expression: `has(.b) as $c | .`,
 		expected: []string{
 			"D0, P[], (doc)::a: hello\n",
 		},
diff --git a/pkg/yqlib/operator_pipe.go b/pkg/yqlib/operator_pipe.go
index 77813c67..e1a90d91 100644
--- a/pkg/yqlib/operator_pipe.go
+++ b/pkg/yqlib/operator_pipe.go
@@ -2,9 +2,9 @@ package yqlib
 
 func pipeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 
-	//lhs may update the variable context, we should pass that into the RHS
-	// BUT we still return the original context back (see jq)
-	// https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|...
+	if expressionNode.LHS.Operation.OperationType == assignVariableOpType {
+		return variableLoop(d, context, expressionNode)
+	}
 
 	lhs, err := d.GetMatchingNodes(context, expressionNode.LHS)
 	if err != nil {
diff --git a/pkg/yqlib/operator_subtract_test.go b/pkg/yqlib/operator_subtract_test.go
index aaaa91f6..e5134b06 100644
--- a/pkg/yqlib/operator_subtract_test.go
+++ b/pkg/yqlib/operator_subtract_test.go
@@ -8,7 +8,7 @@ var subtractOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{}`,
-		expression: "(.a - .b) as $x",
+		expression: "(.a - .b) as $x | .",
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},
diff --git a/pkg/yqlib/operator_traverse_path_test.go b/pkg/yqlib/operator_traverse_path_test.go
index 405a523e..8d3e94ad 100644
--- a/pkg/yqlib/operator_traverse_path_test.go
+++ b/pkg/yqlib/operator_traverse_path_test.go
@@ -151,7 +151,7 @@ var traversePathOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `c: dog`,
-		expression: `.[.a.b] as $x`,
+		expression: `.[.a.b] as $x | .`,
 		expected: []string{
 			"D0, P[], (doc)::c: dog\n",
 		},
diff --git a/pkg/yqlib/operator_union_test.go b/pkg/yqlib/operator_union_test.go
index ce12f19a..4b98ac06 100644
--- a/pkg/yqlib/operator_union_test.go
+++ b/pkg/yqlib/operator_union_test.go
@@ -8,7 +8,7 @@ var unionOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   "{}",
-		expression: `(.a, .b.c) as $x`,
+		expression: `(.a, .b.c) as $x | .`,
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},
diff --git a/pkg/yqlib/operator_variables.go b/pkg/yqlib/operator_variables.go
index 5550bded..d9baded9 100644
--- a/pkg/yqlib/operator_variables.go
+++ b/pkg/yqlib/operator_variables.go
@@ -19,24 +19,81 @@ type assignVarPreferences struct {
 	IsReference bool
 }
 
-func assignVariableOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	lhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.LHS)
+func useWithPipe(d *dataTreeNavigator, context Context, originalExp *ExpressionNode) (Context, error) {
+	return Context{}, fmt.Errorf("must use variable with a pipe, e.g. `exp as $x | ...`")
+}
+
+// variables are like loops in jq
+// https://stedolan.github.io/jq/manual/#Variable
+func variableLoop(d *dataTreeNavigator, context Context, originalExp *ExpressionNode) (Context, error) {
+	log.Debug("variable loop!")
+	results := list.New()
+	var evaluateAllTogether = true
+	for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {
+		evaluateAllTogether = evaluateAllTogether && matchEl.Value.(*CandidateNode).EvaluateTogether
+		if !evaluateAllTogether {
+			break
+		}
+	}
+	if evaluateAllTogether {
+		return variableLoopSingleChild(d, context, originalExp)
+	}
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		result, err := variableLoopSingleChild(d, context.SingleChildContext(el.Value.(*CandidateNode)), originalExp)
+		if err != nil {
+			return Context{}, err
+		}
+		results.PushBackList(result.MatchingNodes)
+	}
+	return context.ChildContext(results), nil
+}
+
+func variableLoopSingleChild(d *dataTreeNavigator, context Context, originalExp *ExpressionNode) (Context, error) {
+
+	variableExp := originalExp.LHS
+	lhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), variableExp.LHS)
 	if err != nil {
 		return Context{}, err
 	}
-	if expressionNode.RHS.Operation.OperationType.Type != "GET_VARIABLE" {
+	if variableExp.RHS.Operation.OperationType.Type != "GET_VARIABLE" {
 		return Context{}, fmt.Errorf("RHS of 'as' operator must be a variable name e.g. $foo")
 	}
-	variableName := expressionNode.RHS.Operation.StringValue
+	variableName := variableExp.RHS.Operation.StringValue
 
-	prefs := expressionNode.Operation.Preferences.(assignVarPreferences)
+	prefs := variableExp.Operation.Preferences.(assignVarPreferences)
 
-	var variableValue *list.List
-	if prefs.IsReference {
-		variableValue = lhs.MatchingNodes
-	} else {
-		variableValue = lhs.DeepClone().MatchingNodes
+	results := list.New()
+
+	// now we loop over lhs, set variable to each result and calculate originalExp.Rhs
+	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
+		log.Debug("PROCESSING VARIABLE: ", NodeToString(el.Value.(*CandidateNode)))
+		var variableValue = list.New()
+		if prefs.IsReference {
+			variableValue.PushBack(el.Value)
+		} else {
+			candidateCopy, err := el.Value.(*CandidateNode).Copy()
+			if err != nil {
+				return Context{}, err
+			}
+			variableValue.PushBack(candidateCopy)
+		}
+		newContext := context.ChildContext(context.MatchingNodes)
+		newContext.SetVariable(variableName, variableValue)
+
+		rhs, err := d.GetMatchingNodes(newContext, originalExp.RHS)
+		log.Debug("PROCESSING VARIABLE DONE, got back: ", rhs.MatchingNodes.Len())
+		if err != nil {
+			return Context{}, err
+		}
+		results.PushBackList(rhs.MatchingNodes)
 	}
-	context.SetVariable(variableName, variableValue)
-	return context, nil
+
+	// if there is no LHS - then I guess we just calculate originalExp.Rhs
+	if lhs.MatchingNodes.Len() == 0 {
+		return d.GetMatchingNodes(context, originalExp.RHS)
+	}
+
+	return context.ChildContext(results), nil
+
 }
diff --git a/pkg/yqlib/operator_variables_test.go b/pkg/yqlib/operator_variables_test.go
index 7ab9ee30..64643ab0 100644
--- a/pkg/yqlib/operator_variables_test.go
+++ b/pkg/yqlib/operator_variables_test.go
@@ -8,7 +8,7 @@ var variableOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{}`,
-		expression: `.a.b as $foo`,
+		expression: `.a.b as $foo | .`,
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},
@@ -16,7 +16,7 @@ var variableOperatorScenarios = []expressionScenario{
 	{
 		document:      "a: [cat]",
 		skipDoc:       true,
-		expression:    "(.[] | {.name: .}) as $item",
+		expression:    "(.[] | {.name: .}) as $item | .",
 		expectedError: `cannot index array with 'name' (strconv.ParseInt: parsing "name": invalid syntax)`,
 	},
 	{
@@ -36,6 +36,22 @@ var variableOperatorScenarios = []expressionScenario{
 			"D0, P[1], (!!str)::dog\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `[1, 2]`,
+		expression: `.[] | . as $f | select($f == 2)`,
+		expected: []string{
+			"D0, P[1], (!!int)::2\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[1, 2]`,
+		expression: `[.[] | . as $f | $f + 1]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- 2\n- 3\n",
+		},
+	},
 	{
 		description:    "Using variables as a lookup",
 		subdescription: "Example taken from [jq](https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|...)",

From cf8cfbd8656fec53814d88ad14fb00b3513b575e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Anders=20Bj=C3=B6rklund?= <anders.f.bjorklund@gmail.com>
Date: Wed, 1 Mar 2023 03:19:06 +0100
Subject: [PATCH 40/53] Allow build without json and xml support (#1556)

* Refactor ordered_map into separate files

Separate json and xml, from the regular yaml.

Makes it possible to compile, without those...

* Refactor encoder and decoder creation

Use more consistent parameters vs globals

Return errors instead of calling panic()

* Allow build without json and xml support
---
 cmd/evaluate_all_command.go                |   5 +-
 cmd/evalute_sequence_command.go            |   5 +-
 cmd/utils.go                               |  46 ++++--
 pkg/yqlib/decoder_json.go                  |   2 +
 pkg/yqlib/decoder_xml.go                   |   2 +
 pkg/yqlib/encoder.go                       | 163 +--------------------
 pkg/yqlib/encoder_json.go                  |  17 +--
 pkg/yqlib/encoder_test.go                  |   2 +
 pkg/yqlib/encoder_xml.go                   |   2 +
 pkg/yqlib/json_test.go                     |   2 +
 pkg/yqlib/no_json.go                       |  11 ++
 pkg/yqlib/no_xml.go                        |  11 ++
 pkg/yqlib/operator_encoder_decoder.go      |  26 +++-
 pkg/yqlib/operator_encoder_decoder_test.go |  45 +++---
 pkg/yqlib/operator_load.go                 |   3 +
 pkg/yqlib/operator_load_test.go            |   7 +-
 pkg/yqlib/operators_test.go                |  18 +++
 pkg/yqlib/ordered_map.go                   |  14 ++
 pkg/yqlib/ordered_map_json.go              |  83 +++++++++++
 pkg/yqlib/ordered_map_yaml.go              |  79 ++++++++++
 pkg/yqlib/printer_test.go                  |   6 +-
 pkg/yqlib/xml_test.go                      |   2 +
 22 files changed, 338 insertions(+), 213 deletions(-)
 create mode 100644 pkg/yqlib/no_json.go
 create mode 100644 pkg/yqlib/no_xml.go
 create mode 100644 pkg/yqlib/ordered_map.go
 create mode 100644 pkg/yqlib/ordered_map_json.go
 create mode 100644 pkg/yqlib/ordered_map_yaml.go

diff --git a/cmd/evaluate_all_command.go b/cmd/evaluate_all_command.go
index 63b68871..fff64fa5 100644
--- a/cmd/evaluate_all_command.go
+++ b/cmd/evaluate_all_command.go
@@ -84,7 +84,10 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 	if err != nil {
 		return err
 	}
-	encoder := configureEncoder(format)
+	encoder, err := configureEncoder()
+	if err != nil {
+		return err
+	}
 
 	printer := yqlib.NewPrinter(encoder, printerWriter)
 
diff --git a/cmd/evalute_sequence_command.go b/cmd/evalute_sequence_command.go
index 53108165..dab803c3 100644
--- a/cmd/evalute_sequence_command.go
+++ b/cmd/evalute_sequence_command.go
@@ -93,7 +93,10 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 	if err != nil {
 		return err
 	}
-	encoder := configureEncoder(format)
+	encoder, err := configureEncoder()
+	if err != nil {
+		return err
+	}
 
 	printer := yqlib.NewPrinter(encoder, printerWriter)
 
diff --git a/cmd/utils.go b/cmd/utils.go
index 8dad7c30..671dd86b 100644
--- a/cmd/utils.go
+++ b/cmd/utils.go
@@ -61,7 +61,15 @@ func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
 	if err != nil {
 		return nil, err
 	}
-	switch yqlibInputFormat {
+	yqlibDecoder, err := createDecoder(yqlibInputFormat, evaluateTogether)
+	if yqlibDecoder == nil {
+		return nil, fmt.Errorf("no support for %s input format", inputFormat)
+	}
+	return yqlibDecoder, err
+}
+
+func createDecoder(format yqlib.InputFormat, evaluateTogether bool) (yqlib.Decoder, error) {
+	switch format {
 	case yqlib.XMLInputFormat:
 		return yqlib.NewXMLDecoder(yqlib.ConfiguredXMLPreferences), nil
 	case yqlib.PropertiesInputFormat:
@@ -72,10 +80,12 @@ func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
 		return yqlib.NewCSVObjectDecoder(','), nil
 	case yqlib.TSVObjectInputFormat:
 		return yqlib.NewCSVObjectDecoder('\t'), nil
+	case yqlib.YamlInputFormat:
+		prefs := yqlib.ConfiguredYamlPreferences
+		prefs.EvaluateTogether = evaluateTogether
+		return yqlib.NewYamlDecoder(prefs), nil
 	}
-	prefs := yqlib.ConfiguredYamlPreferences
-	prefs.EvaluateTogether = evaluateTogether
-	return yqlib.NewYamlDecoder(prefs), nil
+	return nil, fmt.Errorf("invalid decoder: %v", format)
 }
 
 func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yqlib.PrinterWriter, error) {
@@ -95,22 +105,34 @@ func configurePrinterWriter(format yqlib.PrinterOutputFormat, out io.Writer) (yq
 	return printerWriter, nil
 }
 
-func configureEncoder(format yqlib.PrinterOutputFormat) yqlib.Encoder {
+func configureEncoder() (yqlib.Encoder, error) {
+	yqlibOutputFormat, err := yqlib.OutputFormatFromString(outputFormat)
+	if err != nil {
+		return nil, err
+	}
+	yqlibEncoder, err := createEncoder(yqlibOutputFormat)
+	if yqlibEncoder == nil {
+		return nil, fmt.Errorf("no support for %s output format", outputFormat)
+	}
+	return yqlibEncoder, err
+}
+
+func createEncoder(format yqlib.PrinterOutputFormat) (yqlib.Encoder, error) {
 	switch format {
 	case yqlib.JSONOutputFormat:
-		return yqlib.NewJSONEncoder(indent, colorsEnabled, unwrapScalar)
+		return yqlib.NewJSONEncoder(indent, colorsEnabled, unwrapScalar), nil
 	case yqlib.PropsOutputFormat:
-		return yqlib.NewPropertiesEncoder(unwrapScalar)
+		return yqlib.NewPropertiesEncoder(unwrapScalar), nil
 	case yqlib.CSVOutputFormat:
-		return yqlib.NewCsvEncoder(',')
+		return yqlib.NewCsvEncoder(','), nil
 	case yqlib.TSVOutputFormat:
-		return yqlib.NewCsvEncoder('\t')
+		return yqlib.NewCsvEncoder('\t'), nil
 	case yqlib.YamlOutputFormat:
-		return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences)
+		return yqlib.NewYamlEncoder(indent, colorsEnabled, yqlib.ConfiguredYamlPreferences), nil
 	case yqlib.XMLOutputFormat:
-		return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences)
+		return yqlib.NewXMLEncoder(indent, yqlib.ConfiguredXMLPreferences), nil
 	}
-	panic("invalid encoder")
+	return nil, fmt.Errorf("invalid encoder: %v", format)
 }
 
 // this is a hack to enable backwards compatibility with githubactions (which pipe /dev/null into everything)
diff --git a/pkg/yqlib/decoder_json.go b/pkg/yqlib/decoder_json.go
index a8e1e605..35da6467 100644
--- a/pkg/yqlib/decoder_json.go
+++ b/pkg/yqlib/decoder_json.go
@@ -1,3 +1,5 @@
+//go:build !yq_nojson
+
 package yqlib
 
 import (
diff --git a/pkg/yqlib/decoder_xml.go b/pkg/yqlib/decoder_xml.go
index ac70e403..bfb18e12 100644
--- a/pkg/yqlib/decoder_xml.go
+++ b/pkg/yqlib/decoder_xml.go
@@ -1,3 +1,5 @@
+//go:build !yq_noxml
+
 package yqlib
 
 import (
diff --git a/pkg/yqlib/encoder.go b/pkg/yqlib/encoder.go
index e9fae6af..e0ecaccf 100644
--- a/pkg/yqlib/encoder.go
+++ b/pkg/yqlib/encoder.go
@@ -1,10 +1,6 @@
 package yqlib
 
 import (
-	"bytes"
-	"encoding/json"
-	"errors"
-	"fmt"
 	"io"
 
 	yaml "gopkg.in/yaml.v3"
@@ -17,162 +13,17 @@ type Encoder interface {
 	CanHandleAliases() bool
 }
 
-// orderedMap allows to marshal and unmarshal JSON and YAML values keeping the
-// order of keys and values in a map or an object.
-type orderedMap struct {
-	// if this is an object, kv != nil. If this is not an object, kv == nil.
-	kv     []orderedMapKV
-	altVal interface{}
-}
+func mapKeysToStrings(node *yaml.Node) {
 
-type orderedMapKV struct {
-	K string
-	V orderedMap
-}
-
-func (o *orderedMap) UnmarshalJSON(data []byte) error {
-	switch data[0] {
-	case '{':
-		// initialise so that even if the object is empty it is not nil
-		o.kv = []orderedMapKV{}
-
-		// create decoder
-		dec := json.NewDecoder(bytes.NewReader(data))
-		_, err := dec.Token() // open object
-		if err != nil {
-			return err
-		}
-
-		// cycle through k/v
-		var tok json.Token
-		for tok, err = dec.Token(); err == nil; tok, err = dec.Token() {
-			// we can expect two types: string or Delim. Delim automatically means
-			// that it is the closing bracket of the object, whereas string means
-			// that there is another key.
-			if _, ok := tok.(json.Delim); ok {
-				break
+	if node.Kind == yaml.MappingNode {
+		for index, child := range node.Content {
+			if index%2 == 0 { // its a map key
+				child.Tag = "!!str"
 			}
-			kv := orderedMapKV{
-				K: tok.(string),
-			}
-			if err := dec.Decode(&kv.V); err != nil {
-				return err
-			}
-			o.kv = append(o.kv, kv)
 		}
-		// unexpected error
-		if err != nil && !errors.Is(err, io.EOF) {
-			return err
-		}
-		return nil
-	case '[':
-		var res []*orderedMap
-		if err := json.Unmarshal(data, &res); err != nil {
-			return err
-		}
-		o.altVal = res
-		o.kv = nil
-		return nil
 	}
 
-	return json.Unmarshal(data, &o.altVal)
-}
-
-func (o orderedMap) MarshalJSON() ([]byte, error) {
-	buf := new(bytes.Buffer)
-	enc := json.NewEncoder(buf)
-	enc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
-	if o.kv == nil {
-		if err := enc.Encode(o.altVal); err != nil {
-			return nil, err
-		}
-		return buf.Bytes(), nil
-	}
-	buf.WriteByte('{')
-	for idx, el := range o.kv {
-		if err := enc.Encode(el.K); err != nil {
-			return nil, err
-		}
-		buf.WriteByte(':')
-		if err := enc.Encode(el.V); err != nil {
-			return nil, err
-		}
-		if idx != len(o.kv)-1 {
-			buf.WriteByte(',')
-		}
-	}
-	buf.WriteByte('}')
-	return buf.Bytes(), nil
-}
-
-func (o *orderedMap) UnmarshalYAML(node *yaml.Node) error {
-	switch node.Kind {
-	case yaml.DocumentNode:
-		if len(node.Content) == 0 {
-			return nil
-		}
-		return o.UnmarshalYAML(node.Content[0])
-	case yaml.AliasNode:
-		return o.UnmarshalYAML(node.Alias)
-	case yaml.ScalarNode:
-		return node.Decode(&o.altVal)
-	case yaml.MappingNode:
-		// set kv to non-nil
-		o.kv = []orderedMapKV{}
-		for i := 0; i < len(node.Content); i += 2 {
-			var key string
-			var val orderedMap
-			if err := node.Content[i].Decode(&key); err != nil {
-				return err
-			}
-			if err := node.Content[i+1].Decode(&val); err != nil {
-				return err
-			}
-			o.kv = append(o.kv, orderedMapKV{
-				K: key,
-				V: val,
-			})
-		}
-		return nil
-	case yaml.SequenceNode:
-		// note that this has to be a pointer, so that nulls can be represented.
-		var res []*orderedMap
-		if err := node.Decode(&res); err != nil {
-			return err
-		}
-		o.altVal = res
-		o.kv = nil
-		return nil
-	case 0:
-		// null
-		o.kv = nil
-		o.altVal = nil
-		return nil
-	default:
-		return fmt.Errorf("orderedMap: invalid yaml node")
+	for _, child := range node.Content {
+		mapKeysToStrings(child)
 	}
 }
-
-func (o *orderedMap) MarshalYAML() (interface{}, error) {
-	// fast path: kv is nil, use altVal
-	if o.kv == nil {
-		return o.altVal, nil
-	}
-	content := make([]*yaml.Node, 0, len(o.kv)*2)
-	for _, val := range o.kv {
-		n := new(yaml.Node)
-		if err := n.Encode(val.V); err != nil {
-			return nil, err
-		}
-		content = append(content, &yaml.Node{
-			Kind:  yaml.ScalarNode,
-			Tag:   "!!str",
-			Value: val.K,
-		}, n)
-	}
-	return &yaml.Node{
-		Kind:    yaml.MappingNode,
-		Tag:     "!!map",
-		Content: content,
-	}, nil
-}
diff --git a/pkg/yqlib/encoder_json.go b/pkg/yqlib/encoder_json.go
index 9f5b71ac..53c6bd06 100644
--- a/pkg/yqlib/encoder_json.go
+++ b/pkg/yqlib/encoder_json.go
@@ -1,3 +1,5 @@
+//go:build !yq_nojson
+
 package yqlib
 
 import (
@@ -14,21 +16,6 @@ type jsonEncoder struct {
 	UnwrapScalar bool
 }
 
-func mapKeysToStrings(node *yaml.Node) {
-
-	if node.Kind == yaml.MappingNode {
-		for index, child := range node.Content {
-			if index%2 == 0 { // its a map key
-				child.Tag = "!!str"
-			}
-		}
-	}
-
-	for _, child := range node.Content {
-		mapKeysToStrings(child)
-	}
-}
-
 func NewJSONEncoder(indent int, colorise bool, unwrapScalar bool) Encoder {
 	var indentString = ""
 
diff --git a/pkg/yqlib/encoder_test.go b/pkg/yqlib/encoder_test.go
index a17c51a9..cb384562 100644
--- a/pkg/yqlib/encoder_test.go
+++ b/pkg/yqlib/encoder_test.go
@@ -1,3 +1,5 @@
+//go:build !yq_nojson
+
 package yqlib
 
 import (
diff --git a/pkg/yqlib/encoder_xml.go b/pkg/yqlib/encoder_xml.go
index 1d2e7798..d60ede8f 100644
--- a/pkg/yqlib/encoder_xml.go
+++ b/pkg/yqlib/encoder_xml.go
@@ -1,3 +1,5 @@
+//go:build !yq_noxml
+
 package yqlib
 
 import (
diff --git a/pkg/yqlib/json_test.go b/pkg/yqlib/json_test.go
index ad1b6bb8..b2eddb58 100644
--- a/pkg/yqlib/json_test.go
+++ b/pkg/yqlib/json_test.go
@@ -1,3 +1,5 @@
+//go:build !yq_nojson
+
 package yqlib
 
 import (
diff --git a/pkg/yqlib/no_json.go b/pkg/yqlib/no_json.go
new file mode 100644
index 00000000..ae9d531a
--- /dev/null
+++ b/pkg/yqlib/no_json.go
@@ -0,0 +1,11 @@
+//go:build yq_nojson
+
+package yqlib
+
+func NewJSONDecoder() Decoder {
+	return nil
+}
+
+func NewJSONEncoder(indent int, colorise bool, unwrapScalar bool) Encoder {
+	return nil
+}
diff --git a/pkg/yqlib/no_xml.go b/pkg/yqlib/no_xml.go
new file mode 100644
index 00000000..d3f96bb6
--- /dev/null
+++ b/pkg/yqlib/no_xml.go
@@ -0,0 +1,11 @@
+//go:build yq_noxml
+
+package yqlib
+
+func NewXMLDecoder(prefs XmlPreferences) Decoder {
+	return nil
+}
+
+func NewXMLEncoder(indent int, prefs XmlPreferences) Encoder {
+	return nil
+}
diff --git a/pkg/yqlib/operator_encoder_decoder.go b/pkg/yqlib/operator_encoder_decoder.go
index 255b0664..1c9582d1 100644
--- a/pkg/yqlib/operator_encoder_decoder.go
+++ b/pkg/yqlib/operator_encoder_decoder.go
@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"container/list"
+	"errors"
 	"regexp"
 	"strings"
 
@@ -39,6 +40,9 @@ func encodeToString(candidate *CandidateNode, prefs encoderPreferences) (string,
 	log.Debug("printing with indent: %v", prefs.indent)
 
 	encoder := configureEncoder(prefs.format, prefs.indent)
+	if encoder == nil {
+		return "", errors.New("no support for output format")
+	}
 
 	printer := NewPrinter(encoder, NewSinglePrinterWriter(bufio.NewWriter(&output)))
 	err := printer.PrintResults(candidate.AsList())
@@ -98,13 +102,11 @@ type decoderPreferences struct {
 	format InputFormat
 }
 
-/* takes a string and decodes it back into an object */
-func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-
-	preferences := expressionNode.Operation.Preferences.(decoderPreferences)
-
+func createDecoder(format InputFormat) Decoder {
 	var decoder Decoder
-	switch preferences.format {
+	switch format {
+	case JsonInputFormat:
+		decoder = NewJSONDecoder()
 	case YamlInputFormat:
 		decoder = NewYamlDecoder(ConfiguredYamlPreferences)
 	case XMLInputFormat:
@@ -120,6 +122,18 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 	case UriInputFormat:
 		decoder = NewUriDecoder()
 	}
+	return decoder
+}
+
+/* takes a string and decodes it back into an object */
+func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	preferences := expressionNode.Operation.Preferences.(decoderPreferences)
+
+	decoder := createDecoder(preferences.format)
+	if decoder == nil {
+		return Context{}, errors.New("no support for input format")
+	}
 
 	var results = list.New()
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
diff --git a/pkg/yqlib/operator_encoder_decoder_test.go b/pkg/yqlib/operator_encoder_decoder_test.go
index 13448adf..00441f73 100644
--- a/pkg/yqlib/operator_encoder_decoder_test.go
+++ b/pkg/yqlib/operator_encoder_decoder_test.go
@@ -8,15 +8,17 @@ var prefix = "D0, P[], (doc)::a:\n    cool:\n        bob: dylan\n"
 
 var encoderDecoderOperatorScenarios = []expressionScenario{
 	{
-		description: "Encode value as json string",
-		document:    `{a: {cool: "thing"}}`,
-		expression:  `.b = (.a | to_json)`,
+		requiresFormat: "json",
+		description:    "Encode value as json string",
+		document:       `{a: {cool: "thing"}}`,
+		expression:     `.b = (.a | to_json)`,
 		expected: []string{
 			`D0, P[], (doc)::{a: {cool: "thing"}, b: "{\n  \"cool\": \"thing\"\n}\n"}
 `,
 		},
 	},
 	{
+		requiresFormat: "json",
 		description:    "Encode value as json string, on one line",
 		subdescription: "Pass in a 0 indent to print json on a single line.",
 		document:       `{a: {cool: "thing"}}`,
@@ -27,6 +29,7 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
+		requiresFormat: "json",
 		description:    "Encode value as json string, on one line shorthand",
 		subdescription: "Pass in a 0 indent to print json on a single line.",
 		document:       `{a: {cool: "thing"}}`,
@@ -37,6 +40,7 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
+		requiresFormat: "json",
 		description:    "Decode a json encoded string",
 		subdescription: "Keep in mind JSON is a subset of YAML. If you want idiomatic yaml, pipe through the style operator to clear out the JSON styling.",
 		document:       `a: '{"cool":"thing"}'`,
@@ -193,33 +197,37 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description: "Encode value as xml string",
-		document:    `{a: {cool: {foo: "bar", +@id: hi}}}`,
-		expression:  `.a | to_xml`,
+		requiresFormat: "xml",
+		description:    "Encode value as xml string",
+		document:       `{a: {cool: {foo: "bar", +@id: hi}}}`,
+		expression:     `.a | to_xml`,
 		expected: []string{
 			"D0, P[a], (!!str)::<cool id=\"hi\">\n  <foo>bar</foo>\n</cool>\n\n",
 		},
 	},
 	{
-		description: "Encode value as xml string on a single line",
-		document:    `{a: {cool: {foo: "bar", +@id: hi}}}`,
-		expression:  `.a | @xml`,
+		requiresFormat: "xml",
+		description:    "Encode value as xml string on a single line",
+		document:       `{a: {cool: {foo: "bar", +@id: hi}}}`,
+		expression:     `.a | @xml`,
 		expected: []string{
 			"D0, P[a], (!!str)::<cool id=\"hi\"><foo>bar</foo></cool>\n\n",
 		},
 	},
 	{
-		description: "Encode value as xml string with custom indentation",
-		document:    `{a: {cool: {foo: "bar", +@id: hi}}}`,
-		expression:  `{"cat": .a | to_xml(1)}`,
+		requiresFormat: "xml",
+		description:    "Encode value as xml string with custom indentation",
+		document:       `{a: {cool: {foo: "bar", +@id: hi}}}`,
+		expression:     `{"cat": .a | to_xml(1)}`,
 		expected: []string{
 			"D0, P[], (!!map)::cat: |\n    <cool id=\"hi\">\n     <foo>bar</foo>\n    </cool>\n",
 		},
 	},
 	{
-		description: "Decode a xml encoded string",
-		document:    `a: "<foo>bar</foo>"`,
-		expression:  `.b = (.a | from_xml)`,
+		requiresFormat: "xml",
+		description:    "Decode a xml encoded string",
+		document:       `a: "<foo>bar</foo>"`,
+		expression:     `.b = (.a | from_xml)`,
 		expected: []string{
 			"D0, P[], (doc)::a: \"<foo>bar</foo>\"\nb:\n    foo: bar\n",
 		},
@@ -303,9 +311,10 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description: "empty xml decode",
-		skipDoc:     true,
-		expression:  `"" | @xmld`,
+		requiresFormat: "xml",
+		description:    "empty xml decode",
+		skipDoc:        true,
+		expression:     `"" | @xmld`,
 		expected: []string{
 			"D0, P[], (!!null)::\n",
 		},
diff --git a/pkg/yqlib/operator_load.go b/pkg/yqlib/operator_load.go
index 7d083d5b..678854f6 100644
--- a/pkg/yqlib/operator_load.go
+++ b/pkg/yqlib/operator_load.go
@@ -34,6 +34,9 @@ func loadString(filename string) (*CandidateNode, error) {
 }
 
 func loadYaml(filename string, decoder Decoder) (*CandidateNode, error) {
+	if decoder == nil {
+		return nil, fmt.Errorf("could not load %s", filename)
+	}
 
 	file, err := os.Open(filename) // #nosec
 	if err != nil {
diff --git a/pkg/yqlib/operator_load_test.go b/pkg/yqlib/operator_load_test.go
index b27e8f70..5bb30321 100644
--- a/pkg/yqlib/operator_load_test.go
+++ b/pkg/yqlib/operator_load_test.go
@@ -74,9 +74,10 @@ var loadScenarios = []expressionScenario{
 		},
 	},
 	{
-		description: "Load from XML",
-		document:    "cool: things",
-		expression:  `.more_stuff = load_xml("../../examples/small.xml")`,
+		requiresFormat: "xml",
+		description:    "Load from XML",
+		document:       "cool: things",
+		expression:     `.more_stuff = load_xml("../../examples/small.xml")`,
 		expected: []string{
 			"D0, P[], (doc)::cool: things\nmore_stuff:\n    this: is some xml\n",
 		},
diff --git a/pkg/yqlib/operators_test.go b/pkg/yqlib/operators_test.go
index 79757895..c040cbed 100644
--- a/pkg/yqlib/operators_test.go
+++ b/pkg/yqlib/operators_test.go
@@ -28,6 +28,7 @@ type expressionScenario struct {
 	skipDoc               bool
 	expectedError         string
 	dontFormatInputForDoc bool // dont format input doc for documentation generation
+	requiresFormat        string
 }
 
 func TestMain(m *testing.M) {
@@ -103,6 +104,23 @@ func testScenario(t *testing.T, s *expressionScenario) {
 		return
 	}
 
+	if s.requiresFormat != "" {
+		format := s.requiresFormat
+		inputFormat, err := InputFormatFromString(format)
+		if err != nil {
+			t.Error(err)
+		}
+		if decoder := createDecoder(inputFormat); decoder == nil {
+			t.Skipf("no support for %s input format", format)
+		}
+		outputFormat, err := OutputFormatFromString(format)
+		if err != nil {
+			t.Error(err)
+		}
+		if encoder := configureEncoder(outputFormat, 4); encoder == nil {
+			t.Skipf("no support for %s output format", format)
+		}
+	}
 	if err != nil {
 		t.Error(fmt.Errorf("%w: %v: %v", err, s.description, s.expression))
 		return
diff --git a/pkg/yqlib/ordered_map.go b/pkg/yqlib/ordered_map.go
new file mode 100644
index 00000000..a783423b
--- /dev/null
+++ b/pkg/yqlib/ordered_map.go
@@ -0,0 +1,14 @@
+package yqlib
+
+// orderedMap allows to marshal and unmarshal JSON and YAML values keeping the
+// order of keys and values in a map or an object.
+type orderedMap struct {
+	// if this is an object, kv != nil. If this is not an object, kv == nil.
+	kv     []orderedMapKV
+	altVal interface{}
+}
+
+type orderedMapKV struct {
+	K string
+	V orderedMap
+}
diff --git a/pkg/yqlib/ordered_map_json.go b/pkg/yqlib/ordered_map_json.go
new file mode 100644
index 00000000..94a1d780
--- /dev/null
+++ b/pkg/yqlib/ordered_map_json.go
@@ -0,0 +1,83 @@
+package yqlib
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"io"
+)
+
+func (o *orderedMap) UnmarshalJSON(data []byte) error {
+	switch data[0] {
+	case '{':
+		// initialise so that even if the object is empty it is not nil
+		o.kv = []orderedMapKV{}
+
+		// create decoder
+		dec := json.NewDecoder(bytes.NewReader(data))
+		_, err := dec.Token() // open object
+		if err != nil {
+			return err
+		}
+
+		// cycle through k/v
+		var tok json.Token
+		for tok, err = dec.Token(); err == nil; tok, err = dec.Token() {
+			// we can expect two types: string or Delim. Delim automatically means
+			// that it is the closing bracket of the object, whereas string means
+			// that there is another key.
+			if _, ok := tok.(json.Delim); ok {
+				break
+			}
+			kv := orderedMapKV{
+				K: tok.(string),
+			}
+			if err := dec.Decode(&kv.V); err != nil {
+				return err
+			}
+			o.kv = append(o.kv, kv)
+		}
+		// unexpected error
+		if err != nil && !errors.Is(err, io.EOF) {
+			return err
+		}
+		return nil
+	case '[':
+		var res []*orderedMap
+		if err := json.Unmarshal(data, &res); err != nil {
+			return err
+		}
+		o.altVal = res
+		o.kv = nil
+		return nil
+	}
+
+	return json.Unmarshal(data, &o.altVal)
+}
+
+func (o orderedMap) MarshalJSON() ([]byte, error) {
+	buf := new(bytes.Buffer)
+	enc := json.NewEncoder(buf)
+	enc.SetEscapeHTML(false) // do not escape html chars e.g. &, <, >
+	if o.kv == nil {
+		if err := enc.Encode(o.altVal); err != nil {
+			return nil, err
+		}
+		return buf.Bytes(), nil
+	}
+	buf.WriteByte('{')
+	for idx, el := range o.kv {
+		if err := enc.Encode(el.K); err != nil {
+			return nil, err
+		}
+		buf.WriteByte(':')
+		if err := enc.Encode(el.V); err != nil {
+			return nil, err
+		}
+		if idx != len(o.kv)-1 {
+			buf.WriteByte(',')
+		}
+	}
+	buf.WriteByte('}')
+	return buf.Bytes(), nil
+}
diff --git a/pkg/yqlib/ordered_map_yaml.go b/pkg/yqlib/ordered_map_yaml.go
new file mode 100644
index 00000000..b3c49442
--- /dev/null
+++ b/pkg/yqlib/ordered_map_yaml.go
@@ -0,0 +1,79 @@
+package yqlib
+
+import (
+	"fmt"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+func (o *orderedMap) UnmarshalYAML(node *yaml.Node) error {
+	switch node.Kind {
+	case yaml.DocumentNode:
+		if len(node.Content) == 0 {
+			return nil
+		}
+		return o.UnmarshalYAML(node.Content[0])
+	case yaml.AliasNode:
+		return o.UnmarshalYAML(node.Alias)
+	case yaml.ScalarNode:
+		return node.Decode(&o.altVal)
+	case yaml.MappingNode:
+		// set kv to non-nil
+		o.kv = []orderedMapKV{}
+		for i := 0; i < len(node.Content); i += 2 {
+			var key string
+			var val orderedMap
+			if err := node.Content[i].Decode(&key); err != nil {
+				return err
+			}
+			if err := node.Content[i+1].Decode(&val); err != nil {
+				return err
+			}
+			o.kv = append(o.kv, orderedMapKV{
+				K: key,
+				V: val,
+			})
+		}
+		return nil
+	case yaml.SequenceNode:
+		// note that this has to be a pointer, so that nulls can be represented.
+		var res []*orderedMap
+		if err := node.Decode(&res); err != nil {
+			return err
+		}
+		o.altVal = res
+		o.kv = nil
+		return nil
+	case 0:
+		// null
+		o.kv = nil
+		o.altVal = nil
+		return nil
+	default:
+		return fmt.Errorf("orderedMap: invalid yaml node")
+	}
+}
+
+func (o *orderedMap) MarshalYAML() (interface{}, error) {
+	// fast path: kv is nil, use altVal
+	if o.kv == nil {
+		return o.altVal, nil
+	}
+	content := make([]*yaml.Node, 0, len(o.kv)*2)
+	for _, val := range o.kv {
+		n := new(yaml.Node)
+		if err := n.Encode(val.V); err != nil {
+			return nil, err
+		}
+		content = append(content, &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!str",
+			Value: val.K,
+		}, n)
+	}
+	return &yaml.Node{
+		Kind:    yaml.MappingNode,
+		Tag:     "!!map",
+		Content: content,
+	}, nil
+}
diff --git a/pkg/yqlib/printer_test.go b/pkg/yqlib/printer_test.go
index cc7b3dbe..f3be6401 100644
--- a/pkg/yqlib/printer_test.go
+++ b/pkg/yqlib/printer_test.go
@@ -314,7 +314,11 @@ func TestPrinterMultipleDocsJson(t *testing.T) {
 	var writer = bufio.NewWriter(&output)
 	// note printDocSeparators is true, it should still not print document separators
 	// when outputing JSON.
-	printer := NewPrinter(NewJSONEncoder(0, false, false), NewSinglePrinterWriter(writer))
+	encoder := NewJSONEncoder(0, false, false)
+	if encoder == nil {
+		t.Skipf("no support for %s output format", "json")
+	}
+	printer := NewPrinter(encoder, NewSinglePrinterWriter(writer))
 
 	inputs, err := readDocuments(strings.NewReader(multiDocSample), "sample.yml", 0, NewYamlDecoder(ConfiguredYamlPreferences))
 	if err != nil {
diff --git a/pkg/yqlib/xml_test.go b/pkg/yqlib/xml_test.go
index 2e9badf4..ec9f5b67 100644
--- a/pkg/yqlib/xml_test.go
+++ b/pkg/yqlib/xml_test.go
@@ -1,3 +1,5 @@
+//go:build !yq_noxml
+
 package yqlib
 
 import (

From 3f1f66a8eeb4712348d6207d6edc7c7ce10dcfb0 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Wed, 1 Mar 2023 13:45:35 +1100
Subject: [PATCH 41/53] Fixed merged anchor reference problem #1482

---
 pkg/yqlib/lib.go                  |  2 +-
 pkg/yqlib/operator_assign_test.go | 13 +++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go
index 65389263..144fed9f 100644
--- a/pkg/yqlib/lib.go
+++ b/pkg/yqlib/lib.go
@@ -338,7 +338,7 @@ func deepCloneWithOptions(node *yaml.Node, cloneContent bool) *yaml.Node {
 		Tag:         node.Tag,
 		Value:       node.Value,
 		Anchor:      node.Anchor,
-		Alias:       deepClone(node.Alias),
+		Alias:       node.Alias,
 		HeadComment: node.HeadComment,
 		LineComment: node.LineComment,
 		FootComment: node.FootComment,
diff --git a/pkg/yqlib/operator_assign_test.go b/pkg/yqlib/operator_assign_test.go
index 4a7cd208..47fcaa4c 100644
--- a/pkg/yqlib/operator_assign_test.go
+++ b/pkg/yqlib/operator_assign_test.go
@@ -4,6 +4,11 @@ import (
 	"testing"
 )
 
+var mergeAnchorAssign = `a: &a
+  x: OriginalValue
+b:
+  <<: *a`
+
 var assignOperatorScenarios = []expressionScenario{
 	{
 		description: "Create yaml file",
@@ -20,6 +25,14 @@ var assignOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: null\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   mergeAnchorAssign,
+		expression: `.c = .b | .a.x = "ModifiedValue" | explode(.)`,
+		expected: []string{
+			"D0, P[], (doc)::a:\n    x: ModifiedValue\nb:\n    x: ModifiedValue\nc:\n    x: ModifiedValue\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   "{}",

From fdb14875c0b56baa9b64d5e2a5fb2b92ce72117b Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Wed, 1 Mar 2023 13:49:28 +1100
Subject: [PATCH 42/53] Preparing release notes

---
 release_notes.txt | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/release_notes.txt b/release_notes.txt
index f4aab574..da5c3c48 100644
--- a/release_notes.txt
+++ b/release_notes.txt
@@ -1,3 +1,9 @@
+4.31.2 (draft):
+  - Fixed variable handling #1458, #1566
+  - Fixed merged anchor reference problem #1482
+  - Allow build without json and xml support (#1556) Thanks @afbjorklund
+  - Bumped dependencies
+
 4.31.1:
   - Added shuffle command #1503
   - Added ability to sort by multiple fields #1541

From 2195df9e7a71476758dbe7d2f8590cf9c196aedc Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Thu, 2 Mar 2023 10:57:54 +1100
Subject: [PATCH 43/53] Fixed xml encoding of ProcInst #1563, improved XML
 comment handling

---
 pkg/yqlib/decoder_xml.go   |  12 ++++-
 pkg/yqlib/doc/usage/xml.md |  13 +++--
 pkg/yqlib/encoder_xml.go   |  86 ++++++++++++++++++++---------
 pkg/yqlib/xml_test.go      | 108 +++++++++++++++++++++++++++++++++++--
 4 files changed, 184 insertions(+), 35 deletions(-)

diff --git a/pkg/yqlib/decoder_xml.go b/pkg/yqlib/decoder_xml.go
index bfb18e12..5c05b97f 100644
--- a/pkg/yqlib/decoder_xml.go
+++ b/pkg/yqlib/decoder_xml.go
@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"regexp"
 	"strings"
 	"unicode"
 
@@ -48,11 +49,19 @@ func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yaml.Node, error) {
 	return yamlNode, nil
 }
 
+var decoderCommentPrefix = regexp.MustCompile(`(^|\n)([[:alpha:]])`)
+
 func (dec *xmlDecoder) processComment(c string) string {
 	if c == "" {
 		return ""
 	}
-	return "#" + strings.TrimRight(c, " ")
+	//need to replace "cat " with "# cat"
+	// "\ncat\n" with "\n cat\n"
+	// ensure non-empty comments starting with newline have a space in front
+
+	replacement := decoderCommentPrefix.ReplaceAllString(c, "$1 $2")
+	replacement = "#" + strings.ReplaceAll(strings.TrimRight(replacement, " "), "\n", "\n#")
+	return replacement
 }
 
 func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
@@ -77,6 +86,7 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yaml.Node, error) {
 		var err error
 
 		if i == 0 {
+			log.Debugf("head comment here")
 			labelNode.HeadComment = dec.processComment(n.HeadComment)
 
 		}
diff --git a/pkg/yqlib/doc/usage/xml.md b/pkg/yqlib/doc/usage/xml.md
index 29a2b585..3d85a4ff 100644
--- a/pkg/yqlib/doc/usage/xml.md
+++ b/pkg/yqlib/doc/usage/xml.md
@@ -407,8 +407,10 @@ A best attempt is made to copy comments to xml.
 
 Given a sample.yml file of:
 ```yaml
+#
 # header comment
 # above_cat
+#
 cat: # inline_cat
   # above_array
   array: # inline_array
@@ -425,9 +427,11 @@ yq -o=xml '.' sample.yml
 will output
 ```xml
 <!--
- header comment
- above_cat
- --><!-- inline_cat --><cat><!-- above_array inline_array -->
+header comment
+above_cat
+-->
+<!-- inline_cat -->
+<cat><!-- above_array inline_array -->
   <array>val1<!-- inline_val1 --></array>
   <array><!-- above_val2 -->val2<!-- inline_val2 --></array>
 </cat><!-- below_cat -->
@@ -489,7 +493,8 @@ yq -p=xml -o=xml '.' sample.xml
 ```
 will output
 ```xml
-<!-- before cat --><cat><!-- in cat before -->
+<!-- before cat -->
+<cat><!-- in cat before -->
   <x>3<!-- multi
 line comment 
 for x --></x><!-- before y -->
diff --git a/pkg/yqlib/encoder_xml.go b/pkg/yqlib/encoder_xml.go
index d60ede8f..345e825e 100644
--- a/pkg/yqlib/encoder_xml.go
+++ b/pkg/yqlib/encoder_xml.go
@@ -19,8 +19,6 @@ type xmlEncoder struct {
 	leadingContent string
 }
 
-var commentPrefix = regexp.MustCompile(`(^|\n)\s*#`)
-
 func NewXMLEncoder(indent int, prefs XmlPreferences) Encoder {
 	var indentString = ""
 
@@ -39,7 +37,7 @@ func (e *xmlEncoder) PrintDocumentSeparator(writer io.Writer) error {
 }
 
 func (e *xmlEncoder) PrintLeadingContent(writer io.Writer, content string) error {
-	e.leadingContent = commentPrefix.ReplaceAllString(content, "\n")
+	e.leadingContent = content
 	return nil
 }
 
@@ -48,12 +46,39 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
 	// hack so we can manually add newlines to procInst and directives
 	e.writer = writer
 	encoder.Indent("", e.indentString)
+	var newLine xml.CharData = []byte("\n")
+
+	mapNode := unwrapDoc(node)
+	if mapNode.Tag == "!!map" {
+		// make sure <?xml .. ?> processing instructions are encoded first
+		for i := 0; i < len(mapNode.Content); i += 2 {
+			key := mapNode.Content[i]
+			value := mapNode.Content[i+1]
+
+			if key.Value == (e.prefs.ProcInstPrefix + "xml") {
+				name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1)
+				procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)}
+				if err := encoder.EncodeToken(procInst); err != nil {
+					return err
+				}
+				if _, err := e.writer.Write([]byte("\n")); err != nil {
+					log.Warning("Unable to write newline, skipping: %w", err)
+				}
+			}
+		}
+	}
 
 	if e.leadingContent != "" {
+
+		// remove first and last newlines if present
 		err := e.encodeComment(encoder, e.leadingContent)
 		if err != nil {
 			return err
 		}
+		err = encoder.EncodeToken(newLine)
+		if err != nil {
+			return err
+		}
 	}
 
 	switch node.Kind {
@@ -89,29 +114,12 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yaml.Node) error {
 	default:
 		return fmt.Errorf("unsupported type %v", node.Tag)
 	}
-	var charData xml.CharData = []byte("\n")
-	return encoder.EncodeToken(charData)
+
+	return encoder.EncodeToken(newLine)
 
 }
 
 func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) error {
-	// make sure <?xml .. ?> processing instructions are encoded first
-	for i := 0; i < len(node.Content); i += 2 {
-		key := node.Content[i]
-		value := node.Content[i+1]
-
-		if key.Value == (e.prefs.ProcInstPrefix + "xml") {
-			name := strings.Replace(key.Value, e.prefs.ProcInstPrefix, "", 1)
-			procInst := xml.ProcInst{Target: name, Inst: []byte(value.Value)}
-			if err := encoder.EncodeToken(procInst); err != nil {
-				return err
-			}
-			if _, err := e.writer.Write([]byte("\n")); err != nil {
-				log.Warning("Unable to write newline, skipping: %w", err)
-			}
-		}
-	}
-
 	err := e.encodeComment(encoder, headAndLineComment(node))
 	if err != nil {
 		return err
@@ -126,6 +134,13 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yaml.Node) er
 		if err != nil {
 			return err
 		}
+		if headAndLineComment(key) != "" {
+			var newLine xml.CharData = []byte("\n")
+			err = encoder.EncodeToken(newLine)
+			if err != nil {
+				return err
+			}
+		}
 
 		if key.Value == (e.prefs.ProcInstPrefix + "xml") {
 			// dont double process these.
@@ -206,13 +221,34 @@ func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *yaml.Node, start xml.S
 	return fmt.Errorf("unsupported type %v", node.Tag)
 }
 
+var xmlEncodeMultilineCommentRegex = regexp.MustCompile(`(^|\n) *# ?(.*)`)
+var xmlEncodeSingleLineCommentRegex = regexp.MustCompile(`^\s*#(.*)\n?`)
+var chompRegexp = regexp.MustCompile(`\n$`)
+
 func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) error {
 	if commentStr != "" {
-		log.Debugf("encoding comment %v", commentStr)
-		if !strings.HasSuffix(commentStr, " ") {
-			commentStr = commentStr + " "
+		log.Debugf("got comment [%v]", commentStr)
+		// multi line string
+		if len(commentStr) > 2 && strings.Contains(commentStr[1:len(commentStr)-1], "\n") {
+			commentStr = chompRegexp.ReplaceAllString(commentStr, "")
+			log.Debugf("chompRegexp [%v]", commentStr)
+			commentStr = xmlEncodeMultilineCommentRegex.ReplaceAllString(commentStr, "$1$2")
+			log.Debugf("processed multine [%v]", commentStr)
+			// if the first line is non blank, add a space
+			if commentStr[0] != '\n' && commentStr[0] != ' ' {
+				commentStr = " " + commentStr
+			}
+
+		} else {
+			commentStr = xmlEncodeSingleLineCommentRegex.ReplaceAllString(commentStr, "$1")
 		}
 
+		if !strings.HasSuffix(commentStr, " ") && !strings.HasSuffix(commentStr, "\n") {
+			commentStr = commentStr + " "
+			log.Debugf("added suffix [%v]", commentStr)
+		}
+		log.Debugf("encoding comment [%v]", commentStr)
+
 		var comment xml.Comment = []byte(commentStr)
 		err := encoder.EncodeToken(comment)
 		if err != nil {
diff --git a/pkg/yqlib/xml_test.go b/pkg/yqlib/xml_test.go
index ec9f5b67..b505e6ec 100644
--- a/pkg/yqlib/xml_test.go
+++ b/pkg/yqlib/xml_test.go
@@ -10,6 +10,29 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
+const yamlInputWithProcInstAndHeadComment = `# cats
++p_xml: version="1.0"
+this: is some xml`
+
+const expectedXmlProcInstAndHeadComment = `<?xml version="1.0"?>
+<!-- cats -->
+<this>is some xml</this>
+`
+
+const xmlProcInstAndHeadCommentBlock = `<?xml version="1.0"?>
+<!--
+cats
+-->
+<this>is some xml</this>
+`
+
+const expectedYamlProcInstAndHeadCommentBlock = `#
+# cats
+#
++p_xml: version="1.0"
+this: is some xml
+`
+
 const inputXMLWithComments = `
 <!-- before cat -->
 <cat>
@@ -128,7 +151,8 @@ cat:
 # after cat
 `
 
-const expectedRoundtripXMLWithComments = `<!-- before cat --><cat><!-- in cat before -->
+const expectedRoundtripXMLWithComments = `<!-- before cat -->
+<cat><!-- in cat before -->
   <x>3<!-- multi
 line comment 
 for x --></x><!-- before y -->
@@ -139,8 +163,10 @@ in d before -->
 </cat><!-- after cat -->
 `
 
-const yamlWithComments = `# header comment
+const yamlWithComments = `#
+# header comment
 # above_cat
+#
 cat: # inline_cat
   # above_array
   array: # inline_array
@@ -151,9 +177,11 @@ cat: # inline_cat
 `
 
 const expectedXMLWithComments = `<!--
- header comment
- above_cat
- --><!-- inline_cat --><cat><!-- above_array inline_array -->
+header comment
+above_cat
+-->
+<!-- inline_cat -->
+<cat><!-- above_array inline_array -->
   <array>val1<!-- inline_val1 --></array>
   <array><!-- above_val2 -->val2<!-- inline_val2 --></array>
 </cat><!-- below_cat -->
@@ -266,6 +294,41 @@ var xmlScenarios = []formatScenario{
 		input:    "<root><cats><cat>quick</cat><cat>soft</cat><!-- kitty_comment--><cat>squishy</cat></cats></root>",
 		expected: "root:\n    cats:\n        cat:\n            - quick\n            - soft\n            # kitty_comment\n\n            - squishy\n",
 	},
+	{
+		description:  "ProcInst with head comment",
+		skipDoc:      true,
+		input:        yamlInputWithProcInstAndHeadComment,
+		expected:     expectedXmlProcInstAndHeadComment,
+		scenarioType: "encode",
+	},
+	{
+		description:  "ProcInst with head comment round trip",
+		skipDoc:      true,
+		input:        expectedXmlProcInstAndHeadComment,
+		expected:     expectedXmlProcInstAndHeadComment,
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "ProcInst with block head comment to yaml",
+		skipDoc:      true,
+		input:        xmlProcInstAndHeadCommentBlock,
+		expected:     expectedYamlProcInstAndHeadCommentBlock,
+		scenarioType: "decode",
+	},
+	{
+		description:  "ProcInst with block head comment from yaml",
+		skipDoc:      true,
+		input:        expectedYamlProcInstAndHeadCommentBlock,
+		expected:     xmlProcInstAndHeadCommentBlock,
+		scenarioType: "encode",
+	},
+	{
+		description:  "ProcInst with head comment round trip block",
+		skipDoc:      true,
+		input:        xmlProcInstAndHeadCommentBlock,
+		expected:     xmlProcInstAndHeadCommentBlock,
+		scenarioType: "roundtrip",
+	},
 	{
 		description:    "Parse xml: simple",
 		subdescription: "Notice how all the values are strings, see the next example on how you can fix that.",
@@ -466,6 +529,41 @@ var xmlScenarios = []formatScenario{
 		expected:       "<cat name=\"tiger\">cool</cat>\n",
 		scenarioType:   "encode",
 	},
+	{
+		description:  "round trip multiline 1",
+		skipDoc:      true,
+		input:        "<x><!-- cats --></x>\n",
+		expected:     "<x><!-- cats --></x>\n",
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "round trip multiline 2",
+		skipDoc:      true,
+		input:        "<x><!--\n cats\n --></x>\n",
+		expected:     "<x><!--\ncats\n--></x>\n",
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "round trip multiline 3",
+		skipDoc:      true,
+		input:        "<x><!--\n\tcats\n --></x>\n",
+		expected:     "<x><!--\n\tcats\n--></x>\n",
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "round trip multiline 4",
+		skipDoc:      true,
+		input:        "<x><!--\n\tcats\n\tdogs\n--></x>\n",
+		expected:     "<x><!--\n\tcats\n\tdogs\n--></x>\n",
+		scenarioType: "roundtrip",
+	},
+	{
+		description:  "round trip multiline 5",
+		skipDoc:      true, // pity spaces aren't kept atm.
+		input:        "<x><!--\ncats\ndogs\n--></x>\n",
+		expected:     "<x><!--\ncats\ndogs\n--></x>\n",
+		scenarioType: "roundtrip",
+	},
 	{
 		description:    "Encode xml: comments",
 		subdescription: "A best attempt is made to copy comments to xml.",

From 47f4f8c7939f887e851b35f14def6741b8f5396e Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Thu, 2 Mar 2023 10:58:57 +1100
Subject: [PATCH 44/53] Bumping version

---
 cmd/version.go      | 2 +-
 snap/snapcraft.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/cmd/version.go b/cmd/version.go
index 14b17304..97a28c29 100644
--- a/cmd/version.go
+++ b/cmd/version.go
@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "v4.31.1"
+	Version = "v4.31.2"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release
diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml
index 5d0b528b..5adfb7c8 100644
--- a/snap/snapcraft.yaml
+++ b/snap/snapcraft.yaml
@@ -1,5 +1,5 @@
 name: yq
-version: 'v4.31.1'
+version: 'v4.31.2'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.

From e5260d855fb896f5b777991eb6b4359634494a80 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Thu, 2 Mar 2023 10:59:10 +1100
Subject: [PATCH 45/53] Release notes

---
 release_notes.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/release_notes.txt b/release_notes.txt
index da5c3c48..5bd70db5 100644
--- a/release_notes.txt
+++ b/release_notes.txt
@@ -1,6 +1,7 @@
 4.31.2 (draft):
   - Fixed variable handling #1458, #1566
   - Fixed merged anchor reference problem #1482
+  - Fixed xml encoding of ProcInst #1563, improved XML comment handling
   - Allow build without json and xml support (#1556) Thanks @afbjorklund
   - Bumped dependencies
 

From 2340ce6a8c07d7a4d152c6ed89f08bfce28c6d03 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Thu, 2 Mar 2023 11:21:22 +1100
Subject: [PATCH 46/53] Release notes

---
 release_notes.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release_notes.txt b/release_notes.txt
index 5bd70db5..073abbc9 100644
--- a/release_notes.txt
+++ b/release_notes.txt
@@ -1,4 +1,4 @@
-4.31.2 (draft):
+4.31.2:
   - Fixed variable handling #1458, #1566
   - Fixed merged anchor reference problem #1482
   - Fixed xml encoding of ProcInst #1563, improved XML comment handling

From fed96f67eafbd2bfad418e778b4f56dafde31913 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 3 Mar 2023 13:37:22 +1100
Subject: [PATCH 47/53] Bump github.com/goccy/go-yaml from 1.9.8 to 1.10.0
 (#1581)

Bumps [github.com/goccy/go-yaml](https://github.com/goccy/go-yaml) from 1.9.8 to 1.10.0.
- [Release notes](https://github.com/goccy/go-yaml/releases)
- [Changelog](https://github.com/goccy/go-yaml/blob/master/CHANGELOG.md)
- [Commits](https://github.com/goccy/go-yaml/compare/v1.9.8...v1.10.0)

---
updated-dependencies:
- dependency-name: github.com/goccy/go-yaml
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod | 2 +-
 go.sum | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/go.mod b/go.mod
index d340e9f5..fa3f6f65 100644
--- a/go.mod
+++ b/go.mod
@@ -8,7 +8,7 @@ require (
 	github.com/elliotchance/orderedmap v1.5.0
 	github.com/fatih/color v1.14.1
 	github.com/goccy/go-json v0.10.0
-	github.com/goccy/go-yaml v1.9.8
+	github.com/goccy/go-yaml v1.10.0
 	github.com/jinzhu/copier v0.3.5
 	github.com/magiconair/properties v1.8.7
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
diff --git a/go.sum b/go.sum
index 68287a2a..2780f7a2 100644
--- a/go.sum
+++ b/go.sum
@@ -21,8 +21,9 @@ github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA=
 github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
-github.com/goccy/go-yaml v1.9.8 h1:5gMyLUeU1/6zl+WFfR1hN7D2kf+1/eRGa7DFtToiBvQ=
-github.com/goccy/go-yaml v1.9.8/go.mod h1:JubOolP3gh0HpiBc4BLRD4YmjEjHAmIIB2aaXKkTfoE=
+github.com/goccy/go-yaml v1.10.0 h1:rBi+5HGuznOxx0JZ+60LDY85gc0dyIJCIMvsMJTKSKQ=
+github.com/goccy/go-yaml v1.10.0/go.mod h1:h/18Lr6oSQ3mvmqFoWmQ47KChOgpfHpTyIHl3yVmpiY=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
 github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
 github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=

From d30941b57515969a7e9f6373b63a18c2c32a0a66 Mon Sep 17 00:00:00 2001
From: ryenus <ryenus@gmail.com>
Date: Thu, 9 Mar 2023 07:17:19 +0800
Subject: [PATCH 48/53] Detect input format based on file name extension
 (#1582)

* detect inputFormat from filename

* refactor and extract func InputFormatFromFilename

* detect inputFormat only when file is provided

* add test for automatic input format detection
---
 acceptance_tests/inputs-format-auto.sh | 221 +++++++++++++++++++++++++
 cmd/constant.go                        |   3 +-
 cmd/evaluate_all_command.go            |   7 +-
 cmd/evalute_sequence_command.go        |   7 +-
 cmd/root.go                            |   2 +-
 cmd/utils.go                           |   7 +-
 pkg/yqlib/decoder.go                   |  20 ++-
 7 files changed, 260 insertions(+), 7 deletions(-)
 create mode 100755 acceptance_tests/inputs-format-auto.sh

diff --git a/acceptance_tests/inputs-format-auto.sh b/acceptance_tests/inputs-format-auto.sh
new file mode 100755
index 00000000..4732eb81
--- /dev/null
+++ b/acceptance_tests/inputs-format-auto.sh
@@ -0,0 +1,221 @@
+#!/bin/bash
+
+setUp() {
+  rm test*.yml 2>/dev/null || true
+  rm test*.json 2>/dev/null || true
+  rm test*.properties 2>/dev/null || true
+  rm test*.csv 2>/dev/null || true
+  rm test*.tsv 2>/dev/null || true
+  rm test*.xml 2>/dev/null || true
+}
+
+testInputJson() {
+  cat >test.json <<EOL
+{ "mike" : { "things": "cool" } }
+EOL
+
+  read -r -d '' expected << EOM
+mike:
+  things: cool
+EOM
+
+  X=$(./yq test.json)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea test.json)
+  assertEquals "$expected" "$X"
+}
+
+testInputProperties() {
+  cat >test.properties <<EOL
+mike.things = hello
+EOL
+
+  read -r -d '' expected << EOM
+mike:
+  things: hello
+EOM
+
+  X=$(./yq e test.properties)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea test.properties)
+  assertEquals "$expected" "$X"
+}
+
+testInputPropertiesGitHubAction() {
+  cat >test.properties <<EOL
+mike.things = hello
+EOL
+
+  read -r -d '' expected << EOM
+mike:
+  things: hello
+EOM
+
+  X=$(cat /dev/null | ./yq e test.properties)
+  assertEquals "$expected" "$X"
+
+  X=$(cat /dev/null | ./yq ea test.properties)
+  assertEquals "$expected" "$X"
+}
+
+testInputCSV() {
+  cat >test.csv <<EOL
+fruit,yumLevel
+apple,5
+banana,4
+EOL
+
+  read -r -d '' expected << EOM
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOM
+
+  X=$(./yq e test.csv)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea test.csv)
+  assertEquals "$expected" "$X"
+}
+
+testInputCSVUTF8() {
+  read -r -d '' expected << EOM
+- id: 1
+  first: john
+  last: smith
+- id: 1
+  first: jane
+  last: smith
+EOM
+
+  X=$(./yq utf8.csv)
+  assertEquals "$expected" "$X"
+}
+
+testInputTSV() {
+  cat >test.tsv <<EOL
+fruit	yumLevel
+apple	5
+banana	4
+EOL
+
+  read -r -d '' expected << EOM
+- fruit: apple
+  yumLevel: 5
+- fruit: banana
+  yumLevel: 4
+EOM
+
+  X=$(./yq e test.tsv)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea test.tsv)
+  assertEquals "$expected" "$X"
+}
+
+
+
+
+testInputXml() {
+  cat >test.xml <<EOL
+<cat legs="4">BiBi</cat>
+EOL
+
+  read -r -d '' expected << EOM
+cat:
+  +content: BiBi
+  +@legs: "4"
+EOM
+
+  X=$(./yq e test.xml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea test.xml)
+  assertEquals "$expected" "$X"
+}
+
+testInputXmlNamespaces() {
+  cat >test.xml <<EOL
+<?xml version="1.0"?>
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
+</map>
+EOL
+
+  read -r -d '' expected << EOM
++p_xml: version="1.0"
+map:
+  +@xmlns: some-namespace
+  +@xmlns:xsi: some-instance
+  +@xsi:schemaLocation: some-url
+EOM
+
+  X=$(./yq e test.xml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea test.xml)
+  assertEquals "$expected" "$X"
+}
+
+testInputXmlRoundtrip() {
+  cat >test.xml <<EOL
+<?xml version="1.0"?>
+<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">Meow</map>
+EOL
+
+  read -r -d '' expected << EOM
+<?xml version="1.0"?>
+<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">Meow</map>
+EOM
+
+  X=$(./yq -o=xml test.xml)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea -o=xml test.xml)
+  assertEquals "$expected" "$X"
+}
+
+
+testInputXmlStrict() {
+  cat >test.xml <<EOL
+<?xml version="1.0"?>
+<!DOCTYPE root [
+<!ENTITY writer "Catherine.">
+<!ENTITY copyright "(r) Great">
+]>
+<root>
+    <item>&writer;&copyright;</item>
+</root>
+EOL
+
+  X=$(./yq --xml-strict-mode test.xml -o=xml 2>&1)
+  assertEquals 1 $?
+  assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X"
+
+  X=$(./yq ea --xml-strict-mode test.xml -o=xml 2>&1)
+  assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X"
+}
+
+testInputXmlGithubAction() {
+  cat >test.xml <<EOL
+<cat legs="4">BiBi</cat>
+EOL
+
+  read -r -d '' expected << EOM
+cat:
+  +content: BiBi
+  +@legs: "4"
+EOM
+
+  X=$(cat /dev/null | ./yq e test.xml)
+  assertEquals "$expected" "$X"
+
+  X=$(cat /dev/null | ./yq ea test.xml)
+  assertEquals "$expected" "$X"
+}
+
+source ./scripts/shunit2
diff --git a/cmd/constant.go b/cmd/constant.go
index 573a72fb..8b94dc2c 100644
--- a/cmd/constant.go
+++ b/cmd/constant.go
@@ -7,7 +7,8 @@ var unwrapScalar = false
 var writeInplace = false
 var outputToJSON = false
 var outputFormat = "yaml"
-var inputFormat = "yaml"
+var inputFormatDefault = "yaml"
+var inputFormat = ""
 
 var exitStatus = false
 var forceColor = false
diff --git a/cmd/evaluate_all_command.go b/cmd/evaluate_all_command.go
index fff64fa5..b56a9cc4 100644
--- a/cmd/evaluate_all_command.go
+++ b/cmd/evaluate_all_command.go
@@ -75,7 +75,12 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 		return err
 	}
 
-	decoder, err := configureDecoder(true)
+	inputFilename := ""
+	if len(args) > 0 {
+		inputFilename = args[0]
+	}
+
+	decoder, err := configureDecoder(true, inputFilename)
 	if err != nil {
 		return err
 	}
diff --git a/cmd/evalute_sequence_command.go b/cmd/evalute_sequence_command.go
index dab803c3..bc69698f 100644
--- a/cmd/evalute_sequence_command.go
+++ b/cmd/evalute_sequence_command.go
@@ -100,7 +100,12 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 
 	printer := yqlib.NewPrinter(encoder, printerWriter)
 
-	decoder, err := configureDecoder(false)
+	inputFilename := ""
+	if len(args) > 0 {
+		inputFilename = args[0]
+	}
+
+	decoder, err := configureDecoder(false, inputFilename)
 	if err != nil {
 		return err
 	}
diff --git a/cmd/root.go b/cmd/root.go
index 97d9beaf..76782ccb 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -85,7 +85,7 @@ yq -P sample.json
 	}
 
 	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "yaml", "[yaml|y|json|j|props|p|xml|x] output format type.")
-	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "yaml", "[yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.")
+	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "", "[yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.")
 
 	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
 	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")
diff --git a/cmd/utils.go b/cmd/utils.go
index 671dd86b..cc20a827 100644
--- a/cmd/utils.go
+++ b/cmd/utils.go
@@ -56,7 +56,12 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 	return expression, args, nil
 }
 
-func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
+func configureDecoder(evaluateTogether bool, inputFilename string) (yqlib.Decoder, error) {
+	if inputFormat == "" {
+		inputFormat = yqlib.InputFormatFromFilename(inputFilename, inputFormatDefault)
+	} else {
+		yqlib.GetLogger().Debugf("user specified inputFormat '%s'", inputFormat)
+	}
 	yqlibInputFormat, err := yqlib.InputFormatFromString(inputFormat)
 	if err != nil {
 		return nil, err
diff --git a/pkg/yqlib/decoder.go b/pkg/yqlib/decoder.go
index ff7ab2db..c06a5a72 100644
--- a/pkg/yqlib/decoder.go
+++ b/pkg/yqlib/decoder.go
@@ -3,6 +3,7 @@ package yqlib
 import (
 	"fmt"
 	"io"
+	"strings"
 )
 
 type InputFormat uint
@@ -25,11 +26,11 @@ type Decoder interface {
 
 func InputFormatFromString(format string) (InputFormat, error) {
 	switch format {
-	case "yaml", "y":
+	case "yaml", "yml", "y":
 		return YamlInputFormat, nil
 	case "xml", "x":
 		return XMLInputFormat, nil
-	case "props", "p":
+	case "properties", "props", "p":
 		return PropertiesInputFormat, nil
 	case "json", "ndjson", "j":
 		return JsonInputFormat, nil
@@ -41,3 +42,18 @@ func InputFormatFromString(format string) (InputFormat, error) {
 		return 0, fmt.Errorf("unknown format '%v' please use [yaml|xml|props]", format)
 	}
 }
+
+func InputFormatFromFilename(filename string, defaultFormat string) string {
+	if filename != "" {
+		GetLogger().Debugf("checking filename '%s' for inputFormat", filename)
+		nPos := strings.LastIndex(filename, ".")
+		if nPos > -1 {
+			inputFormat := filename[nPos+1:]
+			GetLogger().Debugf("detected inputFormat '%s'", inputFormat)
+			return inputFormat
+		}
+	}
+
+	GetLogger().Debugf("using default inputFormat '%s'", defaultFormat)
+	return defaultFormat
+}

From 9539877ff6b8dfbbb726cf4eaad4dcb7f0165e5a Mon Sep 17 00:00:00 2001
From: Robert Brennan <accounts@rbren.io>
Date: Wed, 8 Mar 2023 18:30:47 -0500
Subject: [PATCH 49/53] Add filter operation (#1588)

* add filter operation

* add tests

* add tests

* revert debug

* simplify filter

* fix tests

* remove logs
---
 pkg/yqlib/doc/operators/filter.md | 18 ++++++++++++
 pkg/yqlib/lexer_participle.go     |  1 +
 pkg/yqlib/lib.go                  |  1 +
 pkg/yqlib/operator_filter.go      | 33 +++++++++++++++++++++
 pkg/yqlib/operator_filter_test.go | 49 +++++++++++++++++++++++++++++++
 5 files changed, 102 insertions(+)
 create mode 100644 pkg/yqlib/doc/operators/filter.md
 create mode 100644 pkg/yqlib/operator_filter.go
 create mode 100644 pkg/yqlib/operator_filter_test.go

diff --git a/pkg/yqlib/doc/operators/filter.md b/pkg/yqlib/doc/operators/filter.md
new file mode 100644
index 00000000..d35269aa
--- /dev/null
+++ b/pkg/yqlib/doc/operators/filter.md
@@ -0,0 +1,18 @@
+
+## Filter array
+Given a sample.yml file of:
+```yaml
+- 1
+- 2
+- 3
+```
+then
+```bash
+yq 'filter(. < 3)' sample.yml
+```
+will output
+```yaml
+- 1
+- 2
+```
+
diff --git a/pkg/yqlib/lexer_participle.go b/pkg/yqlib/lexer_participle.go
index 33756fef..e6d4d58f 100644
--- a/pkg/yqlib/lexer_participle.go
+++ b/pkg/yqlib/lexer_participle.go
@@ -37,6 +37,7 @@ var participleYqRules = []*participleYqRule{
 
 	{"MapValues", `map_?values`, opToken(mapValuesOpType), 0},
 	simpleOp("map", mapOpType),
+	simpleOp("filter", filterOpType),
 	simpleOp("pick", pickOpType),
 
 	{"FlattenWithDepth", `flatten\([0-9]+\)`, flattenWithDepth(), 0},
diff --git a/pkg/yqlib/lib.go b/pkg/yqlib/lib.go
index 144fed9f..a24d18f2 100644
--- a/pkg/yqlib/lib.go
+++ b/pkg/yqlib/lib.go
@@ -84,6 +84,7 @@ var expressionOpType = &operationType{Type: "EXP", NumArgs: 0, Precedence: 50, H
 
 var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50, Handler: collectOperator}
 var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler: mapOperator}
+var filterOpType = &operationType{Type: "FILTER", NumArgs: 1, Precedence: 50, Handler: filterOperator}
 var errorOpType = &operationType{Type: "ERROR", NumArgs: 1, Precedence: 50, Handler: errorOperator}
 var pickOpType = &operationType{Type: "PICK", NumArgs: 1, Precedence: 50, Handler: pickOperator}
 var evalOpType = &operationType{Type: "EVAL", NumArgs: 1, Precedence: 50, Handler: evalOperator}
diff --git a/pkg/yqlib/operator_filter.go b/pkg/yqlib/operator_filter.go
new file mode 100644
index 00000000..451b0beb
--- /dev/null
+++ b/pkg/yqlib/operator_filter.go
@@ -0,0 +1,33 @@
+package yqlib
+
+import (
+	"container/list"
+)
+
+func filterOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	log.Debugf("-- filterOperation")
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		children := context.SingleChildContext(candidate)
+		splatted, err := splat(children, traversePreferences{})
+		if err != nil {
+			return Context{}, err
+		}
+		filtered, err := selectOperator(d, splatted, expressionNode)
+		if err != nil {
+			return Context{}, err
+		}
+
+		selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
+		collected, err := collectTogether(d, filtered, selfExpression)
+		if err != nil {
+			return Context{}, err
+		}
+		collected.Node.Style = unwrapDoc(candidate.Node).Style
+		results.PushBack(collected)
+	}
+	return context.ChildContext(results), nil
+}
+
diff --git a/pkg/yqlib/operator_filter_test.go b/pkg/yqlib/operator_filter_test.go
new file mode 100644
index 00000000..05a68161
--- /dev/null
+++ b/pkg/yqlib/operator_filter_test.go
@@ -0,0 +1,49 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var filterOperatorScenarios = []expressionScenario{
+	{
+		description: "Filter array",
+		document:   `[1,2,3]`,
+		expression: `filter(. < 3)`,
+		expected: []string{
+			"D0, P[], (!!seq)::[1, 2]\n",
+		},
+	},
+	{
+		skipDoc: true,
+		document:    `[1,2,3]`,
+		expression:  `filter(. > 1)`,
+		expected: []string{
+			"D0, P[], (!!seq)::[2, 3]\n",
+		},
+	},
+	{
+		skipDoc: true,
+		description: "Filter array to empty",
+		document:    `[1,2,3]`,
+		expression:  `filter(. > 4)`,
+		expected: []string{
+			"D0, P[], (!!seq)::[]\n",
+		},
+	},
+	{
+		skipDoc: true,
+		description: "Filter empty array",
+		document:    `[]`,
+		expression:  `filter(. > 1)`,
+		expected: []string{
+			"D0, P[], (!!seq)::[]\n",
+		},
+	},
+}
+
+func TestFilterOperatorScenarios(t *testing.T) {
+	for _, tt := range filterOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentOperatorScenarios(t, "filter", filterOperatorScenarios)
+}

From eafcc3bafa9b4415442d0239322af2b30de12de3 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 10 Mar 2023 10:13:11 +1100
Subject: [PATCH 50/53] Bump golang.org/x/net from 0.7.0 to 0.8.0 (#1590)

Bumps [golang.org/x/net](https://github.com/golang/net) from 0.7.0 to 0.8.0.
- [Release notes](https://github.com/golang/net/releases)
- [Commits](https://github.com/golang/net/compare/v0.7.0...v0.8.0)

---
updated-dependencies:
- dependency-name: golang.org/x/net
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 go.mod |  6 +++---
 go.sum | 12 ++++++------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/go.mod b/go.mod
index fa3f6f65..cc2ebe05 100644
--- a/go.mod
+++ b/go.mod
@@ -14,7 +14,7 @@ require (
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/pflag v1.0.5
-	golang.org/x/net v0.7.0
+	golang.org/x/net v0.8.0
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -23,8 +23,8 @@ require (
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
-	golang.org/x/sys v0.5.0 // indirect
-	golang.org/x/text v0.7.0 // indirect
+	golang.org/x/sys v0.6.0 // indirect
+	golang.org/x/text v0.8.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 )
 
diff --git a/go.sum b/go.sum
index 2780f7a2..08cb016e 100644
--- a/go.sum
+++ b/go.sum
@@ -55,20 +55,20 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
-golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ=
+golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68=
+golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0=

From 2c14c9840898a01052e8e53d1455b1655cb1e048 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 10 Mar 2023 10:13:22 +1100
Subject: [PATCH 51/53] Bump golang from 1.20.1 to 1.20.2 (#1589)

Bumps golang from 1.20.1 to 1.20.2.

---
updated-dependencies:
- dependency-name: golang
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 Dockerfile     | 2 +-
 Dockerfile.dev | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index b5fd237f..9cc6545d 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.20.1 as builder
+FROM golang:1.20.2 as builder
 
 WORKDIR /go/src/mikefarah/yq
 
diff --git a/Dockerfile.dev b/Dockerfile.dev
index 1bd9517c..8c7c8dbc 100644
--- a/Dockerfile.dev
+++ b/Dockerfile.dev
@@ -1,4 +1,4 @@
-FROM golang:1.20.1
+FROM golang:1.20.2
 
 COPY scripts/devtools.sh /opt/devtools.sh
 

From 08a6cb65fe9e1a343d4c6b5cc182f4d4e65156d4 Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Wed, 15 Mar 2023 13:22:58 +1100
Subject: [PATCH 52/53] Auto output format (#1599)

* Use file extension to auto detect output format!

* Use file extension to auto detect output format!

* formatting
---
 acceptance_tests/inputs-format-auto.sh | 94 ++++++++++++--------------
 acceptance_tests/inputs-format.sh      |  6 +-
 cmd/constant.go                        |  5 +-
 cmd/evaluate_all_command.go            |  7 +-
 cmd/evalute_sequence_command.go        |  7 +-
 cmd/root.go                            | 23 +------
 cmd/utils.go                           | 49 ++++++++++++--
 examples/mike.xml                      |  6 +-
 pkg/yqlib/decoder.go                   | 17 ++---
 pkg/yqlib/operator_filter.go           |  1 -
 pkg/yqlib/operator_filter_test.go      | 14 ++--
 pkg/yqlib/printer.go                   |  4 +-
 12 files changed, 114 insertions(+), 119 deletions(-)

diff --git a/acceptance_tests/inputs-format-auto.sh b/acceptance_tests/inputs-format-auto.sh
index 4732eb81..302f3025 100755
--- a/acceptance_tests/inputs-format-auto.sh
+++ b/acceptance_tests/inputs-format-auto.sh
@@ -15,8 +15,11 @@ testInputJson() {
 EOL
 
   read -r -d '' expected << EOM
-mike:
-  things: cool
+{
+  "mike": {
+    "things": "cool"
+  }
+}
 EOM
 
   X=$(./yq test.json)
@@ -26,19 +29,38 @@ EOM
   assertEquals "$expected" "$X"
 }
 
+testInputJsonOutputYaml() {
+  cat >test.json <<EOL
+{ "mike" : { "things": "cool" } }
+EOL
+
+  read -r -d '' expected << EOM
+mike:
+  things: cool
+EOM
+
+  X=$(./yq test.json -oy)
+  assertEquals "$expected" "$X"
+
+  X=$(./yq ea test.json -oy)
+  assertEquals "$expected" "$X"
+}
+
 testInputProperties() {
   cat >test.properties <<EOL
 mike.things = hello
 EOL
 
   read -r -d '' expected << EOM
-mike:
-  things: hello
+mike.things = hello
 EOM
 
   X=$(./yq e test.properties)
   assertEquals "$expected" "$X"
 
+  X=$(./yq test.properties)
+  assertEquals "$expected" "$X"
+
   X=$(./yq ea test.properties)
   assertEquals "$expected" "$X"
 }
@@ -49,8 +71,7 @@ mike.things = hello
 EOL
 
   read -r -d '' expected << EOM
-mike:
-  things: hello
+mike.things = hello
 EOM
 
   X=$(cat /dev/null | ./yq e test.properties)
@@ -68,10 +89,9 @@ banana,4
 EOL
 
   read -r -d '' expected << EOM
-- fruit: apple
-  yumLevel: 5
-- fruit: banana
-  yumLevel: 4
+fruit,yumLevel
+apple,5
+banana,4
 EOM
 
   X=$(./yq e test.csv)
@@ -83,12 +103,9 @@ EOM
 
 testInputCSVUTF8() {
   read -r -d '' expected << EOM
-- id: 1
-  first: john
-  last: smith
-- id: 1
-  first: jane
-  last: smith
+id,first,last
+1,john,smith
+1,jane,smith
 EOM
 
   X=$(./yq utf8.csv)
@@ -103,10 +120,9 @@ banana	4
 EOL
 
   read -r -d '' expected << EOM
-- fruit: apple
-  yumLevel: 5
-- fruit: banana
-  yumLevel: 4
+fruit	yumLevel
+apple	5
+banana	4
 EOM
 
   X=$(./yq e test.tsv)
@@ -125,9 +141,7 @@ testInputXml() {
 EOL
 
   read -r -d '' expected << EOM
-cat:
-  +content: BiBi
-  +@legs: "4"
+<cat legs="4">BiBi</cat>
 EOM
 
   X=$(./yq e test.xml)
@@ -145,11 +159,8 @@ testInputXmlNamespaces() {
 EOL
 
   read -r -d '' expected << EOM
-+p_xml: version="1.0"
-map:
-  +@xmlns: some-namespace
-  +@xmlns:xsi: some-instance
-  +@xsi:schemaLocation: some-url
+<?xml version="1.0"?>
+<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url"></map>
 EOM
 
   X=$(./yq e test.xml)
@@ -159,25 +170,6 @@ EOM
   assertEquals "$expected" "$X"
 }
 
-testInputXmlRoundtrip() {
-  cat >test.xml <<EOL
-<?xml version="1.0"?>
-<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
-<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">Meow</map>
-EOL
-
-  read -r -d '' expected << EOM
-<?xml version="1.0"?>
-<!DOCTYPE config SYSTEM "/etc/iwatch/iwatch.dtd" >
-<map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">Meow</map>
-EOM
-
-  X=$(./yq -o=xml test.xml)
-  assertEquals "$expected" "$X"
-
-  X=$(./yq ea -o=xml test.xml)
-  assertEquals "$expected" "$X"
-}
 
 
 testInputXmlStrict() {
@@ -192,11 +184,11 @@ testInputXmlStrict() {
 </root>
 EOL
 
-  X=$(./yq --xml-strict-mode test.xml -o=xml 2>&1)
+  X=$(./yq --xml-strict-mode test.xml  2>&1)
   assertEquals 1 $?
   assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X"
 
-  X=$(./yq ea --xml-strict-mode test.xml -o=xml 2>&1)
+  X=$(./yq ea --xml-strict-mode test.xml  2>&1)
   assertEquals "Error: bad file 'test.xml': XML syntax error on line 7: invalid character entity &writer;" "$X"
 }
 
@@ -206,9 +198,7 @@ testInputXmlGithubAction() {
 EOL
 
   read -r -d '' expected << EOM
-cat:
-  +content: BiBi
-  +@legs: "4"
+<cat legs="4">BiBi</cat>
 EOM
 
   X=$(cat /dev/null | ./yq e test.xml)
diff --git a/acceptance_tests/inputs-format.sh b/acceptance_tests/inputs-format.sh
index 2eebd501..cd48cb4a 100755
--- a/acceptance_tests/inputs-format.sh
+++ b/acceptance_tests/inputs-format.sh
@@ -120,7 +120,7 @@ EOM
 }
 
 testInputXmlNamespaces() {
-  cat >test.yml <<EOL
+  cat >test.xml <<EOL
 <?xml version="1.0"?>
 <map xmlns="some-namespace" xmlns:xsi="some-instance" xsi:schemaLocation="some-url">
 </map>
@@ -134,10 +134,10 @@ map:
   +@xsi:schemaLocation: some-url
 EOM
 
-  X=$(./yq e -p=xml test.yml)
+  X=$(./yq e -p=xml test.xml)
   assertEquals "$expected" "$X"
 
-  X=$(./yq ea -p=xml test.yml)
+  X=$(./yq ea -p=xml test.xml)
   assertEquals "$expected" "$X"
 }
 
diff --git a/cmd/constant.go b/cmd/constant.go
index 8b94dc2c..a8428aae 100644
--- a/cmd/constant.go
+++ b/cmd/constant.go
@@ -6,8 +6,9 @@ var unwrapScalar = false
 
 var writeInplace = false
 var outputToJSON = false
-var outputFormat = "yaml"
-var inputFormatDefault = "yaml"
+
+var outputFormat = ""
+
 var inputFormat = ""
 
 var exitStatus = false
diff --git a/cmd/evaluate_all_command.go b/cmd/evaluate_all_command.go
index b56a9cc4..fff64fa5 100644
--- a/cmd/evaluate_all_command.go
+++ b/cmd/evaluate_all_command.go
@@ -75,12 +75,7 @@ func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) {
 		return err
 	}
 
-	inputFilename := ""
-	if len(args) > 0 {
-		inputFilename = args[0]
-	}
-
-	decoder, err := configureDecoder(true, inputFilename)
+	decoder, err := configureDecoder(true)
 	if err != nil {
 		return err
 	}
diff --git a/cmd/evalute_sequence_command.go b/cmd/evalute_sequence_command.go
index bc69698f..dab803c3 100644
--- a/cmd/evalute_sequence_command.go
+++ b/cmd/evalute_sequence_command.go
@@ -100,12 +100,7 @@ func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) {
 
 	printer := yqlib.NewPrinter(encoder, printerWriter)
 
-	inputFilename := ""
-	if len(args) > 0 {
-		inputFilename = args[0]
-	}
-
-	decoder, err := configureDecoder(false, inputFilename)
+	decoder, err := configureDecoder(false)
 	if err != nil {
 		return err
 	}
diff --git a/cmd/root.go b/cmd/root.go
index 76782ccb..98e52752 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -53,25 +53,6 @@ yq -P sample.json
 			logging.SetBackend(backend)
 			yqlib.InitExpressionParser()
 
-			outputFormatType, err := yqlib.OutputFormatFromString(outputFormat)
-
-			if err != nil {
-				return err
-			}
-
-			if outputFormatType == yqlib.YamlOutputFormat ||
-				outputFormatType == yqlib.PropsOutputFormat {
-				unwrapScalar = true
-			}
-			if unwrapScalarFlag.IsExplicitySet() {
-				unwrapScalar = unwrapScalarFlag.IsSet()
-			}
-
-			//copy preference form global setting
-			yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
-
-			yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
-
 			return nil
 		},
 	}
@@ -84,8 +65,8 @@ yq -P sample.json
 		panic(err)
 	}
 
-	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "yaml", "[yaml|y|json|j|props|p|xml|x] output format type.")
-	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "", "[yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.")
+	rootCmd.PersistentFlags().StringVarP(&outputFormat, "output-format", "o", "auto", "[auto|a|yaml|y|json|j|props|p|xml|x] output format type.")
+	rootCmd.PersistentFlags().StringVarP(&inputFormat, "input-format", "p", "auto", "[auto|a|yaml|y|props|p|xml|x] parse format for input. Note that json is a subset of yaml.")
 
 	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
 	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")
diff --git a/cmd/utils.go b/cmd/utils.go
index cc20a827..c8ba95fa 100644
--- a/cmd/utils.go
+++ b/cmd/utils.go
@@ -53,15 +53,52 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 		return "", nil, fmt.Errorf("cannot pass files in when using null-input flag")
 	}
 
+	inputFilename := ""
+	if len(args) > 0 {
+		inputFilename = args[0]
+	}
+	if inputFormat == "" || inputFormat == "auto" || inputFormat == "a" {
+
+		inputFormat = yqlib.FormatFromFilename(inputFilename)
+		if outputFormat == "" || outputFormat == "auto" || outputFormat == "a" {
+			outputFormat = yqlib.FormatFromFilename(inputFilename)
+		}
+	} else if outputFormat == "" || outputFormat == "auto" || outputFormat == "a" {
+		// backwards compatibility -
+		// before this was introduced, `yq -pcsv things.csv`
+		// would produce *yaml* output.
+		//
+		outputFormat = yqlib.FormatFromFilename(inputFilename)
+		if inputFilename != "-" {
+			yqlib.GetLogger().Warning("yq default output is now 'auto' (based on the filename extension). Normally yq would output '%v', but for backwards compatibility 'yaml' has been set. Please use -oy to specify yaml, or drop the -p flag.", outputFormat)
+		}
+		outputFormat = "yaml"
+	}
+
+	outputFormatType, err := yqlib.OutputFormatFromString(outputFormat)
+
+	if err != nil {
+		return "", nil, err
+	}
+	yqlib.GetLogger().Debug("Using outputformat %v", outputFormat)
+
+	if outputFormatType == yqlib.YamlOutputFormat ||
+		outputFormatType == yqlib.PropsOutputFormat {
+		unwrapScalar = true
+	}
+	if unwrapScalarFlag.IsExplicitySet() {
+		unwrapScalar = unwrapScalarFlag.IsSet()
+	}
+
+	//copy preference form global setting
+	yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
+
+	yqlib.ConfiguredYamlPreferences.PrintDocSeparators = !noDocSeparators
+
 	return expression, args, nil
 }
 
-func configureDecoder(evaluateTogether bool, inputFilename string) (yqlib.Decoder, error) {
-	if inputFormat == "" {
-		inputFormat = yqlib.InputFormatFromFilename(inputFilename, inputFormatDefault)
-	} else {
-		yqlib.GetLogger().Debugf("user specified inputFormat '%s'", inputFormat)
-	}
+func configureDecoder(evaluateTogether bool) (yqlib.Decoder, error) {
 	yqlibInputFormat, err := yqlib.InputFormatFromString(inputFormat)
 	if err != nil {
 		return nil, err
diff --git a/examples/mike.xml b/examples/mike.xml
index cdc6a9df..b582fff1 100644
--- a/examples/mike.xml
+++ b/examples/mike.xml
@@ -1,5 +1 @@
-date: "2022-11-25"
-raw:
-  id: 1
-  XML: text_outside_1<Tag1 />text_outside_2<Tag2Kling Klong</Tag2>text_outside_3
-  time: 09:19:00
\ No newline at end of file
+<cat>3</cat>
\ No newline at end of file
diff --git a/pkg/yqlib/decoder.go b/pkg/yqlib/decoder.go
index c06a5a72..9fba83d3 100644
--- a/pkg/yqlib/decoder.go
+++ b/pkg/yqlib/decoder.go
@@ -39,21 +39,22 @@ func InputFormatFromString(format string) (InputFormat, error) {
 	case "tsv", "t":
 		return TSVObjectInputFormat, nil
 	default:
-		return 0, fmt.Errorf("unknown format '%v' please use [yaml|xml|props]", format)
+		return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv|xml]", format)
 	}
 }
 
-func InputFormatFromFilename(filename string, defaultFormat string) string {
+func FormatFromFilename(filename string) string {
+
 	if filename != "" {
-		GetLogger().Debugf("checking filename '%s' for inputFormat", filename)
+		GetLogger().Debugf("checking file extension '%s' for auto format detection", filename)
 		nPos := strings.LastIndex(filename, ".")
 		if nPos > -1 {
-			inputFormat := filename[nPos+1:]
-			GetLogger().Debugf("detected inputFormat '%s'", inputFormat)
-			return inputFormat
+			format := filename[nPos+1:]
+			GetLogger().Debugf("detected format '%s'", format)
+			return format
 		}
 	}
 
-	GetLogger().Debugf("using default inputFormat '%s'", defaultFormat)
-	return defaultFormat
+	GetLogger().Debugf("using default inputFormat 'yaml'")
+	return "yaml"
 }
diff --git a/pkg/yqlib/operator_filter.go b/pkg/yqlib/operator_filter.go
index 451b0beb..8cf28303 100644
--- a/pkg/yqlib/operator_filter.go
+++ b/pkg/yqlib/operator_filter.go
@@ -30,4 +30,3 @@ func filterOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 	}
 	return context.ChildContext(results), nil
 }
-
diff --git a/pkg/yqlib/operator_filter_test.go b/pkg/yqlib/operator_filter_test.go
index 05a68161..fc050796 100644
--- a/pkg/yqlib/operator_filter_test.go
+++ b/pkg/yqlib/operator_filter_test.go
@@ -7,22 +7,22 @@ import (
 var filterOperatorScenarios = []expressionScenario{
 	{
 		description: "Filter array",
-		document:   `[1,2,3]`,
-		expression: `filter(. < 3)`,
+		document:    `[1,2,3]`,
+		expression:  `filter(. < 3)`,
 		expected: []string{
 			"D0, P[], (!!seq)::[1, 2]\n",
 		},
 	},
 	{
-		skipDoc: true,
-		document:    `[1,2,3]`,
-		expression:  `filter(. > 1)`,
+		skipDoc:    true,
+		document:   `[1,2,3]`,
+		expression: `filter(. > 1)`,
 		expected: []string{
 			"D0, P[], (!!seq)::[2, 3]\n",
 		},
 	},
 	{
-		skipDoc: true,
+		skipDoc:     true,
 		description: "Filter array to empty",
 		document:    `[1,2,3]`,
 		expression:  `filter(. > 4)`,
@@ -31,7 +31,7 @@ var filterOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		skipDoc: true,
+		skipDoc:     true,
 		description: "Filter empty array",
 		document:    `[]`,
 		expression:  `filter(. > 1)`,
diff --git a/pkg/yqlib/printer.go b/pkg/yqlib/printer.go
index 802793a9..aca370e8 100644
--- a/pkg/yqlib/printer.go
+++ b/pkg/yqlib/printer.go
@@ -33,11 +33,11 @@ const (
 
 func OutputFormatFromString(format string) (PrinterOutputFormat, error) {
 	switch format {
-	case "yaml", "y":
+	case "yaml", "y", "yml":
 		return YamlOutputFormat, nil
 	case "json", "j":
 		return JSONOutputFormat, nil
-	case "props", "p":
+	case "props", "p", "properties":
 		return PropsOutputFormat, nil
 	case "csv", "c":
 		return CSVOutputFormat, nil

From 360a47fddc28a451dcf5bd3720c33c7f44a2088d Mon Sep 17 00:00:00 2001
From: Mike Farah <mikefarah@gmail.com>
Date: Wed, 15 Mar 2023 13:57:47 +1100
Subject: [PATCH 53/53] Fixed npe in log #1596

---
 pkg/yqlib/operator_variables.go      | 3 ++-
 pkg/yqlib/operator_variables_test.go | 6 ++++++
 2 files changed, 8 insertions(+), 1 deletion(-)

diff --git a/pkg/yqlib/operator_variables.go b/pkg/yqlib/operator_variables.go
index d9baded9..32248ffc 100644
--- a/pkg/yqlib/operator_variables.go
+++ b/pkg/yqlib/operator_variables.go
@@ -82,10 +82,11 @@ func variableLoopSingleChild(d *dataTreeNavigator, context Context, originalExp
 		newContext.SetVariable(variableName, variableValue)
 
 		rhs, err := d.GetMatchingNodes(newContext, originalExp.RHS)
-		log.Debug("PROCESSING VARIABLE DONE, got back: ", rhs.MatchingNodes.Len())
+
 		if err != nil {
 			return Context{}, err
 		}
+		log.Debug("PROCESSING VARIABLE DONE, got back: ", rhs.MatchingNodes.Len())
 		results.PushBackList(rhs.MatchingNodes)
 	}
 
diff --git a/pkg/yqlib/operator_variables_test.go b/pkg/yqlib/operator_variables_test.go
index 64643ab0..f1555f28 100644
--- a/pkg/yqlib/operator_variables_test.go
+++ b/pkg/yqlib/operator_variables_test.go
@@ -13,6 +13,12 @@ var variableOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{}\n",
 		},
 	},
+	{
+		skipDoc:       true,
+		document:      `{}`,
+		expression:    `.a.b as $foo`,
+		expectedError: "must use variable with a pipe, e.g. `exp as $x | ...`",
+	},
 	{
 		document:      "a: [cat]",
 		skipDoc:       true,