Fixed variable operator to work like jq

Bumps golang from 1.20.0 to 1.20.1.

---
updated-dependencies:
- dependency-name: golang
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/go.yml

@@ -10,10 +10,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - name: Set up Go 1.19
+    - name: Set up Go
       uses: actions/setup-go@v3
       with:
-        go-version: 1.19
+        go-version: '^1.20'
       id: go
 
     - name: Check out code into the Go module directory

.github/workflows/release.yml

@@ -12,7 +12,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v3
         with:
-          go-version: '^1.19'
+          go-version: '^1.20'
           check-latest: true
       - name: Compile man page markup
         id: gen-man-page-md

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.19.5 as builder
+FROM golang:1.20.1 as builder
 
 WORKDIR /go/src/mikefarah/yq
 

Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM golang:1.19.5
+FROM golang:1.20.1
 
 COPY scripts/devtools.sh /opt/devtools.sh
 

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "v4.30.7"
+	Version = "v4.31.1"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

go.mod

@@ -1,12 +1,12 @@
 module github.com/mikefarah/yq/v4
 
 require (
-	github.com/a8m/envsubst v1.3.0
+	github.com/a8m/envsubst v1.4.1
 	github.com/alecthomas/participle/v2 v2.0.0-beta.5
 	github.com/alecthomas/repr v0.2.0
 	github.com/dimchansky/utfbom v1.1.1
 	github.com/elliotchance/orderedmap v1.5.0
-	github.com/fatih/color v1.13.0
+	github.com/fatih/color v1.14.1
 	github.com/goccy/go-json v0.10.0
 	github.com/goccy/go-yaml v1.9.8
 	github.com/jinzhu/copier v0.3.5
@@ -14,18 +14,18 @@ require (
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
 	github.com/spf13/cobra v1.6.1
 	github.com/spf13/pflag v1.0.5
-	golang.org/x/net v0.0.0-20220906165146-f3363e06e74c
+	golang.org/x/net v0.1.1-0.20221104162952-702349b0e862
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
-	github.com/mattn/go-colorable v0.1.12 // indirect
-	github.com/mattn/go-isatty v0.0.14 // indirect
-	golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
-	golang.org/x/text v0.3.8 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
+	golang.org/x/sys v0.3.0 // indirect
+	golang.org/x/text v0.4.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
 )
 
-go 1.19
+go 1.20

go.sum

@@ -1,5 +1,5 @@
-github.com/a8m/envsubst v1.3.0 h1:GmXKmVssap0YtlU3E230W98RWtWCyIZzjtf1apWWyAg=
-github.com/a8m/envsubst v1.3.0/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
+github.com/a8m/envsubst v1.4.1 h1:RShc2OKbQpIIp5qR2glVsBOJCImTVHFrDkJvmGh0Qi0=
+github.com/a8m/envsubst v1.4.1/go.mod h1:MVUTQNGQ3tsjOOtKCNd+fl8RzhsXcDvvAEzkhGtlsbY=
 github.com/alecthomas/assert/v2 v2.0.3 h1:WKqJODfOiQG0nEJKFKzDIG3E29CN2/4zR9XGJzKIkbg=
 github.com/alecthomas/participle/v2 v2.0.0-beta.5 h1:y6dsSYVb1G5eK6mgmy+BgI3Mw35a3WghArZ/Hbebrjo=
 github.com/alecthomas/participle/v2 v2.0.0-beta.5/go.mod h1:RC764t6n4L8D8ITAJv0qdokritYSNR3wV5cVwmIEaMM=
@@ -13,8 +13,8 @@ github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/
 github.com/elliotchance/orderedmap v1.5.0 h1:1IsExUsjv5XNBD3ZdC7jkAAqLWOOKdbPTmkHx63OsBg=
 github.com/elliotchance/orderedmap v1.5.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
-github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
-github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
+github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w=
+github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
@@ -32,12 +32,12 @@ github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgx
 github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
 github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
-github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
-github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
+github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -54,21 +54,20 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20220906165146-f3363e06e74c h1:yKufUcDwucU5urd+50/Opbt4AYpqthk7wHpHok8f1lo=
-golang.org/x/net v0.0.0-20220906165146-f3363e06e74c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
+golang.org/x/net v0.1.1-0.20221104162952-702349b0e862 h1:KrLJ+iz8J6j6VVr/OCfULAcK+xozUmWE43fKpMR4MlI=
+golang.org/x/net v0.1.1-0.20221104162952-702349b0e862/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg=
-golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
+golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
-golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=
+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f h1:uF6paiQQebLeSXkrTqHqz0MXhXXS1KgF41eUdBNvxK0=

pkg/yqlib/chown_linux.go

@@ -12,7 +12,14 @@ func changeOwner(info fs.FileInfo, file *os.File) error {
 	if stat, ok := info.Sys().(*syscall.Stat_t); ok {
 		uid := int(stat.Uid)
 		gid := int(stat.Gid)
-		return os.Chown(file.Name(), uid, gid)
+
+		err := os.Chown(file.Name(), uid, gid)
+		if err != nil {
+			// this happens with snap confinement
+			// not really a big issue as users can chown
+			// the file themselves if required.
+			log.Info("Skipping chown: %v", err)
+		}
 	}
 	return nil
 }

pkg/yqlib/decoder.go

@@ -15,6 +15,7 @@ const (
 	JsonInputFormat
 	CSVObjectInputFormat
 	TSVObjectInputFormat
+	UriInputFormat
 )
 
 type Decoder interface {

pkg/yqlib/decoder_uri.go

@@ -0,0 +1,60 @@
+package yqlib
+
+import (
+	"bytes"
+	"io"
+	"net/url"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+type uriDecoder struct {
+	reader       io.Reader
+	finished     bool
+	readAnything bool
+}
+
+func NewUriDecoder() Decoder {
+	return &uriDecoder{finished: false}
+}
+
+func (dec *uriDecoder) Init(reader io.Reader) error {
+	dec.reader = reader
+	dec.readAnything = false
+	dec.finished = false
+	return nil
+}
+
+func (dec *uriDecoder) Decode() (*CandidateNode, error) {
+	if dec.finished {
+		return nil, io.EOF
+	}
+
+	buf := new(bytes.Buffer)
+
+	if _, err := buf.ReadFrom(dec.reader); err != nil {
+		return nil, err
+	}
+	if buf.Len() == 0 {
+		dec.finished = true
+
+		// if we've read _only_ an empty string, lets return that
+		// otherwise if we've already read some bytes, and now we get
+		// an empty string, then we are done.
+		if dec.readAnything {
+			return nil, io.EOF
+		}
+	}
+	newValue, err := url.QueryUnescape(buf.String())
+	if err != nil {
+		return nil, err
+	}
+	dec.readAnything = true
+	return &CandidateNode{
+		Node: &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!str",
+			Value: newValue,
+		},
+	}, nil
+}

pkg/yqlib/doc/operators/datetime.md

@@ -86,6 +86,30 @@ a: cool
 updated: 2021-05-19T01:02:03Z
 ```
 
+## From Unix
+Converts from unix time. Note, you don't have to pipe through the tz operator :)
+
+Running
+```bash
+yq --null-input '1675301929 | from_unix | tz("UTC")'
+```
+will output
+```yaml
+2023-02-02T01:38:49Z
+```
+
+## To Unix
+Converts to unix time
+
+Running
+```bash
+yq --null-input 'now | to_unix'
+```
+will output
+```yaml
+1621386123
+```
+
 ## Timezone: from standard RFC3339 format
 Returns a new datetime in the specified timezone. Specify standard IANA Time Zone format or 'utc', 'local'. When given a single parameter, this assumes the datetime is in RFC3339 format.
 

pkg/yqlib/doc/operators/encode-decode.md

@@ -16,6 +16,8 @@ These operators are useful to process yaml documents that have stringified embed
 | TSV | from_tsv/@tsvd | to_tsv/@tsv |
 | XML | from_xml/@xmld | to_xml(i)/@xml |
 | Base64 | @base64d | @base64 |
+| URI | @urid | @uri |
+| Shell |  | @sh |
 
 
 See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) for accepted formats.
@@ -435,6 +437,50 @@ will output
 YTogYXBwbGUK
 ```
 
+## Encode a string to uri
+Given a sample.yml file of:
+```yaml
+coolData: this has & special () characters *
+```
+then
+```bash
+yq '.coolData | @uri' sample.yml
+```
+will output
+```yaml
+this+has+%26+special+%28%29+characters+%2A
+```
+
+## Decode a URI to a string
+Given a sample.yml file of:
+```yaml
+this+has+%26+special+%28%29+characters+%2A
+```
+then
+```bash
+yq '@urid' sample.yml
+```
+will output
+```yaml
+this has & special () characters *
+```
+
+## Encode a string to sh
+Sh/Bash friendly string
+
+Given a sample.yml file of:
+```yaml
+coolData: strings with spaces and a 'quote'
+```
+then
+```bash
+yq '.coolData | @sh' sample.yml
+```
+will output
+```yaml
+strings' with spaces and a '\'quote\'
+```
+
 ## Decode a base64 encoded string
 Decoded data is assumed to be a string.
 

pkg/yqlib/doc/operators/headers/encode-decode.md

@@ -16,6 +16,8 @@ These operators are useful to process yaml documents that have stringified embed
 | TSV | from_tsv/@tsvd | to_tsv/@tsv |
 | XML | from_xml/@xmld | to_xml(i)/@xml |
 | Base64 | @base64d | @base64 |
+| URI | @urid | @uri |
+| Shell |  | @sh |
 
 
 See CSV and TSV [documentation](https://mikefarah.gitbook.io/yq/usage/csv-tsv) for accepted formats.

pkg/yqlib/doc/operators/headers/shuffle.md

@@ -0,0 +1,4 @@
+# Shuffle
+
+Shuffles an array. Note that this command does _not_ use a cryptographically secure random number generator to randomise the array order.
+

pkg/yqlib/doc/operators/shuffle.md

@@ -0,0 +1,51 @@
+# Shuffle
+
+Shuffles an array. Note that this command does _not_ use a cryptographically secure random number generator to randomise the array order.
+
+
+## Shuffle array
+Given a sample.yml file of:
+```yaml
+- 1
+- 2
+- 3
+- 4
+- 5
+```
+then
+```bash
+yq 'shuffle' sample.yml
+```
+will output
+```yaml
+- 5
+- 2
+- 4
+- 1
+- 3
+```
+
+## Shuffle array in place
+Given a sample.yml file of:
+```yaml
+cool:
+  - 1
+  - 2
+  - 3
+  - 4
+  - 5
+```
+then
+```bash
+yq '.cool |= shuffle' sample.yml
+```
+will output
+```yaml
+cool:
+  - 5
+  - 2
+  - 4
+  - 1
+  - 3
+```
+

pkg/yqlib/doc/operators/sort.md

@@ -25,6 +25,28 @@ will output
 - a: cat
 ```
 
+## Sort by multiple fields
+Given a sample.yml file of:
+```yaml
+- a: dog
+- a: cat
+  b: banana
+- a: cat
+  b: apple
+```
+then
+```bash
+yq 'sort_by(.a, .b)' sample.yml
+```
+will output
+```yaml
+- a: cat
+  b: apple
+- a: cat
+  b: banana
+- a: dog
+```
+
 ## Sort descending by string field
 Use sort with reverse to sort in descending order.
 

pkg/yqlib/encoder_sh.go

@@ -0,0 +1,79 @@
+package yqlib
+
+import (
+	"fmt"
+	"io"
+	"regexp"
+	"strings"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+var unsafeChars = regexp.MustCompile(`[^\w@%+=:,./-]`)
+
+type shEncoder struct {
+	quoteAll bool
+}
+
+func NewShEncoder() Encoder {
+	return &shEncoder{false}
+}
+
+func (e *shEncoder) CanHandleAliases() bool {
+	return false
+}
+
+func (e *shEncoder) PrintDocumentSeparator(writer io.Writer) error {
+	return nil
+}
+
+func (e *shEncoder) PrintLeadingContent(writer io.Writer, content string) error {
+	return nil
+}
+
+func (e *shEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error {
+	node := unwrapDoc(originalNode)
+	if guessTagFromCustomType(node) != "!!str" {
+		return fmt.Errorf("cannot encode %v as URI, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag)
+	}
+
+	return writeString(writer, e.encode(originalNode.Value))
+}
+
+// put any (shell-unsafe) characters into a single-quoted block, close the block lazily
+func (e *shEncoder) encode(input string) string {
+	const quote = '\''
+	var inQuoteBlock = false
+	var encoded strings.Builder
+	encoded.Grow(len(input))
+
+	for _, ir := range input {
+		// open or close a single-quote block
+		if ir == quote {
+			if inQuoteBlock {
+				// get out of a quote block for an input quote
+				encoded.WriteRune(quote)
+				inQuoteBlock = !inQuoteBlock
+			}
+			// escape the quote with a backslash
+			encoded.WriteRune('\\')
+		} else {
+			if e.shouldQuote(ir) && !inQuoteBlock {
+				// start a quote block for any (unsafe) characters
+				encoded.WriteRune(quote)
+				inQuoteBlock = !inQuoteBlock
+			}
+		}
+		// pass on the input character
+		encoded.WriteRune(ir)
+	}
+	// close any pending quote block
+	if inQuoteBlock {
+		encoded.WriteRune(quote)
+	}
+	return encoded.String()
+}
+
+func (e *shEncoder) shouldQuote(ir rune) bool {
+	return e.quoteAll || unsafeChars.MatchString(string(ir))
+}

pkg/yqlib/encoder_uri.go

@@ -0,0 +1,37 @@
+package yqlib
+
+import (
+	"fmt"
+	"io"
+	"net/url"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+type uriEncoder struct {
+}
+
+func NewUriEncoder() Encoder {
+	return &uriEncoder{}
+}
+
+func (e *uriEncoder) CanHandleAliases() bool {
+	return false
+}
+
+func (e *uriEncoder) PrintDocumentSeparator(writer io.Writer) error {
+	return nil
+}
+
+func (e *uriEncoder) PrintLeadingContent(writer io.Writer, content string) error {
+	return nil
+}
+
+func (e *uriEncoder) Encode(writer io.Writer, originalNode *yaml.Node) error {
+	node := unwrapDoc(originalNode)
+	if guessTagFromCustomType(node) != "!!str" {
+		return fmt.Errorf("cannot encode %v as URI, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag)
+	}
+	_, err := writer.Write([]byte(url.QueryEscape(originalNode.Value)))
+	return err
+}

pkg/yqlib/lexer_participle.go

@@ -45,8 +45,11 @@ var participleYqRules = []*participleYqRule{
 	simpleOp("format_datetime", formatDateTimeOpType),
 	simpleOp("now", nowOpType),
 	simpleOp("tz", tzOpType),
+	simpleOp("from_?unix", fromUnixOpType),
+	simpleOp("to_?unix", toUnixOpType),
 	simpleOp("with_dtf", withDtFormatOpType),
 	simpleOp("error", errorOpType),
+	simpleOp("shuffle", shuffleOpType),
 	simpleOp("sortKeys", sortKeysOpType),
 	simpleOp("sort_?keys", sortKeysOpType),
 
@@ -78,6 +81,10 @@ var participleYqRules = []*participleYqRule{
 	{"Base64d", `@base64d`, decodeOp(Base64InputFormat), 0},
 	{"Base64", `@base64`, encodeWithIndent(Base64OutputFormat, 0), 0},
 
+	{"Urid", `@urid`, decodeOp(UriInputFormat), 0},
+	{"Uri", `@uri`, encodeWithIndent(UriOutputFormat, 0), 0},
+	{"SH", `@sh`, encodeWithIndent(ShOutputFormat, 0), 0},
+
 	{"LoadXML", `load_?xml|xml_?load`, loadOp(NewXMLDecoder(ConfiguredXMLPreferences), false), 0},
 
 	{"LoadBase64", `load_?base64`, loadOp(NewBase64Decoder(), false), 0},

pkg/yqlib/lib.go

@@ -53,7 +53,7 @@ var subtractAssignOpType = &operationType{Type: "SUBTRACT_ASSIGN", NumArgs: 2, P
 
 var assignAttributesOpType = &operationType{Type: "ASSIGN_ATTRIBUTES", NumArgs: 2, Precedence: 40, Handler: assignAttributesOperator}
 var assignStyleOpType = &operationType{Type: "ASSIGN_STYLE", NumArgs: 2, Precedence: 40, Handler: assignStyleOperator}
-var assignVariableOpType = &operationType{Type: "ASSIGN_VARIABLE", NumArgs: 2, Precedence: 40, Handler: assignVariableOperator}
+var assignVariableOpType = &operationType{Type: "ASSIGN_VARIABLE", NumArgs: 2, Precedence: 40, Handler: useWithPipe}
 var assignTagOpType = &operationType{Type: "ASSIGN_TAG", NumArgs: 2, Precedence: 40, Handler: assignTagOperator}
 var assignCommentOpType = &operationType{Type: "ASSIGN_COMMENT", NumArgs: 2, Precedence: 40, Handler: assignCommentsOperator}
 var assignAnchorOpType = &operationType{Type: "ASSIGN_ANCHOR", NumArgs: 2, Precedence: 40, Handler: assignAnchorOperator}
@@ -93,6 +93,8 @@ var formatDateTimeOpType = &operationType{Type: "FORMAT_DATE_TIME", NumArgs: 1,
 var withDtFormatOpType = &operationType{Type: "WITH_DATE_TIME_FORMAT", NumArgs: 1, Precedence: 50, Handler: withDateTimeFormat}
 var nowOpType = &operationType{Type: "NOW", NumArgs: 0, Precedence: 50, Handler: nowOp}
 var tzOpType = &operationType{Type: "TIMEZONE", NumArgs: 1, Precedence: 50, Handler: tzOp}
+var fromUnixOpType = &operationType{Type: "FROM_UNIX", NumArgs: 0, Precedence: 50, Handler: fromUnixOp}
+var toUnixOpType = &operationType{Type: "TO_UNIX", NumArgs: 0, Precedence: 50, Handler: toUnixOp}
 
 var encodeOpType = &operationType{Type: "ENCODE", NumArgs: 0, Precedence: 50, Handler: encodeOperator}
 var decodeOpType = &operationType{Type: "DECODE", NumArgs: 0, Precedence: 50, Handler: decodeOperator}
@@ -133,6 +135,7 @@ var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50,
 var sortByOpType = &operationType{Type: "SORT_BY", NumArgs: 1, Precedence: 50, Handler: sortByOperator}
 var reverseOpType = &operationType{Type: "REVERSE", NumArgs: 0, Precedence: 50, Handler: reverseOperator}
 var sortOpType = &operationType{Type: "SORT", NumArgs: 0, Precedence: 50, Handler: sortOperator}
+var shuffleOpType = &operationType{Type: "SHUFFLE", NumArgs: 0, Precedence: 50, Handler: shuffleOperator}
 
 var sortKeysOpType = &operationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: sortKeysOperator}
 

pkg/yqlib/operator_add_test.go

@@ -34,7 +34,7 @@ var addOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{}`,
-		expression: "(.a + .b) as $x",
+		expression: "(.a + .b) as $x | .",
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},

pkg/yqlib/operator_alternative_test.go

@@ -16,7 +16,7 @@ var alternativeOperatorScenarios = []expressionScenario{
 	},
 	{
 		skipDoc:    true,
-		expression: `(.b // "hello") as $x`,
+		expression: `(.b // "hello") as $x | .`,
 		document:   `a: bridge`,
 		expected: []string{
 			"D0, P[], (doc)::a: bridge\n",

pkg/yqlib/operator_booleans_test.go

@@ -102,7 +102,7 @@ var booleanOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `[{pet: cat}]`,
-		expression: `any_c(.name == "harry") as $c`,
+		expression: `any_c(.name == "harry") as $c | .`,
 		expected: []string{
 			"D0, P[], (doc)::[{pet: cat}]\n",
 		},
@@ -110,9 +110,17 @@ var booleanOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `[{pet: cat}]`,
-		expression: `all_c(.name == "harry") as $c`,
+		expression: `any_c(.name == "harry") as $c | $c`,
 		expected: []string{
-			"D0, P[], (doc)::[{pet: cat}]\n",
+			"D0, P[], (!!bool)::false\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{pet: cat}]`,
+		expression: `all_c(.name == "harry") as $c | $c`,
+		expected: []string{
+			"D0, P[], (!!bool)::false\n",
 		},
 	},
 	{
@@ -185,7 +193,7 @@ var booleanOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{}`,
-		expression: `(.a.b or .c) as $x`,
+		expression: `(.a.b or .c) as $x | .`,
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},
@@ -193,7 +201,7 @@ var booleanOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{}`,
-		expression: `(.a.b and .c) as $x`,
+		expression: `(.a.b and .c) as $x | .`,
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},

pkg/yqlib/operator_compare.go

@@ -3,7 +3,6 @@ package yqlib
 import (
 	"fmt"
 	"strconv"
-	"time"
 
 	yaml "gopkg.in/yaml.v3"
 )
@@ -80,7 +79,7 @@ func compareScalars(context Context, prefs compareTypePref, lhs *yaml.Node, rhs
 
 	isDateTime := lhs.Tag == "!!timestamp"
 	// if the lhs is a string, it might be a timestamp in a custom format.
-	if lhsTag == "!!str" && context.GetDateTimeLayout() != time.RFC3339 {
+	if lhsTag == "!!str" {
 		_, err := parseDateTime(context.GetDateTimeLayout(), lhs.Value)
 		isDateTime = err == nil
 	}

pkg/yqlib/operator_datetime.go

@@ -4,6 +4,7 @@ import (
 	"container/list"
 	"errors"
 	"fmt"
+	"strconv"
 	"time"
 
 	"gopkg.in/yaml.v3"
@@ -129,3 +130,69 @@ func tzOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode)
 
 	return context.ChildContext(results), nil
 }
+
+func parseUnixTime(unixTime string) (time.Time, error) {
+	seconds, err := strconv.ParseFloat(unixTime, 64)
+
+	if err != nil {
+		return time.Now(), err
+	}
+
+	return time.UnixMilli(int64(seconds * 1000)), nil
+}
+
+func fromUnixOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+
+		actualTag := guessTagFromCustomType(candidate.Node)
+
+		if actualTag != "!!int" && guessTagFromCustomType(candidate.Node) != "!!float" {
+			return Context{}, fmt.Errorf("from_unix only works on numbers, found %v instead", candidate.Node.Tag)
+		}
+
+		parsedTime, err := parseUnixTime(candidate.Node.Value)
+		if err != nil {
+			return Context{}, err
+		}
+
+		node := &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!timestamp",
+			Value: parsedTime.Format(time.RFC3339),
+		}
+
+		results.PushBack(candidate.CreateReplacement(node))
+	}
+
+	return context.ChildContext(results), nil
+}
+
+func toUnixOp(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	layout := context.GetDateTimeLayout()
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+
+		parsedTime, err := parseDateTime(layout, candidate.Node.Value)
+		if err != nil {
+			return Context{}, fmt.Errorf("could not parse datetime of [%v] using layout [%v]: %w", candidate.GetNicePath(), layout, err)
+		}
+
+		node := &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!int",
+			Value: fmt.Sprintf("%v", parsedTime.Unix()),
+		}
+
+		results.PushBack(candidate.CreateReplacement(node))
+	}
+
+	return context.ChildContext(results), nil
+}

pkg/yqlib/operator_datetime_test.go

@@ -39,6 +39,22 @@ var dateTimeOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: cool\nupdated: 2021-05-19T01:02:03Z\n",
 		},
 	},
+	{
+		description:    "From Unix",
+		subdescription: "Converts from unix time. Note, you don't have to pipe through the tz operator :)",
+		expression:     `1675301929 | from_unix | tz("UTC")`,
+		expected: []string{
+			"D0, P[], (!!timestamp)::2023-02-02T01:38:49Z\n",
+		},
+	},
+	{
+		description:    "To Unix",
+		subdescription: "Converts to unix time",
+		expression:     `now | to_unix`,
+		expected: []string{
+			"D0, P[], (!!int)::1621386123\n",
+		},
+	},
 	{
 		description:    "Timezone: from standard RFC3339 format",
 		subdescription: "Returns a new datetime in the specified timezone. Specify standard IANA Time Zone format or 'utc', 'local'. When given a single parameter, this assumes the datetime is in RFC3339 format.",

pkg/yqlib/operator_encoder_decoder.go

@@ -26,6 +26,10 @@ func configureEncoder(format PrinterOutputFormat, indent int) Encoder {
 		return NewXMLEncoder(indent, ConfiguredXMLPreferences)
 	case Base64OutputFormat:
 		return NewBase64Encoder()
+	case UriOutputFormat:
+		return NewUriEncoder()
+	case ShOutputFormat:
+		return NewShEncoder()
 	}
 	panic("invalid encoder")
 }
@@ -113,6 +117,8 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 		decoder = NewCSVObjectDecoder(',')
 	case TSVObjectInputFormat:
 		decoder = NewCSVObjectDecoder('\t')
+	case UriInputFormat:
+		decoder = NewUriDecoder()
 	}
 
 	var results = list.New()

pkg/yqlib/operator_encoder_decoder_test.go

@@ -241,6 +241,41 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!str)::YTogYXBwbGUK\n",
 		},
 	},
+	{
+		description: "Encode a string to uri",
+		document:    "coolData: this has & special () characters *",
+		expression:  ".coolData | @uri",
+		expected: []string{
+			"D0, P[coolData], (!!str)::this+has+%26+special+%28%29+characters+%2A\n",
+		},
+	},
+	{
+		description: "Decode a URI to a string",
+		document:    "this+has+%26+special+%28%29+characters+%2A",
+		expression:  "@urid",
+		expected: []string{
+			"D0, P[], (!!str)::this has & special () characters *\n",
+		},
+	},
+	{
+		description:    "Encode a string to sh",
+		subdescription: "Sh/Bash friendly string",
+		document:       "coolData: strings with spaces and a 'quote'",
+		expression:     ".coolData | @sh",
+		expected: []string{
+			"D0, P[coolData], (!!str)::strings' with spaces and a '\\'quote\\'\n",
+		},
+	},
+	{
+		description:    "Encode a string to sh",
+		subdescription: "Watch out for stray '' (empty strings)",
+		document:       "coolData: \"'starts, contains more '' and ends with a quote'\"",
+		expression:     ".coolData | @sh",
+		expected: []string{
+			"D0, P[coolData], (!!str)::\\'starts,' contains more '\\'\\'' and ends with a quote'\\'\n",
+		},
+		skipDoc: true,
+	},
 	{
 		description:    "Decode a base64 encoded string",
 		subdescription: "Decoded data is assumed to be a string.",

pkg/yqlib/operator_equals_test.go

@@ -47,7 +47,7 @@ var equalsOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   "{}",
-		expression: "(.a == .b) as $x",
+		expression: "(.a == .b) as $x | .",
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},
@@ -63,7 +63,7 @@ var equalsOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   "{}",
-		expression: "(.a != .b) as $x",
+		expression: "(.a != .b) as $x | .",
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},

pkg/yqlib/operator_has_test.go

@@ -16,7 +16,7 @@ var hasOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `a: hello`,
-		expression: `has(.b) as $c`,
+		expression: `has(.b) as $c | .`,
 		expected: []string{
 			"D0, P[], (doc)::a: hello\n",
 		},

pkg/yqlib/operator_pipe.go

@@ -2,9 +2,9 @@ package yqlib
 
 func pipeOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 
-	//lhs may update the variable context, we should pass that into the RHS
-	// BUT we still return the original context back (see jq)
-	// https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|...
+	if expressionNode.LHS.Operation.OperationType == assignVariableOpType {
+		return variableLoop(d, context, expressionNode)
+	}
 
 	lhs, err := d.GetMatchingNodes(context, expressionNode.LHS)
 	if err != nil {

pkg/yqlib/operator_shuffle.go

@@ -0,0 +1,37 @@
+package yqlib
+
+import (
+	"container/list"
+	"fmt"
+	"math/rand"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+func shuffleOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	// ignore CWE-338 gosec issue of not using crypto/rand
+	// this is just to shuffle an array rather generating a
+	// secret or something that needs proper rand.
+	myRand := rand.New(rand.NewSource(Now().UnixNano())) // #nosec
+
+	results := list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+
+		candidateNode := unwrapDoc(candidate.Node)
+
+		if candidateNode.Kind != yaml.SequenceNode {
+			return context, fmt.Errorf("node at path [%v] is not an array (it's a %v)", candidate.GetNicePath(), candidate.GetNiceTag())
+		}
+
+		result := deepClone(candidateNode)
+
+		a := result.Content
+
+		myRand.Shuffle(len(a), func(i, j int) { a[i], a[j] = a[j], a[i] })
+		results.PushBack(candidate.CreateReplacement(result))
+	}
+	return context.ChildContext(results), nil
+}

pkg/yqlib/operator_shuffle_test.go

@@ -0,0 +1,30 @@
+package yqlib
+
+import "testing"
+
+var shuffleOperatorScenarios = []expressionScenario{
+	{
+		description: "Shuffle array",
+		document:    "[1, 2, 3, 4, 5]",
+		expression:  `shuffle`,
+		expected: []string{
+			"D0, P[], (!!seq)::[5, 2, 4, 1, 3]\n",
+		},
+	},
+
+	{
+		description: "Shuffle array in place",
+		document:    "cool: [1, 2, 3, 4, 5]",
+		expression:  `.cool |= shuffle`,
+		expected: []string{
+			"D0, P[], (doc)::cool: [5, 2, 4, 1, 3]\n",
+		},
+	},
+}
+
+func TestShuffleByOperatorScenarios(t *testing.T) {
+	for _, tt := range shuffleOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentOperatorScenarios(t, "shuffle", shuffleOperatorScenarios)
+}

pkg/yqlib/operator_sort.go

@@ -42,18 +42,7 @@ func sortByOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 				return Context{}, err
 			}
 
-			nodeToCompare := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!null"}
-			if compareContext.MatchingNodes.Len() > 0 {
-				nodeToCompare = compareContext.MatchingNodes.Front().Value.(*CandidateNode).Node
-			}
-
-			log.Debug("going to compare %v by %v", NodeToString(candidate.CreateReplacement(originalNode)), NodeToString(candidate.CreateReplacement(nodeToCompare)))
-
-			sortableArray[i] = sortableNode{Node: originalNode, NodeToCompare: nodeToCompare, dateTimeLayout: context.GetDateTimeLayout()}
-
-			if nodeToCompare.Kind != yaml.ScalarNode {
-				return Context{}, fmt.Errorf("sort only works for scalars, got %v", nodeToCompare.Tag)
-			}
+			sortableArray[i] = sortableNode{Node: originalNode, CompareContext: compareContext, dateTimeLayout: context.GetDateTimeLayout()}
 
 		}
 
@@ -72,7 +61,7 @@ func sortByOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 
 type sortableNode struct {
 	Node           *yaml.Node
-	NodeToCompare  *yaml.Node
+	CompareContext Context
 	dateTimeLayout string
 }
 
@@ -82,9 +71,28 @@ func (a sortableNodeArray) Len() int      { return len(a) }
 func (a sortableNodeArray) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
 
 func (a sortableNodeArray) Less(i, j int) bool {
-	lhs := a[i].NodeToCompare
-	rhs := a[j].NodeToCompare
+	lhsContext := a[i].CompareContext
+	rhsContext := a[j].CompareContext
 
+	rhsEl := rhsContext.MatchingNodes.Front()
+	for lhsEl := lhsContext.MatchingNodes.Front(); lhsEl != nil && rhsEl != nil; lhsEl = lhsEl.Next() {
+		lhs := lhsEl.Value.(*CandidateNode)
+		rhs := rhsEl.Value.(*CandidateNode)
+
+		result := a.compare(lhs.Node, rhs.Node, a[i].dateTimeLayout)
+
+		if result < 0 {
+			return true
+		} else if result > 0 {
+			return false
+		}
+
+		rhsEl = rhsEl.Next()
+	}
+	return false
+}
+
+func (a sortableNodeArray) compare(lhs *yaml.Node, rhs *yaml.Node, dateTimeLayout string) int {
 	lhsTag := lhs.Tag
 	rhsTag := rhs.Tag
 
@@ -99,7 +107,7 @@ func (a sortableNodeArray) Less(i, j int) bool {
 	}
 
 	isDateTime := lhsTag == "!!timestamp" && rhsTag == "!!timestamp"
-	layout := a[i].dateTimeLayout
+	layout := dateTimeLayout
 	// if the lhs is a string, it might be a timestamp in a custom format.
 	if lhsTag == "!!str" && layout != time.RFC3339 {
 		_, errLhs := parseDateTime(layout, lhs.Value)
@@ -108,13 +116,13 @@ func (a sortableNodeArray) Less(i, j int) bool {
 	}
 
 	if lhsTag == "!!null" && rhsTag != "!!null" {
-		return true
+		return -1
 	} else if lhsTag != "!!null" && rhsTag == "!!null" {
-		return false
+		return 1
 	} else if lhsTag == "!!bool" && rhsTag != "!!bool" {
-		return true
+		return -1
 	} else if lhsTag != "!!bool" && rhsTag == "!!bool" {
-		return false
+		return 1
 	} else if lhsTag == "!!bool" && rhsTag == "!!bool" {
 		lhsTruthy, err := isTruthyNode(lhs)
 		if err != nil {
@@ -125,20 +133,30 @@ func (a sortableNodeArray) Less(i, j int) bool {
 		if err != nil {
 			panic(fmt.Errorf("could not parse %v as boolean: %w", rhs.Value, err))
 		}
-
-		return !lhsTruthy && rhsTruthy
+		if lhsTruthy == rhsTruthy {
+			return 0
+		} else if lhsTruthy {
+			return 1
+		}
+		return -1
 	} else if isDateTime {
 		lhsTime, err := parseDateTime(layout, lhs.Value)
 		if err != nil {
 			log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %w", lhs.Value, layout, err)
-			return strings.Compare(lhs.Value, rhs.Value) < 0
+			return strings.Compare(lhs.Value, rhs.Value)
 		}
 		rhsTime, err := parseDateTime(layout, rhs.Value)
 		if err != nil {
 			log.Warningf("Could not parse time %v with layout %v for sort, sorting by string instead: %w", rhs.Value, layout, err)
-			return strings.Compare(lhs.Value, rhs.Value) < 0
+			return strings.Compare(lhs.Value, rhs.Value)
 		}
-		return lhsTime.Before(rhsTime)
+		if lhsTime.Equal(rhsTime) {
+			return 0
+		} else if lhsTime.Before(rhsTime) {
+			return -1
+		}
+
+		return 1
 	} else if lhsTag == "!!int" && rhsTag == "!!int" {
 		_, lhsNum, err := parseInt64(lhs.Value)
 		if err != nil {
@@ -148,7 +166,7 @@ func (a sortableNodeArray) Less(i, j int) bool {
 		if err != nil {
 			panic(err)
 		}
-		return lhsNum < rhsNum
+		return int(lhsNum - rhsNum)
 	} else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") {
 		lhsNum, err := strconv.ParseFloat(lhs.Value, 64)
 		if err != nil {
@@ -158,8 +176,14 @@ func (a sortableNodeArray) Less(i, j int) bool {
 		if err != nil {
 			panic(err)
 		}
-		return lhsNum < rhsNum
+		if lhsNum == rhsNum {
+			return 0
+		} else if lhsNum < rhsNum {
+			return -1
+		}
+
+		return 1
 	}
 
-	return strings.Compare(lhs.Value, rhs.Value) < 0
+	return strings.Compare(lhs.Value, rhs.Value)
 }

pkg/yqlib/operator_sort_test.go

@@ -11,6 +11,32 @@ var sortByOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::[{a: apple}, {a: banana}, {a: cat}]\n",
 		},
 	},
+	{
+		description: "Sort by multiple fields",
+		document:    "[{a: dog},{a: cat, b: banana},{a: cat, b: apple}]",
+		expression:  `sort_by(.a, .b)`,
+		expected: []string{
+			"D0, P[], (!!seq)::[{a: cat, b: apple}, {a: cat, b: banana}, {a: dog}]\n",
+		},
+	},
+	{
+		description: "Sort by multiple fields",
+		skipDoc:     true,
+		document:    "[{a: dog, b: good},{a: cat, c: things},{a: cat, b: apple}]",
+		expression:  `sort_by(.a, .b)`,
+		expected: []string{
+			"D0, P[], (!!seq)::[{a: cat, c: things}, {a: cat, b: apple}, {a: dog, b: good}]\n",
+		},
+	},
+	{
+		description: "Sort by multiple fields",
+		skipDoc:     true,
+		document:    "[{a: dog, b: 0.1},{a: cat, b: 0.01},{a: cat, b: 0.001}]",
+		expression:  `sort_by(.a, .b)`,
+		expected: []string{
+			"D0, P[], (!!seq)::[{a: cat, b: 0.001}, {a: cat, b: 0.01}, {a: dog, b: 0.1}]\n",
+		},
+	},
 	{
 		description:    "Sort descending by string field",
 		subdescription: "Use sort with reverse to sort in descending order.",

pkg/yqlib/operator_subtract_test.go

@@ -8,7 +8,7 @@ var subtractOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{}`,
-		expression: "(.a - .b) as $x",
+		expression: "(.a - .b) as $x | .",
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},

pkg/yqlib/operator_traverse_path_test.go

@@ -151,7 +151,7 @@ var traversePathOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `c: dog`,
-		expression: `.[.a.b] as $x`,
+		expression: `.[.a.b] as $x | .`,
 		expected: []string{
 			"D0, P[], (doc)::c: dog\n",
 		},

pkg/yqlib/operator_union_test.go

@@ -8,7 +8,7 @@ var unionOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   "{}",
-		expression: `(.a, .b.c) as $x`,
+		expression: `(.a, .b.c) as $x | .`,
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},

pkg/yqlib/operator_variables.go

@@ -19,24 +19,81 @@ type assignVarPreferences struct {
 	IsReference bool
 }
 
-func assignVariableOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
-	lhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.LHS)
+func useWithPipe(d *dataTreeNavigator, context Context, originalExp *ExpressionNode) (Context, error) {
+	return Context{}, fmt.Errorf("must use variable with a pipe, e.g. `exp as $x | ...`")
+}
+
+// variables are like loops in jq
+// https://stedolan.github.io/jq/manual/#Variable
+func variableLoop(d *dataTreeNavigator, context Context, originalExp *ExpressionNode) (Context, error) {
+	log.Debug("variable loop!")
+	results := list.New()
+	var evaluateAllTogether = true
+	for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {
+		evaluateAllTogether = evaluateAllTogether && matchEl.Value.(*CandidateNode).EvaluateTogether
+		if !evaluateAllTogether {
+			break
+		}
+	}
+	if evaluateAllTogether {
+		return variableLoopSingleChild(d, context, originalExp)
+	}
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		result, err := variableLoopSingleChild(d, context.SingleChildContext(el.Value.(*CandidateNode)), originalExp)
+		if err != nil {
+			return Context{}, err
+		}
+		results.PushBackList(result.MatchingNodes)
+	}
+	return context.ChildContext(results), nil
+}
+
+func variableLoopSingleChild(d *dataTreeNavigator, context Context, originalExp *ExpressionNode) (Context, error) {
+
+	variableExp := originalExp.LHS
+	lhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), variableExp.LHS)
 	if err != nil {
 		return Context{}, err
 	}
-	if expressionNode.RHS.Operation.OperationType.Type != "GET_VARIABLE" {
+	if variableExp.RHS.Operation.OperationType.Type != "GET_VARIABLE" {
 		return Context{}, fmt.Errorf("RHS of 'as' operator must be a variable name e.g. $foo")
 	}
-	variableName := expressionNode.RHS.Operation.StringValue
+	variableName := variableExp.RHS.Operation.StringValue
 
-	prefs := expressionNode.Operation.Preferences.(assignVarPreferences)
+	prefs := variableExp.Operation.Preferences.(assignVarPreferences)
 
-	var variableValue *list.List
-	if prefs.IsReference {
-		variableValue = lhs.MatchingNodes
-	} else {
-		variableValue = lhs.DeepClone().MatchingNodes
+	results := list.New()
+
+	// now we loop over lhs, set variable to each result and calculate originalExp.Rhs
+	for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() {
+		log.Debug("PROCESSING VARIABLE: ", NodeToString(el.Value.(*CandidateNode)))
+		var variableValue = list.New()
+		if prefs.IsReference {
+			variableValue.PushBack(el.Value)
+		} else {
+			candidateCopy, err := el.Value.(*CandidateNode).Copy()
+			if err != nil {
+				return Context{}, err
+			}
+			variableValue.PushBack(candidateCopy)
+		}
+		newContext := context.ChildContext(context.MatchingNodes)
+		newContext.SetVariable(variableName, variableValue)
+
+		rhs, err := d.GetMatchingNodes(newContext, originalExp.RHS)
+		log.Debug("PROCESSING VARIABLE DONE, got back: ", rhs.MatchingNodes.Len())
+		if err != nil {
+			return Context{}, err
+		}
+		results.PushBackList(rhs.MatchingNodes)
 	}
-	context.SetVariable(variableName, variableValue)
-	return context, nil
+
+	// if there is no LHS - then I guess we just calculate originalExp.Rhs
+	if lhs.MatchingNodes.Len() == 0 {
+		return d.GetMatchingNodes(context, originalExp.RHS)
+	}
+
+	return context.ChildContext(results), nil
+
 }

pkg/yqlib/operator_variables_test.go

@@ -8,7 +8,7 @@ var variableOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{}`,
-		expression: `.a.b as $foo`,
+		expression: `.a.b as $foo | .`,
 		expected: []string{
 			"D0, P[], (doc)::{}\n",
 		},
@@ -16,7 +16,7 @@ var variableOperatorScenarios = []expressionScenario{
 	{
 		document:      "a: [cat]",
 		skipDoc:       true,
-		expression:    "(.[] | {.name: .}) as $item",
+		expression:    "(.[] | {.name: .}) as $item | .",
 		expectedError: `cannot index array with 'name' (strconv.ParseInt: parsing "name": invalid syntax)`,
 	},
 	{
@@ -36,6 +36,22 @@ var variableOperatorScenarios = []expressionScenario{
 			"D0, P[1], (!!str)::dog\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `[1, 2]`,
+		expression: `.[] | . as $f | select($f == 2)`,
+		expected: []string{
+			"D0, P[1], (!!int)::2\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[1, 2]`,
+		expression: `[.[] | . as $f | $f + 1]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- 2\n- 3\n",
+		},
+	},
 	{
 		description:    "Using variables as a lookup",
 		subdescription: "Example taken from [jq](https://stedolan.github.io/jq/manual/#Variable/SymbolicBindingOperator:...as$identifier|...)",

pkg/yqlib/operators_compare_test.go

@@ -21,6 +21,13 @@ var compareOperatorScenarios = []expressionScenario{
 			"D0, P[k], (!!bool)::true\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		expression: `"2022-01-30T15:53:09Z" > "2020-01-30T15:53:09Z"`,
+		expected: []string{
+			"D0, P[], (!!bool)::true\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   "a: 5\nb: 4",

pkg/yqlib/printer.go

@@ -27,6 +27,8 @@ const (
 	TSVOutputFormat
 	XMLOutputFormat
 	Base64OutputFormat
+	UriOutputFormat
+	ShOutputFormat
 )
 
 func OutputFormatFromString(format string) (PrinterOutputFormat, error) {

release_notes.txt

@@ -1,3 +1,16 @@
+4.31.1:
+  - Added shuffle command #1503
+  - Added ability to sort by multiple fields #1541
+  - Added @sh encoder #1526
+  - Added @uri/@urid encoder/decoder #1529
+  - Fixed date comparison with string date #1537
+  - Added from_unix/to_unix Operators
+  - Bumped dependency versions
+
+4.30.8:
+ - Log info message when unable to chown file in linux (e.g. snap confinement) #1521
+
+
 4.30.7:
  - Fixed bug in splice operator #1511
  - Fixed value operator bug  #1515

scripts/devtools.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
 set -ex
 go mod download golang.org/x/tools@latest
-curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.49.0
-wget -O- -nv https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.13.1
+curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.51.1
+wget -O- -nv https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: 'v4.30.7'
+version: 'v4.31.1'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.
@@ -13,7 +13,7 @@ apps:
 parts:
   yq:
     plugin: go
-    go-channel: 1.19/stable
+    go-channel: 1.20/stable
     source: .
     source-type: git
     go-importpath: github.com/mikefarah/yq