Added sort_by operator

Bumps [github.com/jinzhu/copier](https://github.com/jinzhu/copier) from 0.3.2 to 0.3.4.
- [Release notes](https://github.com/jinzhu/copier/releases)
- [Commits](https://github.com/jinzhu/copier/compare/v0.3.2...v0.3.4)

---
updated-dependencies:
- dependency-name: github.com/jinzhu/copier
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/release.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Get the version
         id: get_version
         run: echo ::set-output name=VERSION::${GITHUB_REF##*/}
-        
+
       - name: Generate man page
         uses: docker://pandoc/core:2.14.2
         id: gen-man-page
@@ -35,7 +35,7 @@ jobs:
             --variable=author:"Mike Farah"
             --output=yq.1
             ${{ steps.gen-man-page-md.outputs.man-page-md }}
-        
+
       - name: Cross compile
         run: |
           sudo apt-get install rhash -y
@@ -67,12 +67,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v1
         with:
           platforms: all
-  
+
       - name: Set up Docker Buildx
         id: buildx
         uses: docker/setup-buildx-action@v1
@@ -81,16 +81,29 @@ jobs:
 
       - name: Available platforms
         run: echo ${{ steps.buildx.outputs.platforms }} && docker version
- 
+
       - name: Build and push image
         run: |
           IMAGE_V_VERSION="$(git describe --tags --abbrev=0)"
           IMAGE_VERSION=${IMAGE_V_VERSION:1}
-
-          SHORT_SHA1=$(git rev-parse --short HEAD)
           PLATFORMS="linux/amd64,linux/ppc64le,linux/arm64"
+
           echo "Building and pushing version ${IMAGE_VERSION} of image ${IMAGE_NAME}"
-          echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
-          docker buildx build --platform "${PLATFORMS}" -t "${IMAGE_NAME}:${IMAGE_VERSION}"  -t "${IMAGE_NAME}:latest" -t "${IMAGE_NAME}:4" \
-            --push .
-    
+          echo '${{ secrets.DOCKER_PASSWORD }}' | docker login -u '${{ secrets.DOCKER_USERNAME }}' --password-stdin
+          docker buildx build \
+            --label "org.opencontainers.image.authors=https://github.com/mikefarah/yq/graphs/contributors" \
+            --label "org.opencontainers.image.created=$(date --rfc-3339=seconds)" \
+            --label "org.opencontainers.image.description=yq is a portable command-line YAML processor" \
+            --label "org.opencontainers.image.documentation=https://mikefarah.gitbook.io/yq/" \
+            --label "org.opencontainers.image.licenses=MIT" \
+            --label "org.opencontainers.image.revision=$(git rev-parse HEAD)" \
+            --label "org.opencontainers.image.source=https://github.com/mikefarah/yq" \
+            --label "org.opencontainers.image.title=yq" \
+            --label "org.opencontainers.image.url=https://mikefarah.gitbook.io/yq/" \
+            --label "org.opencontainers.image.version=${IMAGE_VERSION}" \
+            --platform "${PLATFORMS}" \
+            --push \
+            -t "${IMAGE_NAME}:${IMAGE_VERSION}" \
+            -t "${IMAGE_NAME}:4" \
+            -t "${IMAGE_NAME}:latest" \
+            .

.golangci.yml

@@ -0,0 +1,16 @@
+run:
+  timeout: 5m
+linters:
+  enable:
+    - errorlint
+    - gci
+    - gofmt
+    - goimports
+    - gosec
+    - misspell
+issues:
+  exclude-rules:
+  - linters:
+    - gosec
+    text:  "Implicit memory aliasing in for loop."
+    path: _test\.go

1

@@ -1 +0,0 @@
-Error: cannot pass files in when using null-input flag

Dockerfile

@@ -2,7 +2,7 @@ FROM golang:1.17 as builder
 
 WORKDIR /go/src/mikefarah/yq
 
-COPY . /go/src/mikefarah/yq
+COPY . .
 
 RUN CGO_ENABLED=0 go build .
 # RUN ./scripts/test.sh -- this too often times out in the github pipeline.
@@ -11,20 +11,18 @@ RUN ./scripts/acceptance.sh
 # Choose alpine as a base image to make this useful for CI, as many
 # CI tools expect an interactive shell inside the container
 FROM alpine:3 as production
-
-RUN mkdir /home/yq/
-RUN addgroup -g 1000 yq && \
-    adduser -u 1000 -G yq -s /bin/bash -h /home/yq -D yq
-RUN chown -R yq:yq /home/yq/
+LABEL maintainer="Mike Farah <mikefarah@users.noreply.github.com>"
 
 COPY --from=builder /go/src/mikefarah/yq/yq /usr/bin/yq
-RUN chmod +x /usr/bin/yq
-
-ARG VERSION=none
-LABEL version=${VERSION}
-
-USER yq
 
 WORKDIR /workdir
 
+RUN set -eux; \
+  addgroup -g 1000 yq; \
+  adduser -u 1000 -G yq -s /bin/sh -h /home/yq -D yq
+
+RUN chown -R yq:yq /workdir
+
+USER yq
+
 ENTRYPOINT ["/usr/bin/yq"]

Makefile

@@ -2,6 +2,7 @@ MAKEFLAGS += --warn-undefined-variables
 SHELL := /bin/bash
 .SHELLFLAGS := -o pipefail -euc
 .DEFAULT_GOAL := install
+ENGINE := $(shell { command -v podman || command -v docker; } 2>/dev/null)
 
 include Makefile.variables
 
@@ -33,7 +34,7 @@ clean:
 
 ## prefix before other make targets to run in your local dev environment
 local: | quiet
-	@$(eval DOCKRUN= )
+	@$(eval ENGINERUN= )
 	@mkdir -p tmp
 	@touch tmp/dev_image_id
 quiet: # this is silly but shuts up 'Nothing to be done for `local`'
@@ -42,9 +43,9 @@ quiet: # this is silly but shuts up 'Nothing to be done for `local`'
 prepare: tmp/dev_image_id
 tmp/dev_image_id: Dockerfile.dev scripts/devtools.sh
 	@mkdir -p tmp
-	@docker rmi -f ${DEV_IMAGE} > /dev/null 2>&1 || true
-	@docker build -t ${DEV_IMAGE} -f Dockerfile.dev .
-	@docker inspect -f "{{ .ID }}" ${DEV_IMAGE} > tmp/dev_image_id
+	@${ENGINE} rmi -f ${DEV_IMAGE} > /dev/null 2>&1 || true
+	@${ENGINE} build -t ${DEV_IMAGE} -f Dockerfile.dev .
+	@${ENGINE} inspect -f "{{ .ID }}" ${DEV_IMAGE} > tmp/dev_image_id
 
 # ----------------------------------------------
 # build
@@ -54,60 +55,61 @@ build: build/dev
 .PHONY: build/dev
 build/dev: test *.go
 	@mkdir -p bin/
-	${DOCKRUN} go build --ldflags "$(LDFLAGS)"
-	${DOCKRUN} bash ./scripts/acceptance.sh
+	${ENGINERUN} go build --ldflags "$(LDFLAGS)"
+	${ENGINERUN} bash ./scripts/acceptance.sh
 
 ## Compile the project for multiple OS and Architectures.
 xcompile: check
 	@rm -rf build/
 	@mkdir -p build
-	${DOCKRUN} bash ./scripts/xcompile.sh
+	${ENGINERUN} bash ./scripts/xcompile.sh
 	@find build -type d -exec chmod 755 {} \; || :
 	@find build -type f -exec chmod 755 {} \; || :
 
 .PHONY: install
 install: build
-	${DOCKRUN} go install
+	${ENGINERUN} go install
 
 # Each of the fetch should be an entry within vendor.json; not currently included within project
 .PHONY: vendor
 vendor: tmp/dev_image_id
-	${DOCKRUN} go mod vendor
+	@mkdir -p vendor
+	${ENGINERUN} go mod vendor
 
 # ----------------------------------------------
 # develop and test
 
 .PHONY: format
 format: vendor
-	${DOCKRUN} bash ./scripts/format.sh
+	${ENGINERUN} bash ./scripts/format.sh
 
 
 .PHONY: secure
 secure: format
-	${DOCKRUN} bash ./scripts/secure.sh
+	${ENGINERUN} bash ./scripts/secure.sh
 
 .PHONY: check
 check: secure
-	${DOCKRUN} bash ./scripts/check.sh
+	${ENGINERUN} bash ./scripts/check.sh
 
 
 
 .PHONY: test
 test: check
-	${DOCKRUN} bash ./scripts/test.sh
+	${ENGINERUN} bash ./scripts/test.sh
 
 .PHONY: cover
 cover: check
 	@rm -rf cover/
 	@mkdir -p cover
-	${DOCKRUN} bash ./scripts/coverage.sh
+	${ENGINERUN} bash ./scripts/coverage.sh
 	@find cover -type d -exec chmod 755 {} \; || :
 	@find cover -type f -exec chmod 644 {} \; || :
 
 
 .PHONY: release
 release: xcompile
-	${DOCKRUN} bash ./scripts/publish.sh
+	${ENGINERUN} bash ./scripts/publish.sh
 
 # ----------------------------------------------
 # utilities

Makefile.variables

@@ -29,7 +29,7 @@ endif
 
 DEV_IMAGE := ${PROJECT}_dev
 
-DOCKRUN := docker run --rm \
+ENGINERUN := ${ENGINE} run --rm \
 	-e LDFLAGS="${LDFLAGS}" \
 	-e GITHUB_TOKEN="${GITHUB_TOKEN}" \
 	-v ${ROOT}/vendor:/go/src \

README.md

@@ -5,7 +5,7 @@
 
 a lightweight and portable command-line YAML processor. `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml files as well as json. It doesn't yet support everything `jq` does - but it does support the most common operations and functions, and more is being added continuously.
 
-yq is written in go - so you can download a dependency free binary for your platform and you are good to go! If you prefer there are a variety of package managers that can be used as well as docker, all listed below.
+yq is written in go - so you can download a dependency free binary for your platform and you are good to go! If you prefer there are a variety of package managers that can be used as well as Docker and Podman, all listed below.
 
 ## Quick Usage Guide
 
@@ -96,7 +96,7 @@ sudo mv /etc/myfile.tmp /etc/myfile
 rm /etc/myfile.tmp
 ```
 
-### Run with Docker
+### Run with Docker or Podman
 
 #### Oneshot use:
 
@@ -104,6 +104,10 @@ rm /etc/myfile.tmp
 docker run --rm -v "${PWD}":/workdir mikefarah/yq <command> [flags] [expression ]FILE...
 ```
 
+```bash
+podman run --rm -v "${PWD}":/workdir mikefarah/yq <command> [flags] [expression ]FILE...
+```
+
 #### Pipe in via STDIN:
 
 You'll need to pass the `-i\--interactive` flag to docker:
@@ -112,12 +116,20 @@ You'll need to pass the `-i\--interactive` flag to docker:
 cat myfile.yml | docker run -i --rm mikefarah/yq e . -
 ```
 
+```bash
+cat myfile.yml | podman run -i --rm mikefarah/yq e . -
+```
+
 #### Run commands interactively:
 
 ```bash
 docker run --rm -it -v "${PWD}":/workdir --entrypoint sh mikefarah/yq
 ```
 
+```bash
+podman run --rm -it -v "${PWD}":/workdir --entrypoint sh mikefarah/yq
+```
+
 It can be useful to have a bash function to avoid typing the whole docker command:
 
 ```bash
@@ -126,17 +138,27 @@ yq() {
 }
 ```
 
+```bash
+yq() {
+  podman run --rm -i -v "${PWD}":/workdir mikefarah/yq "$@"
+}
+```
+
 
 #### Running as root:
 
-`yq`'s docker image no longer runs under root (https://github.com/mikefarah/yq/pull/860). If you'd like to install more things in the docker image, or you're having permissions issues when attempting to read/write files you'll need to either:
+`yq`'s container image no longer runs under root (https://github.com/mikefarah/yq/pull/860). If you'd like to install more things in the container image, or you're having permissions issues when attempting to read/write files you'll need to either:
 
 
 ```
 docker run --user="root" -it --entrypoint sh mikefarah/yq
 ```
 
-Or, in your docker file:
+```
+podman run --user="root" -it --entrypoint sh mikefarah/yq
+```
+
+Or, in your Dockerfile:
 
 ```
 FROM mikefarah/yq
@@ -220,6 +242,7 @@ Supported by @rmescandon (https://launchpad.net/~rmescandon/+archive/ubuntu/yq)
 - [Update yaml inplace](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate#flags)
 - [Complex expressions to select and update](https://mikefarah.gitbook.io/yq/operators/select#select-and-update-matching-values-in-map)
 - Keeps yaml formatting and comments when updating (though there are issues with whitespace)
+- [Load content from other files](https://mikefarah.gitbook.io/yq/operators/load)
 - [Convert to/from json](https://mikefarah.gitbook.io/yq/v/v4.x/usage/convert)
 - [Convert to properties](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties)
 - [Pipe data in by using '-'](https://mikefarah.gitbook.io/yq/v/v4.x/commands/evaluate)
@@ -265,4 +288,4 @@ Use "yq [command] --help" for more information about a command.
 - `yq` attempts to preserve comment positions and whitespace as much as possible, but it does not handle all scenarios (see https://github.com/go-yaml/yaml/tree/v3 for details)
 - Powershell has its own...[opinions on quoting yq](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks#quotes-in-windows-powershell)
 
-See [tips and tricks](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks) for more common problems and solutions.
+See [tips and tricks](https://mikefarah.gitbook.io/yq/usage/tips-and-tricks) for more common problems and solutions.

cmd/utils.go

@@ -25,7 +25,7 @@ func initCommand(cmd *cobra.Command, args []string) (firstFileIndex int, err err
 		firstFileIndex = 1
 	}
 
-	// backwards compatibilty
+	// backwards compatibility
 	if outputToJSON {
 		outputFormat = "json"
 	}

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "4.14.2"
+	Version = "4.15.1"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

examples/data1.yaml

@@ -1,2 +1,15 @@
-a:
-  include: 'data2.yaml'
+strings:
+  - a: banana
+  - a: cat
+  - a: apple
+
+numbers:
+  - a: 12
+  - a: 13
+  - a: 120
+
+
+obj:
+  - a: {cat: "adog"}
+  - a: {cat: "doga"}
+  - a: apple

github-action/Dockerfile

@@ -1,4 +1,4 @@
-FROM mikefarah/yq:4.14.2
+FROM mikefarah/yq:4
 
 COPY entrypoint.sh /entrypoint.sh
 

go.mod

@@ -4,7 +4,7 @@ require (
 	github.com/elliotchance/orderedmap v1.4.0
 	github.com/fatih/color v1.13.0
 	github.com/goccy/go-yaml v1.9.4
-	github.com/jinzhu/copier v0.3.2
+	github.com/jinzhu/copier v0.3.4
 	github.com/magiconair/properties v1.8.5
 	github.com/spf13/cobra v1.2.1
 	github.com/timtadh/lexmachine v0.2.2

go.sum

@@ -172,8 +172,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
 github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/jinzhu/copier v0.3.2 h1:QdBOCbaouLDYaIPFfi1bKv5F5tPpeTwXe4sD0jqtz5w=
-github.com/jinzhu/copier v0.3.2/go.mod h1:24xnZezI2Yqac9J61UC6/dG/k76ttpq0DdJI3QmUvro=
+github.com/jinzhu/copier v0.3.4 h1:mfU6jI9PtCeUjkjQ322dlff9ELjGDu975C2p/nrubVI=
+github.com/jinzhu/copier v0.3.4/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=

merge-array-by-key.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# ./yq ea '.[]'  examples/data*.yaml
+
+
+./yq ea '
+  ((.[] | {.name: .}) as $item ireduce ({}; . * $item )) as $uniqueMap
+  | ( $uniqueMap  | to_entries | .[]) as $item ireduce([]; . + $item.value)
+' examples/data*.yaml

pkg/yqlib/candidate_node.go

@@ -11,6 +11,7 @@ import (
 type CandidateNode struct {
 	Node   *yaml.Node     // the actual node
 	Parent *CandidateNode // parent node
+	Key    *yaml.Node     // node key, if this is a value from a map (or index in an array)
 
 	LeadingContent string
 
@@ -38,11 +39,41 @@ func (n *CandidateNode) AsList() *list.List {
 	return elMap
 }
 
-func (n *CandidateNode) CreateChild(path interface{}, node *yaml.Node) *CandidateNode {
+func (n *CandidateNode) CreateChildInMap(key *yaml.Node, node *yaml.Node) *CandidateNode {
+	var value interface{} = nil
+	if key != nil {
+		value = key.Value
+	}
 	return &CandidateNode{
 		Node:      node,
-		Path:      n.createChildPath(path),
+		Path:      n.createChildPath(value),
 		Parent:    n,
+		Key:       key,
+		Document:  n.Document,
+		Filename:  n.Filename,
+		FileIndex: n.FileIndex,
+	}
+}
+
+func (n *CandidateNode) CreateChildInArray(index int, node *yaml.Node) *CandidateNode {
+	return &CandidateNode{
+		Node:      node,
+		Path:      n.createChildPath(index),
+		Parent:    n,
+		Key:       &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%v", index), Tag: "!!int"},
+		Document:  n.Document,
+		Filename:  n.Filename,
+		FileIndex: n.FileIndex,
+	}
+}
+
+func (n *CandidateNode) CreateReplacement(node *yaml.Node) *CandidateNode {
+	return &CandidateNode{
+		Node:      node,
+		Path:      n.createChildPath(nil),
+		Parent:    n.Parent,
+		Key:       n.Key,
+		IsMapKey:  n.IsMapKey,
 		Document:  n.Document,
 		Filename:  n.Filename,
 		FileIndex: n.FileIndex,
@@ -73,14 +104,14 @@ func (n *CandidateNode) Copy() (*CandidateNode, error) {
 }
 
 // updates this candidate from the given candidate node
-func (n *CandidateNode) UpdateFrom(other *CandidateNode) {
+func (n *CandidateNode) UpdateFrom(other *CandidateNode, prefs assignPreferences) {
 
-	n.UpdateAttributesFrom(other)
+	n.UpdateAttributesFrom(other, prefs)
 	n.Node.Content = other.Node.Content
 	n.Node.Value = other.Node.Value
 }
 
-func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode) {
+func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignPreferences) {
 	log.Debug("UpdateAttributesFrom: n: %v other: %v", n.GetKey(), other.GetKey())
 	if n.Node.Kind != other.Node.Kind {
 		// clear out the contents when switching to a different type
@@ -91,7 +122,10 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode) {
 	n.Node.Kind = other.Node.Kind
 	n.Node.Tag = other.Node.Tag
 	n.Node.Alias = other.Node.Alias
-	n.Node.Anchor = other.Node.Anchor
+
+	if !prefs.DontOverWriteAnchor {
+		n.Node.Anchor = other.Node.Anchor
+	}
 
 	// merge will pickup the style of the new thing
 	// when autocreating nodes

pkg/yqlib/doc/assign-update.md

@@ -196,6 +196,22 @@ will output
 {a: {b: bogs}}
 ```
 
+## Update node value that has an anchor
+Anchor will remaple
+
+Given a sample.yml file of:
+```yaml
+a: &cool cat
+```
+then
+```bash
+yq eval '.a = "dog"' sample.yml
+```
+will output
+```yaml
+a: &cool dog
+```
+
 ## Update empty object and array
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/comment-operators.md

@@ -139,7 +139,7 @@ will output
 welcome!
 ```
 
-## 
+## Head comment with document split
 Given a sample.yml file of:
 ```yaml
 # welcome!

pkg/yqlib/doc/encode-decode.md

@@ -6,6 +6,111 @@ Note that you can optionally pass an indent value to the encode functions (see b
 
 These operators are useful to process yaml documents that have stringified embeded yaml/json/props in them.
 
+
+| Format | Decode (from string) | Encode (to string) |
+| --- | -- | --|
+| Yaml | from_yaml | to_yaml(i)/@yaml |
+| JSON | from_json | to_json(i)/@json |
+| Properties |  | to_props/@props |
+| CSV |  | to_csv/@csv |
+| TSV |  | to_tsv/@tsv |
+
+
+CSV and TSV format both accept either a single array or scalars (representing a single row), or an array of array of scalars (representing multiple rows). 
+
+
+## Encode value as json string
+Given a sample.yml file of:
+```yaml
+a:
+  cool: thing
+```
+then
+```bash
+yq eval '.b = (.a | to_json)' sample.yml
+```
+will output
+```yaml
+a:
+  cool: thing
+b: |
+  {
+    "cool": "thing"
+  }
+```
+
+## Encode value as json string, on one line
+Pass in a 0 indent to print json on a single line.
+
+Given a sample.yml file of:
+```yaml
+a:
+  cool: thing
+```
+then
+```bash
+yq eval '.b = (.a | to_json(0))' sample.yml
+```
+will output
+```yaml
+a:
+  cool: thing
+b: '{"cool":"thing"}'
+```
+
+## Encode value as json string, on one line shorthand
+Pass in a 0 indent to print json on a single line.
+
+Given a sample.yml file of:
+```yaml
+a:
+  cool: thing
+```
+then
+```bash
+yq eval '.b = (.a | @json)' sample.yml
+```
+will output
+```yaml
+a:
+  cool: thing
+b: '{"cool":"thing"}'
+```
+
+## Decode a json encoded string
+Keep in mind JSON is a subset of YAML. If you want idiomatic yaml, pipe through the style operator to clear out the JSON styling.
+
+Given a sample.yml file of:
+```yaml
+a: '{"cool":"thing"}'
+```
+then
+```bash
+yq eval '.a | from_json | ... style=""' sample.yml
+```
+will output
+```yaml
+cool: thing
+```
+
+## Encode value as props string
+Given a sample.yml file of:
+```yaml
+a:
+  cool: thing
+```
+then
+```bash
+yq eval '.b = (.a | @props)' sample.yml
+```
+will output
+```yaml
+a:
+  cool: thing
+b: |
+  cool = thing
+```
+
 ## Encode value as yaml string
 Indent defaults to 2
 
@@ -52,83 +157,6 @@ b: |
           bob: dylan
 ```
 
-## Encode value as yaml string, using toyaml
-Does the same thing as to_yaml, matching jq naming convention.
-
-Given a sample.yml file of:
-```yaml
-a:
-  cool: thing
-```
-then
-```bash
-yq eval '.b = (.a | to_yaml)' sample.yml
-```
-will output
-```yaml
-a:
-  cool: thing
-b: |
-  cool: thing
-```
-
-## Encode value as json string
-Given a sample.yml file of:
-```yaml
-a:
-  cool: thing
-```
-then
-```bash
-yq eval '.b = (.a | to_json)' sample.yml
-```
-will output
-```yaml
-a:
-  cool: thing
-b: |
-  {
-    "cool": "thing"
-  }
-```
-
-## Encode value as json string, on one line
-Pass in a 0 indent to print json on a single line.
-
-Given a sample.yml file of:
-```yaml
-a:
-  cool: thing
-```
-then
-```bash
-yq eval '.b = (.a | to_json(0))' sample.yml
-```
-will output
-```yaml
-a:
-  cool: thing
-b: '{"cool":"thing"}'
-```
-
-## Encode value as props string
-Given a sample.yml file of:
-```yaml
-a:
-  cool: thing
-```
-then
-```bash
-yq eval '.b = (.a | to_props)' sample.yml
-```
-will output
-```yaml
-a:
-  cool: thing
-b: |
-  cool = thing
-```
-
 ## Decode a yaml encoded string
 Given a sample.yml file of:
 ```yaml
@@ -178,3 +206,68 @@ will output
 a: 'foo: cat'
 ```
 
+## Encode array of scalars as csv string
+Scalars are strings, numbers and booleans.
+
+Given a sample.yml file of:
+```yaml
+- cat
+- thing1,thing2
+- true
+- 3.40
+```
+then
+```bash
+yq eval '@csv' sample.yml
+```
+will output
+```yaml
+cat,"thing1,thing2",true,3.40
+```
+
+## Encode array of arrays as csv string
+Given a sample.yml file of:
+```yaml
+- - cat
+  - thing1,thing2
+  - true
+  - 3.40
+- - dog
+  - thing3
+  - false
+  - 12
+```
+then
+```bash
+yq eval '@csv' sample.yml
+```
+will output
+```yaml
+cat,"thing1,thing2",true,3.40
+dog,thing3,false,12
+```
+
+## Encode array of array scalars as tsv string
+Scalars are strings, numbers and booleans.
+
+Given a sample.yml file of:
+```yaml
+- - cat
+  - thing1,thing2
+  - true
+  - 3.40
+- - dog
+  - thing3
+  - false
+  - 12
+```
+then
+```bash
+yq eval '@tsv' sample.yml
+```
+will output
+```yaml
+cat	thing1,thing2	true	3.40
+dog	thing3	false	12
+```
+

pkg/yqlib/doc/equals.md

@@ -108,7 +108,7 @@ will output
 a: frog
 ```
 
-## Two non existant keys are equal
+## Two non existent keys are equal
 Given a sample.yml file of:
 ```yaml
 a: frog

pkg/yqlib/doc/headers/encode-decode.md

@@ -5,3 +5,16 @@ Encode operators will take the piped in object structure and encode it as a stri
 Note that you can optionally pass an indent value to the encode functions (see below).
 
 These operators are useful to process yaml documents that have stringified embeded yaml/json/props in them.
+
+
+| Format | Decode (from string) | Encode (to string) |
+| --- | -- | --|
+| Yaml | from_yaml | to_yaml(i)/@yaml |
+| JSON | from_json | to_json(i)/@json |
+| Properties |  | to_props/@props |
+| CSV |  | to_csv/@csv |
+| TSV |  | to_tsv/@tsv |
+
+
+CSV and TSV format both accept either a single array or scalars (representing a single row), or an array of array of scalars (representing multiple rows). 
+

pkg/yqlib/doc/headers/load.md

@@ -4,6 +4,7 @@ The `load`/`strload` operator allows you to load in content from another file re
 
 Note that you can use string operators like `+` and `sub` to modify the value in the yaml file to a path that exists in your system.
 
+Use `strload` to load text based content as a string block, and `load` to interpret the file as yaml.
 
 Lets say there is a file `../../examples/thing.yml`:
 

pkg/yqlib/doc/headers/map.md

@@ -0,0 +1,3 @@
+# Map
+
+Maps values of an array. Use `map_values` to map values of an object.

pkg/yqlib/doc/headers/parent.md

@@ -0,0 +1,3 @@
+# Parent
+
+Parent simply returns the parent nodes of the matching nodes.

pkg/yqlib/doc/headers/sort.md

@@ -0,0 +1,5 @@
+# Sort
+
+Sorts an array. Use `sort` to sort an array as is, or `sort_by` to sort by a particular subfield.
+
+Note that at this stage, `yq` only sorts scalar fields.

pkg/yqlib/doc/keys.md

@@ -34,3 +34,81 @@ will output
 - 1
 ```
 
+## Retrieve array key
+Given a sample.yml file of:
+```yaml
+- 1
+- 2
+- 3
+```
+then
+```bash
+yq eval '.[1] | key' sample.yml
+```
+will output
+```yaml
+1
+```
+
+## Retrieve map key
+Given a sample.yml file of:
+```yaml
+a: thing
+```
+then
+```bash
+yq eval '.a | key' sample.yml
+```
+will output
+```yaml
+a
+```
+
+## No key
+Given a sample.yml file of:
+```yaml
+{}
+```
+then
+```bash
+yq eval 'key' sample.yml
+```
+will output
+```yaml
+```
+
+## Update map key
+Given a sample.yml file of:
+```yaml
+a:
+  x: 3
+  y: 4
+```
+then
+```bash
+yq eval '(.a.x | key) = "meow"' sample.yml
+```
+will output
+```yaml
+a:
+  meow: 3
+  y: 4
+```
+
+## Get comment from map key
+Given a sample.yml file of:
+```yaml
+a:
+  # comment on key
+  x: 3
+  y: 4
+```
+then
+```bash
+yq eval '.a.x | key | headComment' sample.yml
+```
+will output
+```yaml
+comment on key
+```
+

pkg/yqlib/doc/load.md

@@ -4,6 +4,7 @@ The `load`/`strload` operator allows you to load in content from another file re
 
 Note that you can use string operators like `+` and `sub` to modify the value in the yaml file to a path that exists in your system.
 
+Use `strload` to load text based content as a string block, and `load` to interpret the file as yaml.
 
 Lets say there is a file `../../examples/thing.yml`:
 

pkg/yqlib/doc/map.md

@@ -0,0 +1,40 @@
+# Map
+
+Maps values of an array. Use `map_values` to map values of an object.
+
+## Map array
+Given a sample.yml file of:
+```yaml
+- 1
+- 2
+- 3
+```
+then
+```bash
+yq eval 'map(. + 1)' sample.yml
+```
+will output
+```yaml
+- 2
+- 3
+- 4
+```
+
+## Map object values
+Given a sample.yml file of:
+```yaml
+a: 1
+b: 2
+c: 3
+```
+then
+```bash
+yq eval 'map_values(. + 1)' sample.yml
+```
+will output
+```yaml
+a: 2
+b: 3
+c: 4
+```
+

pkg/yqlib/doc/multiply-merge.md

@@ -234,43 +234,60 @@ will output
 ```
 
 ## Merge arrays of objects together, matching on a key
-It's a complex command, the trickyness comes from needing to have the right context in the expressions.
-First we save the second array into a variable '$two' which lets us reference it later.
-We then need to update the first array. We will use the relative update (|=) because we need to update relative to the current element of the array in the LHS in the RHS expression. 
-We set the current element of the first array as $cur. Now we multiply (merge) $cur with the matching entry in $two, by passing $two through a select filter.
+There are several parts of the complex expression. 
+The first part is doing the hard work, it creates a map from the arrays keyed by '.a', so that there are no duplicates. 
+Then there's another reduce that converts that map back to an array.
+Finally, we set the result of the merged array back into the first doc.
+
+To use this, you will need to update '.myArray' to be the expression to your array (e.g. .my.array), and '.a' to be the key field of your array (e.g. '.name')
+
+Thanks Kev from [stackoverflow](https://stackoverflow.com/a/70109529/1168223)
 
 
 Given a sample.yml file of:
 ```yaml
-- a: apple
-  b: appleB
-- a: kiwi
-  b: kiwiB
-- a: banana
-  b: bananaB
+myArray:
+  - a: apple
+    b: appleB
+  - a: kiwi
+    b: kiwiB
+  - a: banana
+    b: bananaB
+something: else
 ```
 And another sample another.yml file of:
 ```yaml
-- a: banana
-  c: bananaC
-- a: apple
-  b: appleB2
-- a: dingo
-  c: dingoC
+myArray:
+  - a: banana
+    c: bananaC
+  - a: apple
+    b: appleB2
+  - a: dingo
+    c: dingoC
 ```
 then
 ```bash
-yq eval-all '(select(fi==1) | .[]) as $two | select(fi==0) | .[] |= (. as $cur |  $cur * ($two | select(.a == $cur.a)))' sample.yml another.yml
+yq eval-all '
+(
+  ((.myArray[] | {.a: .}) as $item ireduce ({}; . * $item )) as $uniqueMap
+  | ( $uniqueMap  | to_entries | .[]) as $item ireduce([]; . + $item.value)
+) as $mergedArray
+| select(fi == 0) | .myArray = $mergedArray
+' sample.yml another.yml
 ```
 will output
 ```yaml
-- a: apple
-  b: appleB2
-- a: kiwi
-  b: kiwiB
-- a: banana
-  b: bananaB
-  c: bananaC
+myArray:
+  - a: apple
+    b: appleB2
+  - a: kiwi
+    b: kiwiB
+  - a: banana
+    b: bananaB
+    c: bananaC
+  - a: dingo
+    c: dingoC
+something: else
 ```
 
 ## Merge to prefix an element

pkg/yqlib/doc/parent.md

@@ -0,0 +1,52 @@
+# Parent
+
+Parent simply returns the parent nodes of the matching nodes.
+
+## Simple example
+Given a sample.yml file of:
+```yaml
+a:
+  nested: cat
+```
+then
+```bash
+yq eval '.a.nested | parent' sample.yml
+```
+will output
+```yaml
+nested: cat
+```
+
+## Parent of nested matches
+Given a sample.yml file of:
+```yaml
+a:
+  fruit: apple
+  name: bob
+b:
+  fruit: banana
+  name: sam
+```
+then
+```bash
+yq eval '.. | select(. == "banana") | parent' sample.yml
+```
+will output
+```yaml
+fruit: banana
+name: sam
+```
+
+## No parent
+Given a sample.yml file of:
+```yaml
+{}
+```
+then
+```bash
+yq eval 'parent' sample.yml
+```
+will output
+```yaml
+```
+

pkg/yqlib/doc/path.md

@@ -78,7 +78,7 @@ a:
 ```
 then
 ```bash
-yq eval '.a.[] | select(. == "*og") | [{"path":path, "value":.}]' sample.yml
+yq eval '.a[] | select(. == "*og") | [{"path":path, "value":.}]' sample.yml
 ```
 will output
 ```yaml

pkg/yqlib/doc/sort.md

@@ -0,0 +1,98 @@
+# Sort
+
+Sorts an array. Use `sort` to sort an array as is, or `sort_by` to sort by a particular subfield.
+
+Note that at this stage, `yq` only sorts scalar fields.
+
+## Sort by string field
+Given a sample.yml file of:
+```yaml
+- a: banana
+- a: cat
+- a: apple
+```
+then
+```bash
+yq eval 'sort_by(.a)' sample.yml
+```
+will output
+```yaml
+- a: apple
+- a: banana
+- a: cat
+```
+
+## Sort is stable
+Note the order of the elements in unchanged when equal in sorting.
+
+Given a sample.yml file of:
+```yaml
+- a: banana
+  b: 1
+- a: banana
+  b: 2
+- a: banana
+  b: 3
+- a: banana
+  b: 4
+```
+then
+```bash
+yq eval 'sort_by(.a)' sample.yml
+```
+will output
+```yaml
+- a: banana
+  b: 1
+- a: banana
+  b: 2
+- a: banana
+  b: 3
+- a: banana
+  b: 4
+```
+
+## Sort by numeric field
+Given a sample.yml file of:
+```yaml
+- a: 10
+- a: 100
+- a: 1
+```
+then
+```bash
+yq eval 'sort_by(.a)' sample.yml
+```
+will output
+```yaml
+- a: 1
+- a: 10
+- a: 100
+```
+
+## Sort, nulls come first
+Given a sample.yml file of:
+```yaml
+- 8
+- 3
+- null
+- 6
+- true
+- false
+- cat
+```
+then
+```bash
+yq eval 'sort' sample.yml
+```
+will output
+```yaml
+- null
+- false
+- true
+- 3
+- 6
+- 8
+- cat
+```
+

pkg/yqlib/doc/string-operators.md

@@ -209,7 +209,7 @@ will output
 ```
 
 ## Test using regex
-Like jq'q equivalant, this works like match but only returns true/false instead of full match details
+Like jq'q equivalent, this works like match but only returns true/false instead of full match details
 
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/encoder.go

@@ -3,6 +3,7 @@ package yqlib
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 
@@ -126,7 +127,7 @@ func (o *orderedMap) UnmarshalJSON(data []byte) error {
 
 		// cycle through k/v
 		var tok json.Token
-		for tok, err = dec.Token(); err != io.EOF; tok, err = dec.Token() {
+		for tok, err = dec.Token(); !errors.Is(err, io.EOF); tok, err = dec.Token() {
 			// we can expect two types: string or Delim. Delim automatically means
 			// that it is the closing bracket of the object, whereas string means
 			// that there is another key.
@@ -142,7 +143,7 @@ func (o *orderedMap) UnmarshalJSON(data []byte) error {
 			o.kv = append(o.kv, kv)
 		}
 		// unexpected error
-		if err != nil && err != io.EOF {
+		if err != nil && !errors.Is(err, io.EOF) {
 			return err
 		}
 		return nil

pkg/yqlib/encoder_csv.go

@@ -0,0 +1,57 @@
+package yqlib
+
+import (
+	"encoding/csv"
+	"fmt"
+	"io"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+type csvEncoder struct {
+	destination csv.Writer
+}
+
+func NewCsvEncoder(destination io.Writer, separator rune) Encoder {
+	csvWriter := *csv.NewWriter(destination)
+	csvWriter.Comma = separator
+	return &csvEncoder{csvWriter}
+}
+
+func (e *csvEncoder) encodeRow(contents []*yaml.Node) error {
+	stringValues := make([]string, len(contents))
+
+	for i, child := range contents {
+
+		if child.Kind != yaml.ScalarNode {
+			return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag)
+		}
+		stringValues[i] = child.Value
+	}
+	return e.destination.Write(stringValues)
+}
+
+func (e *csvEncoder) Encode(originalNode *yaml.Node) error {
+	// node must be a sequence
+	node := unwrapDoc(originalNode)
+	if node.Kind != yaml.SequenceNode {
+		return fmt.Errorf("csv encoding only works for arrays, got: %v", node.Tag)
+	} else if len(node.Content) == 0 {
+		return nil
+	}
+	if node.Content[0].Kind == yaml.ScalarNode {
+		return e.encodeRow(node.Content)
+	}
+
+	for i, child := range node.Content {
+
+		if child.Kind != yaml.SequenceNode {
+			return fmt.Errorf("csv encoding only works for arrays of scalars (string/numbers/booleans), child[%v] is a %v", i, child.Tag)
+		}
+		err := e.encodeRow(child.Content)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/yqlib/encoder_csv_test.go

@@ -0,0 +1,60 @@
+package yqlib
+
+import (
+	"bufio"
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+func yamlToCsv(sampleYaml string, separator rune) string {
+	var output bytes.Buffer
+	writer := bufio.NewWriter(&output)
+
+	var jsonEncoder = NewCsvEncoder(writer, separator)
+	inputs, err := readDocuments(strings.NewReader(sampleYaml), "sample.yml", 0)
+	if err != nil {
+		panic(err)
+	}
+	node := inputs.Front().Value.(*CandidateNode).Node
+	err = jsonEncoder.Encode(node)
+	if err != nil {
+		panic(err)
+	}
+	writer.Flush()
+
+	return strings.TrimSuffix(output.String(), "\n")
+}
+
+var sampleYaml = `["apple", apple2, "comma, in, value", "new
+line", 3, 3.40, true, "tab	here"]`
+
+var sampleYamlArray = "[" + sampleYaml + ", [bob, cat, meow, puss]]"
+
+func TestCsvEncoderEmptyArray(t *testing.T) {
+	var actualCsv = yamlToCsv(`[]`, ',')
+	test.AssertResult(t, "", actualCsv)
+}
+
+func TestCsvEncoder(t *testing.T) {
+	var expectedCsv = `apple,apple2,"comma, in, value",new line,3,3.40,true,tab	here`
+
+	var actualCsv = yamlToCsv(sampleYaml, ',')
+	test.AssertResult(t, expectedCsv, actualCsv)
+}
+
+func TestCsvEncoderArrayOfArrays(t *testing.T) {
+	var actualCsv = yamlToCsv(sampleYamlArray, ',')
+	var expectedCsv = "apple,apple2,\"comma, in, value\",new line,3,3.40,true,tab	here\nbob,cat,meow,puss"
+	test.AssertResult(t, expectedCsv, actualCsv)
+}
+
+func TestTsvEncoder(t *testing.T) {
+
+	var expectedCsv = `apple	apple2	comma, in, value	new line	3	3.40	true	"tab	here"`
+
+	var actualCsv = yamlToCsv(sampleYaml, '\t')
+	test.AssertResult(t, expectedCsv, actualCsv)
+}

pkg/yqlib/expression_postfix.go

@@ -68,25 +68,29 @@ func (p *expressionPostFixerImpl) ConvertToPostfix(infixTokens []*token) ([]*Ope
 			}
 			// now we should have [ as the last element on the opStack, get rid of it
 			opStack = opStack[0 : len(opStack)-1]
-			log.Debugf("deleteing open bracket from opstack")
+			log.Debugf("deleting open bracket from opstack")
 
 			//and append a collect to the result
+
 			// hack - see if there's the optional traverse flag
-			// on the close op - move it to the collect op.
+			// on the close op - move it to the traverse array op
 			// allows for .["cat"]?
 			prefs := traversePreferences{}
 			closeTokenMatch := string(currentToken.Match.Bytes)
 			if closeTokenMatch[len(closeTokenMatch)-1:] == "?" {
 				prefs.OptionalTraverse = true
 			}
-			result = append(result, &Operation{OperationType: collectOperator, Preferences: prefs})
+			result = append(result, &Operation{OperationType: collectOperator})
 			log.Debugf("put collect onto the result")
-			result = append(result, &Operation{OperationType: shortPipeOpType})
-			log.Debugf("put shortpipe onto the result")
+			if opener != openCollect {
+				result = append(result, &Operation{OperationType: shortPipeOpType})
+				log.Debugf("put shortpipe onto the result")
+			}
 
 			//traverseArrayCollect is a sneaky op that needs to be included too
-			//when closing a []
+			//when closing a ]
 			if len(opStack) > 0 && opStack[len(opStack)-1].Operation != nil && opStack[len(opStack)-1].Operation.OperationType == traverseArrayOpType {
+				opStack[len(opStack)-1].Operation.Preferences = prefs
 				opStack, result = popOpToResult(opStack, result)
 			}
 

pkg/yqlib/expression_processing_test.go

@@ -15,15 +15,25 @@ var pathTests = []struct {
 	expectedTokens  []interface{}
 	expectedPostFix []interface{}
 }{
+	{
+		`[.a, .b]`,
+		append(make([]interface{}, 0), "[", "a", "UNION", "b", "]"),
+		append(make([]interface{}, 0), "a", "b", "UNION", "COLLECT"),
+	},
+	{
+		`.[env(myenv)]`,
+		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "ENV", "]"),
+		append(make([]interface{}, 0), "SELF", "ENV", "COLLECT", "TRAVERSE_ARRAY"),
+	},
 	{
 		`.["cat"].["dog"]`,
 		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "cat (string)", "]", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "dog (string)", "]"),
-		append(make([]interface{}, 0), "SELF", "cat (string)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SELF", "dog (string)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
+		append(make([]interface{}, 0), "SELF", "cat (string)", "COLLECT", "TRAVERSE_ARRAY", "SELF", "dog (string)", "COLLECT", "TRAVERSE_ARRAY", "SHORT_PIPE"),
 	},
 	{
 		`.["cat"]`,
 		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "cat (string)", "]"),
-		append(make([]interface{}, 0), "SELF", "cat (string)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+		append(make([]interface{}, 0), "SELF", "cat (string)", "COLLECT", "TRAVERSE_ARRAY"),
 	},
 	{
 		"with(.a;.=3)",
@@ -53,12 +63,12 @@ var pathTests = []struct {
 	{
 		`.[0]`,
 		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
-		append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+		append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "TRAVERSE_ARRAY"),
 	},
 	{
 		`.[0][1]`,
 		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]", "TRAVERSE_ARRAY", "[", "1 (int64)", "]"),
-		append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "1 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+		append(make([]interface{}, 0), "SELF", "0 (int64)", "COLLECT", "TRAVERSE_ARRAY", "1 (int64)", "COLLECT", "TRAVERSE_ARRAY"),
 	},
 	{
 		`"\""`,
@@ -68,7 +78,7 @@ var pathTests = []struct {
 	{
 		`[]|join(".")`,
 		append(make([]interface{}, 0), "[", "EMPTY", "]", "PIPE", "JOIN", "(", ". (string)", ")"),
-		append(make([]interface{}, 0), "EMPTY", "COLLECT", "SHORT_PIPE", ". (string)", "JOIN", "PIPE"),
+		append(make([]interface{}, 0), "EMPTY", "COLLECT", ". (string)", "JOIN", "PIPE"),
 	},
 	{
 		`{"cool": .b or .c}`,
@@ -78,7 +88,7 @@ var pathTests = []struct {
 	{
 		`{"cool": []|join(".")}`,
 		append(make([]interface{}, 0), "{", "cool (string)", "CREATE_MAP", "[", "EMPTY", "]", "PIPE", "JOIN", "(", ". (string)", ")", "}"),
-		append(make([]interface{}, 0), "cool (string)", "EMPTY", "COLLECT", "SHORT_PIPE", ". (string)", "JOIN", "PIPE", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"),
+		append(make([]interface{}, 0), "cool (string)", "EMPTY", "COLLECT", ". (string)", "JOIN", "PIPE", "CREATE_MAP", "COLLECT_OBJECT", "SHORT_PIPE"),
 	},
 	{
 		`.a as $item ireduce (0; . + $item)`, // note - add code to shuffle reduce to this position for postfix
@@ -93,7 +103,7 @@ var pathTests = []struct {
 	{
 		`[]`,
 		append(make([]interface{}, 0), "[", "EMPTY", "]"),
-		append(make([]interface{}, 0), "EMPTY", "COLLECT", "SHORT_PIPE"),
+		append(make([]interface{}, 0), "EMPTY", "COLLECT"),
 	},
 	{
 		`{}`,
@@ -103,67 +113,67 @@ var pathTests = []struct {
 	{
 		`[{}]`,
 		append(make([]interface{}, 0), "[", "{", "EMPTY", "}", "]"),
-		append(make([]interface{}, 0), "EMPTY", "COLLECT_OBJECT", "SHORT_PIPE", "COLLECT", "SHORT_PIPE"),
+		append(make([]interface{}, 0), "EMPTY", "COLLECT_OBJECT", "SHORT_PIPE", "COLLECT"),
 	},
 	{
 		`.realnames as $names | $names["anon"]`,
 		append(make([]interface{}, 0), "realnames", "ASSIGN_VARIABLE", "GET_VARIABLE", "PIPE", "GET_VARIABLE", "TRAVERSE_ARRAY", "[", "anon (string)", "]"),
-		append(make([]interface{}, 0), "realnames", "GET_VARIABLE", "ASSIGN_VARIABLE", "GET_VARIABLE", "anon (string)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "PIPE"),
+		append(make([]interface{}, 0), "realnames", "GET_VARIABLE", "ASSIGN_VARIABLE", "GET_VARIABLE", "anon (string)", "COLLECT", "TRAVERSE_ARRAY", "PIPE"),
 	},
 	{
 		`.b[.a]`,
 		append(make([]interface{}, 0), "b", "TRAVERSE_ARRAY", "[", "a", "]"),
-		append(make([]interface{}, 0), "b", "a", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+		append(make([]interface{}, 0), "b", "a", "COLLECT", "TRAVERSE_ARRAY"),
 	},
 	{
 		`.b[.a]?`,
 		append(make([]interface{}, 0), "b", "TRAVERSE_ARRAY", "[", "a", "]"),
-		append(make([]interface{}, 0), "b", "a", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+		append(make([]interface{}, 0), "b", "a", "COLLECT", "TRAVERSE_ARRAY"),
 	},
 	{
 		`.[]`,
 		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
-		append(make([]interface{}, 0), "SELF", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+		append(make([]interface{}, 0), "SELF", "EMPTY", "COLLECT", "TRAVERSE_ARRAY"),
 	},
 	{
 		`.a[]`,
 		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
-		append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+		append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "TRAVERSE_ARRAY"),
 	},
 	{
 		`.a[]?`,
 		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
-		append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+		append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "TRAVERSE_ARRAY"),
 	},
 	{
 		`.a.[]`,
 		append(make([]interface{}, 0), "a", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]"),
-		append(make([]interface{}, 0), "a", "SELF", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
+		append(make([]interface{}, 0), "a", "SELF", "EMPTY", "COLLECT", "TRAVERSE_ARRAY", "SHORT_PIPE"),
 	},
 	{
 		`.a[0]`,
 		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
-		append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+		append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "TRAVERSE_ARRAY"),
 	},
 	{
 		`.a[0]?`,
 		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
-		append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY"),
+		append(make([]interface{}, 0), "a", "0 (int64)", "COLLECT", "TRAVERSE_ARRAY"),
 	},
 	{
 		`.a.[0]`,
 		append(make([]interface{}, 0), "a", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "0 (int64)", "]"),
-		append(make([]interface{}, 0), "a", "SELF", "0 (int64)", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE"),
+		append(make([]interface{}, 0), "a", "SELF", "0 (int64)", "COLLECT", "TRAVERSE_ARRAY", "SHORT_PIPE"),
 	},
 	{
 		`.a[].c`,
 		append(make([]interface{}, 0), "a", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "SHORT_PIPE", "c"),
-		append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "c", "SHORT_PIPE"),
+		append(make([]interface{}, 0), "a", "EMPTY", "COLLECT", "TRAVERSE_ARRAY", "c", "SHORT_PIPE"),
 	},
 	{
 		`[3]`,
 		append(make([]interface{}, 0), "[", "3 (int64)", "]"),
-		append(make([]interface{}, 0), "3 (int64)", "COLLECT", "SHORT_PIPE"),
+		append(make([]interface{}, 0), "3 (int64)", "COLLECT"),
 	},
 	{
 		`.key.array + .key.array2`,
@@ -183,27 +193,27 @@ var pathTests = []struct {
 	{
 		`.a | .[].b == "apple"`,
 		append(make([]interface{}, 0), "a", "PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "SHORT_PIPE", "b", "EQUALS", "apple (string)"),
-		append(make([]interface{}, 0), "a", "SELF", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "b", "SHORT_PIPE", "apple (string)", "EQUALS", "PIPE"),
+		append(make([]interface{}, 0), "a", "SELF", "EMPTY", "COLLECT", "TRAVERSE_ARRAY", "b", "SHORT_PIPE", "apple (string)", "EQUALS", "PIPE"),
 	},
 	{
 		`(.a | .[].b) == "apple"`,
 		append(make([]interface{}, 0), "(", "a", "PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "SHORT_PIPE", "b", ")", "EQUALS", "apple (string)"),
-		append(make([]interface{}, 0), "a", "SELF", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "b", "SHORT_PIPE", "PIPE", "apple (string)", "EQUALS"),
+		append(make([]interface{}, 0), "a", "SELF", "EMPTY", "COLLECT", "TRAVERSE_ARRAY", "b", "SHORT_PIPE", "PIPE", "apple (string)", "EQUALS"),
 	},
 	{
 		`.[] | select(. == "*at")`,
 		append(make([]interface{}, 0), "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "PIPE", "SELECT", "(", "SELF", "EQUALS", "*at (string)", ")"),
-		append(make([]interface{}, 0), "SELF", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SELF", "*at (string)", "EQUALS", "SELECT", "PIPE"),
+		append(make([]interface{}, 0), "SELF", "EMPTY", "COLLECT", "TRAVERSE_ARRAY", "SELF", "*at (string)", "EQUALS", "SELECT", "PIPE"),
 	},
 	{
 		`[true]`,
 		append(make([]interface{}, 0), "[", "true (bool)", "]"),
-		append(make([]interface{}, 0), "true (bool)", "COLLECT", "SHORT_PIPE"),
+		append(make([]interface{}, 0), "true (bool)", "COLLECT"),
 	},
 	{
 		`[true, false]`,
 		append(make([]interface{}, 0), "[", "true (bool)", "UNION", "false (bool)", "]"),
-		append(make([]interface{}, 0), "true (bool)", "false (bool)", "UNION", "COLLECT", "SHORT_PIPE"),
+		append(make([]interface{}, 0), "true (bool)", "false (bool)", "UNION", "COLLECT"),
 	},
 	{
 		`"mike": .a`,
@@ -228,7 +238,7 @@ var pathTests = []struct {
 	{
 		`{.a: .c, .b.[]: .f.g[]}`,
 		append(make([]interface{}, 0), "{", "a", "CREATE_MAP", "c", "UNION", "b", "SHORT_PIPE", "SELF", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "CREATE_MAP", "f", "SHORT_PIPE", "g", "TRAVERSE_ARRAY", "[", "EMPTY", "]", "}"),
-		append(make([]interface{}, 0), "a", "c", "CREATE_MAP", "b", "SELF", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE", "f", "g", "EMPTY", "COLLECT", "SHORT_PIPE", "TRAVERSE_ARRAY", "SHORT_PIPE", "CREATE_MAP", "UNION", "COLLECT_OBJECT", "SHORT_PIPE"),
+		append(make([]interface{}, 0), "a", "c", "CREATE_MAP", "b", "SELF", "EMPTY", "COLLECT", "TRAVERSE_ARRAY", "SHORT_PIPE", "f", "g", "EMPTY", "COLLECT", "TRAVERSE_ARRAY", "SHORT_PIPE", "CREATE_MAP", "UNION", "COLLECT_OBJECT", "SHORT_PIPE"),
 	},
 	{
 		`explode(.a.b)`,

pkg/yqlib/expression_tokeniser.go

@@ -94,7 +94,8 @@ func assignOpToken(updateAssign bool) lex.Action {
 	return func(s *lex.Scanner, m *machines.Match) (interface{}, error) {
 		log.Debug("assignOpToken %v", string(m.Bytes))
 		value := string(m.Bytes)
-		op := &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, UpdateAssign: updateAssign}
+		prefs := assignPreferences{DontOverWriteAnchor: true}
+		op := &Operation{OperationType: assignOpType, Value: assignOpType.Type, StringValue: value, UpdateAssign: updateAssign, Preferences: prefs}
 		return &token{TokenType: operationToken, Operation: op}, nil
 	}
 }
@@ -312,6 +313,9 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`:\s*`), opToken(createMapOpType))
 	lexer.Add([]byte(`length`), opToken(lengthOpType))
 
+	lexer.Add([]byte(`map`), opToken(mapOpType))
+	lexer.Add([]byte(`map_values`), opToken(mapValuesOpType))
+
 	lexer.Add([]byte(`flatten\([0-9]+\)`), flattenWithDepth())
 	lexer.Add([]byte(`flatten`), opTokenWithPrefs(flattenOpType, nil, flattenPreferences{depth: -1}))
 
@@ -322,11 +326,23 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`to_json\([0-9]+\)`), encodeWithIndent(JsonOutputFormat))
 
 	lexer.Add([]byte(`toyaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
+	// 0 indent doesn't work with yaml.
+	lexer.Add([]byte(`@yaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
+
 	lexer.Add([]byte(`tojson`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JsonOutputFormat, indent: 2}))
 	lexer.Add([]byte(`toprops`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2}))
+	lexer.Add([]byte(`@props`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2}))
 
 	lexer.Add([]byte(`to_yaml`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: YamlOutputFormat, indent: 2}))
 	lexer.Add([]byte(`to_json`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JsonOutputFormat, indent: 2}))
+	lexer.Add([]byte(`@json`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: JsonOutputFormat, indent: 0}))
+
+	lexer.Add([]byte(`to_csv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: CsvOutputFormat}))
+	lexer.Add([]byte(`@csv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: CsvOutputFormat}))
+
+	lexer.Add([]byte(`to_tsv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: TsvOutputFormat}))
+	lexer.Add([]byte(`@tsv`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: TsvOutputFormat}))
+
 	lexer.Add([]byte(`to_props`), opTokenWithPrefs(encodeOpType, nil, encoderPreferences{format: PropsOutputFormat, indent: 2}))
 
 	lexer.Add([]byte(`fromyaml`), opToken(decodeOpType))
@@ -336,6 +352,8 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`from_json`), opToken(decodeOpType))
 
 	lexer.Add([]byte(`sortKeys`), opToken(sortKeysOpType))
+	lexer.Add([]byte(`sort_keys`), opToken(sortKeysOpType))
+
 	lexer.Add([]byte(`load`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: false}))
 	lexer.Add([]byte(`strload`), opTokenWithPrefs(loadOpType, nil, loadPrefs{loadAsString: true}))
 	lexer.Add([]byte(`select`), opToken(selectOpType))
@@ -361,6 +379,9 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`capture`), opToken(captureOpType))
 	lexer.Add([]byte(`test`), opToken(testOpType))
 
+	lexer.Add([]byte(`sort`), opToken(sortOpType))
+	lexer.Add([]byte(`sort_by`), opToken(sortByOpType))
+
 	lexer.Add([]byte(`any`), opToken(anyOpType))
 	lexer.Add([]byte(`any_c`), opToken(anyConditionOpType))
 	lexer.Add([]byte(`all`), opToken(allOpType))
@@ -368,6 +389,9 @@ func initLexer() (*lex.Lexer, error) {
 	lexer.Add([]byte(`contains`), opToken(containsOpType))
 
 	lexer.Add([]byte(`split`), opToken(splitStringOpType))
+
+	lexer.Add([]byte(`parent`), opToken(getParentOpType))
+	lexer.Add([]byte(`key`), opToken(getKeyOpType))
 	lexer.Add([]byte(`keys`), opToken(keysOpType))
 
 	lexer.Add([]byte(`style`), opAssignableToken(getStyleOpType, assignStyleOpType))
@@ -467,7 +491,7 @@ func (p *expressionTokeniserImpl) Tokenise(expression string) ([]*token, error)
 	scanner, err := p.lexer.Scanner([]byte(expression))
 
 	if err != nil {
-		return nil, fmt.Errorf("Parsing expression: %v", err)
+		return nil, fmt.Errorf("Parsing expression: %w", err)
 	}
 	var tokens []*token
 	for tok, err, eof := scanner.Next(); !eof; tok, err, eof = scanner.Next() {
@@ -478,7 +502,7 @@ func (p *expressionTokeniserImpl) Tokenise(expression string) ([]*token, error)
 			tokens = append(tokens, currentToken)
 		}
 		if err != nil {
-			return nil, fmt.Errorf("Parsing expression: %v", err)
+			return nil, fmt.Errorf("Parsing expression: %w", err)
 		}
 	}
 	var postProcessedTokens = make([]*token, 0)

pkg/yqlib/file_utils.go

@@ -34,7 +34,7 @@ func tryRemoveFile(filename string) {
 
 // thanks https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang
 func copyFileContents(src, dst string) (err error) {
-	// ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory,
+	// ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory,
 	// and ensuring that it's not possible to give a path to a file outside thar directory.
 
 	in, err := os.Open(src) // #nosec

pkg/yqlib/front_matter.go

@@ -2,6 +2,7 @@ package yqlib
 
 import (
 	"bufio"
+	"errors"
 	"io"
 	"os"
 )
@@ -63,7 +64,7 @@ func (f *frontMatterHandlerImpl) Split() error {
 
 	for {
 		peekBytes, err := reader.Peek(3)
-		if err == io.EOF {
+		if errors.Is(err, io.EOF) {
 			// we've finished reading the yaml content..I guess
 			break
 		} else if err != nil {
@@ -75,11 +76,11 @@ func (f *frontMatterHandlerImpl) Split() error {
 		}
 		line, errReading := reader.ReadString('\n')
 		lineCount = lineCount + 1
-		if errReading != nil && errReading != io.EOF {
+		if errReading != nil && !errors.Is(errReading, io.EOF) {
 			return errReading
 		}
 
-		_, errWriting := yamlTempFile.Write([]byte(line))
+		_, errWriting := yamlTempFile.WriteString(line)
 
 		if errWriting != nil {
 			return errWriting

pkg/yqlib/lib.go

@@ -15,6 +15,11 @@ import (
 
 var log = logging.MustGetLogger("yq-lib")
 
+// GetLogger returns the yq logger instance.
+func GetLogger() *logging.Logger {
+	return log
+}
+
 type operationType struct {
 	Type       string
 	NumArgs    uint // number of arguments to the op
@@ -58,7 +63,9 @@ var createMapOpType = &operationType{Type: "CREATE_MAP", NumArgs: 2, Precedence:
 var shortPipeOpType = &operationType{Type: "SHORT_PIPE", NumArgs: 2, Precedence: 45, Handler: pipeOperator}
 
 var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Handler: lengthOperator}
-var collectOpType = &operationType{Type: "COLLECT", NumArgs: 0, Precedence: 50, Handler: collectOperator}
+var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50, Handler: collectOperator}
+var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler: mapOperator}
+var mapValuesOpType = &operationType{Type: "MAP_VALUES", NumArgs: 1, Precedence: 50, Handler: mapValuesOperator}
 var encodeOpType = &operationType{Type: "ENCODE", NumArgs: 0, Precedence: 50, Handler: encodeOperator}
 var decodeOpType = &operationType{Type: "DECODE", NumArgs: 0, Precedence: 50, Handler: decodeOperator}
 
@@ -78,6 +85,10 @@ var splitDocumentOpType = &operationType{Type: "SPLIT_DOC", NumArgs: 0, Preceden
 var getVariableOpType = &operationType{Type: "GET_VARIABLE", NumArgs: 0, Precedence: 55, Handler: getVariableOperator}
 var getStyleOpType = &operationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: getStyleOperator}
 var getTagOpType = &operationType{Type: "GET_TAG", NumArgs: 0, Precedence: 50, Handler: getTagOperator}
+
+var getKeyOpType = &operationType{Type: "GET_KEY", NumArgs: 0, Precedence: 50, Handler: getKeyOperator}
+var getParentOpType = &operationType{Type: "GET_PARENT", NumArgs: 0, Precedence: 50, Handler: getParentOperator}
+
 var getCommentOpType = &operationType{Type: "GET_COMMENT", NumArgs: 0, Precedence: 50, Handler: getCommentsOperator}
 var getAnchorOpType = &operationType{Type: "GET_ANCHOR", NumArgs: 0, Precedence: 50, Handler: getAnchorOperator}
 var getAliasOptype = &operationType{Type: "GET_ALIAS", NumArgs: 0, Precedence: 50, Handler: getAliasOperator}
@@ -87,6 +98,8 @@ var getFileIndexOpType = &operationType{Type: "GET_FILE_INDEX", NumArgs: 0, Prec
 var getPathOpType = &operationType{Type: "GET_PATH", NumArgs: 0, Precedence: 50, Handler: getPathOperator}
 
 var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: explodeOperator}
+var sortByOpType = &operationType{Type: "SORT_BY", NumArgs: 1, Precedence: 50, Handler: sortByOperator}
+var sortOpType = &operationType{Type: "SORT", NumArgs: 0, Precedence: 50, Handler: sortOperator}
 var sortKeysOpType = &operationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: sortKeysOperator}
 var joinStringOpType = &operationType{Type: "JOIN", NumArgs: 1, Precedence: 50, Handler: joinStringOperator}
 var subStringOpType = &operationType{Type: "SUBSTR", NumArgs: 1, Precedence: 50, Handler: substituteStringOperator}

pkg/yqlib/lib_test.go

@@ -0,0 +1,10 @@
+package yqlib
+
+import "testing"
+
+func TestGetLogger(t *testing.T) {
+	l := GetLogger()
+	if l != log {
+		t.Fatal("GetLogger should return the yq logger instance, not a copy")
+	}
+}

pkg/yqlib/operator_add.go

@@ -2,7 +2,6 @@ package yqlib
 
 import (
 	"fmt"
-
 	"strconv"
 
 	yaml "gopkg.in/yaml.v3"
@@ -45,14 +44,14 @@ func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Candida
 	lhsNode := lhs.Node
 
 	if lhsNode.Tag == "!!null" {
-		return lhs.CreateChild(nil, rhs.Node), nil
+		return lhs.CreateReplacement(rhs.Node), nil
 	}
 
-	target := lhs.CreateChild(nil, &yaml.Node{})
+	target := lhs.CreateReplacement(&yaml.Node{})
 
 	switch lhsNode.Kind {
 	case yaml.MappingNode:
-		return nil, fmt.Errorf("Maps not yet supported for addition")
+		return nil, fmt.Errorf("maps not yet supported for addition")
 	case yaml.SequenceNode:
 		target.Node.Kind = yaml.SequenceNode
 		target.Node.Style = lhsNode.Style
@@ -60,7 +59,7 @@ func add(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Candida
 		target.Node.Content = append(lhsNode.Content, toNodes(rhs)...)
 	case yaml.ScalarNode:
 		if rhs.Node.Kind != yaml.ScalarNode {
-			return nil, fmt.Errorf("%v (%v) cannot be added to a %v", rhs.Node.Tag, rhs.Path, lhsNode.Tag)
+			return nil, fmt.Errorf("%v (%v) cannot be added to a 2%v", rhs.Node.Tag, rhs.Path, lhsNode.Tag)
 		}
 		target.Node.Kind = yaml.ScalarNode
 		target.Node.Style = lhsNode.Style

pkg/yqlib/operator_anchors_aliases.go

@@ -56,7 +56,7 @@ func getAliasOperator(d *dataTreeNavigator, context Context, expressionNode *Exp
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 		node := &yaml.Node{Kind: yaml.ScalarNode, Value: candidate.Node.Value, Tag: "!!str"}
-		result := candidate.CreateChild(nil, node)
+		result := candidate.CreateReplacement(node)
 		results.PushBack(result)
 	}
 	return context.ChildContext(results), nil
@@ -112,7 +112,7 @@ func getAnchorOperator(d *dataTreeNavigator, context Context, expressionNode *Ex
 		candidate := el.Value.(*CandidateNode)
 		anchor := candidate.Node.Anchor
 		node := &yaml.Node{Kind: yaml.ScalarNode, Value: anchor, Tag: "!!str"}
-		result := candidate.CreateChild(nil, node)
+		result := candidate.CreateReplacement(node)
 		results.PushBack(result)
 	}
 	return context.ChildContext(results), nil

pkg/yqlib/operator_assign.go

@@ -1,9 +1,16 @@
 package yqlib
 
-func assignUpdateFunc(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-	rhs.Node = unwrapDoc(rhs.Node)
-	lhs.UpdateFrom(rhs)
-	return lhs, nil
+type assignPreferences struct {
+	DontOverWriteAnchor bool
+}
+
+func assignUpdateFunc(prefs assignPreferences) crossFunctionCalculation {
+	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+		rhs.Node = unwrapDoc(rhs.Node)
+
+		lhs.UpdateFrom(rhs, prefs)
+		return lhs, nil
+	}
 }
 
 func assignUpdateOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
@@ -12,9 +19,14 @@ func assignUpdateOperator(d *dataTreeNavigator, context Context, expressionNode
 		return Context{}, err
 	}
 
+	prefs := assignPreferences{}
+	if expressionNode.Operation.Preferences != nil {
+		prefs = expressionNode.Operation.Preferences.(assignPreferences)
+	}
+
 	if !expressionNode.Operation.UpdateAssign {
 		// this works because we already ran against LHS with an editable context.
-		_, err := crossFunction(d, context.ReadOnlyClone(), expressionNode, assignUpdateFunc, false)
+		_, err := crossFunction(d, context.ReadOnlyClone(), expressionNode, assignUpdateFunc(prefs), false)
 		return context, err
 	}
 
@@ -33,7 +45,7 @@ func assignUpdateOperator(d *dataTreeNavigator, context Context, expressionNode
 		if first != nil {
 			rhsCandidate := first.Value.(*CandidateNode)
 			rhsCandidate.Node = unwrapDoc(rhsCandidate.Node)
-			candidate.UpdateFrom(rhsCandidate)
+			candidate.UpdateFrom(rhsCandidate, prefs)
 		}
 	}
 
@@ -60,7 +72,11 @@ func assignAttributesOperator(d *dataTreeNavigator, context Context, expressionN
 		first := rhs.MatchingNodes.Front()
 
 		if first != nil {
-			candidate.UpdateAttributesFrom(first.Value.(*CandidateNode))
+			prefs := assignPreferences{}
+			if expressionNode.Operation.Preferences != nil {
+				prefs = expressionNode.Operation.Preferences.(assignPreferences)
+			}
+			candidate.UpdateAttributesFrom(first.Value.(*CandidateNode), prefs)
 		}
 	}
 	return context, nil

pkg/yqlib/operator_assign_test.go

@@ -145,6 +145,16 @@ var assignOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::{a: {b: bogs}}\n",
 		},
 	},
+	{
+		description:           "Update node value that has an anchor",
+		subdescription:        "Anchor will remaple",
+		dontFormatInputForDoc: true,
+		document:              `a: &cool cat`,
+		expression:            `.a = "dog"`,
+		expected: []string{
+			"D0, P[], (doc)::a: &cool dog\n",
+		},
+	},
 	{
 		description:           "Update empty object and array",
 		dontFormatInputForDoc: true,

pkg/yqlib/operator_collect.go

@@ -6,6 +6,23 @@ import (
 	yaml "gopkg.in/yaml.v3"
 )
 
+func collectTogether(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (*CandidateNode, error) {
+	collectedNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		collectExpResults, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode)
+		if err != nil {
+			return nil, err
+		}
+		for result := collectExpResults.MatchingNodes.Front(); result != nil; result = result.Next() {
+			resultC := result.Value.(*CandidateNode)
+			log.Debugf("found this: %v", NodeToString(resultC))
+			collectedNode.Content = append(collectedNode.Content, unwrapDoc(resultC.Node))
+		}
+	}
+	return &CandidateNode{Node: collectedNode}, nil
+}
+
 func collectOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- collectOperation")
 
@@ -15,26 +32,43 @@ func collectOperator(d *dataTreeNavigator, context Context, expressionNode *Expr
 		return context.SingleChildContext(candidate), nil
 	}
 
-	var results = list.New()
-
-	node := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
-	var collectC *CandidateNode
-	if context.MatchingNodes.Front() != nil {
-		collectC = context.MatchingNodes.Front().Value.(*CandidateNode).CreateChild(nil, node)
-		if len(collectC.Path) > 0 {
-			collectC.Path = collectC.Path[:len(collectC.Path)-1]
+	var evaluateAllTogether = true
+	for matchEl := context.MatchingNodes.Front(); matchEl != nil; matchEl = matchEl.Next() {
+		evaluateAllTogether = evaluateAllTogether && matchEl.Value.(*CandidateNode).EvaluateTogether
+		if !evaluateAllTogether {
+			break
 		}
-	} else {
-		collectC = &CandidateNode{Node: node}
 	}
 
+	if evaluateAllTogether {
+		collectedNode, err := collectTogether(d, context, expressionNode.Rhs)
+		if err != nil {
+			return Context{}, err
+		}
+		return context.SingleChildContext(collectedNode), nil
+
+	}
+
+	var results = list.New()
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
-		log.Debugf("Collecting %v", NodeToString(candidate))
-		node.Content = append(node.Content, unwrapDoc(candidate.Node))
-	}
 
-	results.PushBack(collectC)
+		collectedNode := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
+		collectCandidate := candidate.CreateReplacement(collectedNode)
+
+		collectExpResults, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(candidate), expressionNode.Rhs)
+		if err != nil {
+			return Context{}, err
+		}
+
+		for result := collectExpResults.MatchingNodes.Front(); result != nil; result = result.Next() {
+			resultC := result.Value.(*CandidateNode)
+			log.Debugf("found this: %v", NodeToString(resultC))
+			collectedNode.Content = append(collectedNode.Content, unwrapDoc(resultC.Node))
+		}
+
+		results.PushBack(collectCandidate)
+	}
 
 	return context.ChildContext(results), nil
 }

pkg/yqlib/operator_collect_object.go

@@ -37,7 +37,7 @@ func collectObjectOperator(d *dataTreeNavigator, originalContext Context, expres
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidateNode := el.Value.(*CandidateNode)
 		for i := 0; i < len(first.Node.Content); i++ {
-			rotated[i].PushBack(candidateNode.CreateChild(i, candidateNode.Node.Content[i]))
+			rotated[i].PushBack(candidateNode.CreateChildInArray(i, candidateNode.Node.Content[i]))
 		}
 	}
 

pkg/yqlib/operator_collect_object_test.go

@@ -5,6 +5,15 @@ import (
 )
 
 var collectObjectOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   `[{name: cat}, {name: dog}]`,
+		expression: `.[] | {.name: "great"}`,
+		expected: []string{
+			"D0, P[], (!!map)::cat: great\n",
+			"D0, P[], (!!map)::dog: great\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   "a: []",
@@ -45,6 +54,16 @@ var collectObjectOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!map)::wrap: {name: Mike}\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   "{name: Mike}\n",
+		document2:  "{name: Bob}\n",
+		expression: `{"wrap": .}`,
+		expected: []string{
+			"D0, P[], (!!map)::wrap: {name: Mike}\n",
+			"D0, P[], (!!map)::wrap: {name: Bob}\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   "{name: Mike}\n---\n{name: Bob}",
@@ -97,11 +116,11 @@ var collectObjectOperatorScenarios = []expressionScenario{
 	{
 		skipDoc:    true,
 		document:   `{name: Mike, pets: {cows: [apl, bba]}}`,
+		document2:  `{name: Rosey, pets: {sheep: [frog, meow]}}`,
 		expression: `{"a":.name, "b":.pets}`,
 		expected: []string{
-			`D0, P[], (!!map)::a: Mike
-b: {cows: [apl, bba]}
-`,
+			"D0, P[], (!!map)::a: Mike\nb: {cows: [apl, bba]}\n",
+			"D0, P[], (!!map)::a: Rosey\nb: {sheep: [frog, meow]}\n",
 		},
 	},
 	{
@@ -112,6 +131,13 @@ b: {cows: [apl, bba]}
 			"D0, P[], (!!map)::wrap: frog\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		expression: `{"wrap": "frog", "bing": "bong"}`,
+		expected: []string{
+			"D0, P[], (!!map)::wrap: frog\nbing: bong\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `{name: Mike}`,

pkg/yqlib/operator_collect_test.go

@@ -13,6 +13,13 @@ var collectOperatorScenarios = []expressionScenario{
 			"D0, P[], ()::a:\n    - 0\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		expression: `[1,2,3]`,
+		expected: []string{
+			"D0, P[], (!!seq)::- 1\n- 2\n- 3\n",
+		},
+	},
 	{
 		description: "Collect empty",
 		document:    ``,
@@ -45,7 +52,8 @@ var collectOperatorScenarios = []expressionScenario{
 		expected: []string{
 			"D0, P[], (!!seq)::- cat\n",
 		},
-	}, {
+	},
+	{
 		document:   ``,
 		skipDoc:    true,
 		expression: `[true]`,
@@ -64,7 +72,7 @@ var collectOperatorScenarios = []expressionScenario{
 	{
 		document:   ``,
 		skipDoc:    true,
-		expression: `1 | collect`,
+		expression: `collect(1)`,
 		expected: []string{
 			"D0, P[], (!!seq)::- 1\n",
 		},
@@ -82,7 +90,16 @@ var collectOperatorScenarios = []expressionScenario{
 		expression: `[.a.b.[]]`,
 		skipDoc:    true,
 		expected: []string{
-			"D0, P[a b], (!!seq)::- 1\n- 2\n- 3\n",
+			"D0, P[], (!!seq)::- 1\n- 2\n- 3\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `[{name: cat, thing: bor}, {name: dog}]`,
+		expression: `.[] | [.name]`,
+		expected: []string{
+			"D0, P[0], (!!seq)::- cat\n",
+			"D0, P[1], (!!seq)::- dog\n",
 		},
 	},
 }

pkg/yqlib/operator_comments.go

@@ -1,8 +1,6 @@
 package yqlib
 
 import (
-	// "bufio"
-	// "bytes"
 	"bufio"
 	"bytes"
 	"container/list"
@@ -105,7 +103,7 @@ func getCommentsOperator(d *dataTreeNavigator, context Context, expressionNode *
 		comment = subsequentCommentCharaterRegExp.ReplaceAllString(comment, "\n")
 
 		node := &yaml.Node{Kind: yaml.ScalarNode, Value: comment, Tag: "!!str"}
-		result := candidate.CreateChild(nil, node)
+		result := candidate.CreateReplacement(node)
 		results.PushBack(result)
 	}
 	return context.ChildContext(results), nil

pkg/yqlib/operator_comments_test.go

@@ -113,7 +113,7 @@ var commentOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		skipDoc:               false,
+		description:           "Head comment with document split",
 		dontFormatInputForDoc: true,
 		document:              "# welcome!\n---\n# bob\na: cat # meow\n\n# have a great day",
 		expression:            `headComment`,

pkg/yqlib/operator_delete.go

@@ -47,7 +47,7 @@ func deleteFromMap(candidate *CandidateNode, childPath interface{}) {
 		key := contents[index]
 		value := contents[index+1]
 
-		childCandidate := candidate.CreateChild(key.Value, value)
+		childCandidate := candidate.CreateChildInMap(key, value)
 
 		shouldDelete := key.Value == childPath
 

pkg/yqlib/operator_document_index.go

@@ -13,7 +13,7 @@ func getDocumentIndexOperator(d *dataTreeNavigator, context Context, expressionN
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 		node := &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%v", candidate.Document), Tag: "!!int"}
-		scalar := candidate.CreateChild(nil, node)
+		scalar := candidate.CreateReplacement(node)
 		results.PushBack(scalar)
 	}
 	return context.ChildContext(results), nil

pkg/yqlib/operator_encoder_decoder.go

@@ -56,12 +56,14 @@ func encodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 		}
 
 		// dont print a new line when printing json on a single line.
-		if preferences.format == JsonOutputFormat && preferences.indent == 0 {
+		if (preferences.format == JsonOutputFormat && preferences.indent == 0) ||
+			preferences.format == CsvOutputFormat ||
+			preferences.format == TsvOutputFormat {
 			stringValue = chomper.ReplaceAllString(stringValue, "")
 		}
 
 		stringContentNode := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: stringValue}
-		results.PushBack(candidate.CreateChild(nil, stringContentNode))
+		results.PushBack(candidate.CreateReplacement(stringContentNode))
 	}
 	return context.ChildContext(results), nil
 }
@@ -85,7 +87,7 @@ func decodeOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 		//first node is a doc
 		node := unwrapDoc(&dataBucket)
 
-		results.PushBack(candidate.CreateChild(nil, node))
+		results.PushBack(candidate.CreateReplacement(node))
 	}
 	return context.ChildContext(results), nil
 }

pkg/yqlib/operator_encoder_decoder_test.go

@@ -7,34 +7,6 @@ import (
 var prefix = "D0, P[], (doc)::a:\n    cool:\n        bob: dylan\n"
 
 var encoderDecoderOperatorScenarios = []expressionScenario{
-	{
-		description:    "Encode value as yaml string",
-		subdescription: "Indent defaults to 2",
-		document:       "a:\n  cool:\n    bob: dylan",
-		expression:     `.b = (.a | to_yaml)`,
-		expected: []string{
-			prefix + "b: |\n    cool:\n      bob: dylan\n",
-		},
-	},
-	{
-		description:    "Encode value as yaml string, with custom indentation",
-		subdescription: "You can specify the indentation level as the first parameter.",
-		document:       "a:\n  cool:\n    bob: dylan",
-		expression:     `.b = (.a | to_yaml(8))`,
-		expected: []string{
-			prefix + "b: |\n    cool:\n            bob: dylan\n",
-		},
-	},
-	{
-		description:    "Encode value as yaml string, using toyaml",
-		subdescription: "Does the same thing as to_yaml, matching jq naming convention.",
-		document:       `{a: {cool: "thing"}}`,
-		expression:     `.b = (.a | to_yaml)`,
-		expected: []string{
-			`D0, P[], (doc)::{a: {cool: "thing"}, b: "{cool: \"thing\"}\n"}
-`,
-		},
-	},
 	{
 		description: "Encode value as json string",
 		document:    `{a: {cool: "thing"}}`,
@@ -55,11 +27,74 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description: "Encode value as props string",
-		document:    `{a: {cool: "thing"}}`,
-		expression:  `.b = (.a | to_props)`,
+		description:    "Encode value as json string, on one line shorthand",
+		subdescription: "Pass in a 0 indent to print json on a single line.",
+		document:       `{a: {cool: "thing"}}`,
+		expression:     `.b = (.a | @json)`,
+		expected: []string{
+			`D0, P[], (doc)::{a: {cool: "thing"}, b: '{"cool":"thing"}'}
+`,
+		},
+	},
+	{
+		description:    "Decode a json encoded string",
+		subdescription: "Keep in mind JSON is a subset of YAML. If you want idiomatic yaml, pipe through the style operator to clear out the JSON styling.",
+		document:       `a: '{"cool":"thing"}'`,
+		expression:     `.a | from_json | ... style=""`,
+		expected: []string{
+			"D0, P[a], (!!map)::cool: thing\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: {cool: "thing"}}`,
+		expression: `.b = (.a | to_props)`,
 		expected: []string{
 			`D0, P[], (doc)::{a: {cool: "thing"}, b: "cool = thing\n"}
+`,
+		},
+	},
+	{
+		description: "Encode value as props string",
+		document:    `{a: {cool: "thing"}}`,
+		expression:  `.b = (.a | @props)`,
+		expected: []string{
+			`D0, P[], (doc)::{a: {cool: "thing"}, b: "cool = thing\n"}
+`,
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "a:\n  cool:\n    bob: dylan",
+		expression: `.b = (.a | @yaml)`,
+		expected: []string{
+			prefix + "b: |\n    cool:\n      bob: dylan\n",
+		},
+	},
+	{
+		description:    "Encode value as yaml string",
+		subdescription: "Indent defaults to 2",
+		document:       "a:\n  cool:\n    bob: dylan",
+		expression:     `.b = (.a | to_yaml)`,
+		expected: []string{
+			prefix + "b: |\n    cool:\n      bob: dylan\n",
+		},
+	},
+	{
+		description:    "Encode value as yaml string, with custom indentation",
+		subdescription: "You can specify the indentation level as the first parameter.",
+		document:       "a:\n  cool:\n    bob: dylan",
+		expression:     `.b = (.a | to_yaml(8))`,
+		expected: []string{
+			prefix + "b: |\n    cool:\n            bob: dylan\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: {cool: "thing"}}`,
+		expression: `.b = (.a | to_yaml)`,
+		expected: []string{
+			`D0, P[], (doc)::{a: {cool: "thing"}, b: "{cool: \"thing\"}\n"}
 `,
 		},
 	},
@@ -98,6 +133,32 @@ var encoderDecoderOperatorScenarios = []expressionScenario{
 			"D0, P[], (doc)::a: 'foo: cat'\n",
 		},
 	},
+	{
+		description:    "Encode array of scalars as csv string",
+		subdescription: "Scalars are strings, numbers and booleans.",
+		document:       `[cat, "thing1,thing2", true, 3.40]`,
+		expression:     `@csv`,
+		expected: []string{
+			"D0, P[], (!!str)::cat,\"thing1,thing2\",true,3.40\n",
+		},
+	},
+	{
+		description: "Encode array of arrays as csv string",
+		document:    `[[cat, "thing1,thing2", true, 3.40], [dog, thing3, false, 12]]`,
+		expression:  `@csv`,
+		expected: []string{
+			"D0, P[], (!!str)::cat,\"thing1,thing2\",true,3.40\ndog,thing3,false,12\n",
+		},
+	},
+	{
+		description:    "Encode array of array scalars as tsv string",
+		subdescription: "Scalars are strings, numbers and booleans.",
+		document:       `[[cat, "thing1,thing2", true, 3.40], [dog, thing3, false, 12]]`,
+		expression:     `@tsv`,
+		expected: []string{
+			"D0, P[], (!!str)::cat\tthing1,thing2\ttrue\t3.40\ndog\tthing3\tfalse\t12\n",
+		},
+	},
 	{
 		skipDoc:               true,
 		dontFormatInputForDoc: true,

pkg/yqlib/operator_entries.go

@@ -20,7 +20,7 @@ func entrySeqFor(key *yaml.Node, value *yaml.Node) *yaml.Node {
 
 func toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode {
 	var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
-	var entriesNode = candidateNode.CreateChild(nil, sequence)
+	var entriesNode = candidateNode.CreateReplacement(sequence)
 
 	var contents = unwrapDoc(candidateNode.Node).Content
 	for index := 0; index < len(contents); index = index + 2 {
@@ -34,7 +34,7 @@ func toEntriesFromMap(candidateNode *CandidateNode) *CandidateNode {
 
 func toEntriesfromSeq(candidateNode *CandidateNode) *CandidateNode {
 	var sequence = &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq"}
-	var entriesNode = candidateNode.CreateChild(nil, sequence)
+	var entriesNode = candidateNode.CreateReplacement(sequence)
 
 	var contents = unwrapDoc(candidateNode.Node).Content
 	for index := 0; index < len(contents); index = index + 1 {
@@ -94,7 +94,7 @@ func parseEntry(d *dataTreeNavigator, entry *yaml.Node, position int) (*yaml.Nod
 
 func fromEntries(d *dataTreeNavigator, candidateNode *CandidateNode) (*CandidateNode, error) {
 	var node = &yaml.Node{Kind: yaml.MappingNode, Tag: "!!map"}
-	var mapCandidateNode = candidateNode.CreateChild(nil, node)
+	var mapCandidateNode = candidateNode.CreateReplacement(node)
 
 	var contents = unwrapDoc(candidateNode.Node).Content
 
@@ -138,25 +138,37 @@ func withEntriesOperator(d *dataTreeNavigator, context Context, expressionNode *
 		return Context{}, err
 	}
 
-	//run expression against entries
-	// splat toEntries and pipe it into Rhs
-	splatted, err := splat(d, toEntries, traversePreferences{})
-	if err != nil {
-		return Context{}, err
-	}
+	var results = list.New()
 
-	result, err := d.GetMatchingNodes(splatted, expressionNode.Rhs)
-	log.Debug("expressionNode.Rhs %v", expressionNode.Rhs.Operation.OperationType)
-	log.Debug("result %v", result)
-	if err != nil {
-		return Context{}, err
-	}
+	for el := toEntries.MatchingNodes.Front(); el != nil; el = el.Next() {
+		//run expression against entries
+		// splat toEntries and pipe it into Rhs
+		splatted, err := splat(d, context.SingleChildContext(el.Value.(*CandidateNode)), traversePreferences{})
+		if err != nil {
+			return Context{}, err
+		}
+
+		result, err := d.GetMatchingNodes(splatted, expressionNode.Rhs)
+		log.Debug("expressionNode.Rhs %v", expressionNode.Rhs.Operation.OperationType)
+		log.Debug("result %v", result)
+		if err != nil {
+			return Context{}, err
+		}
+
+		selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
+		collected, err := collectTogether(d, result, selfExpression)
+		if err != nil {
+			return Context{}, err
+		}
+
+		fromEntries, err := fromEntriesOperator(d, context.SingleChildContext(collected), expressionNode)
+		if err != nil {
+			return Context{}, err
+		}
+		results.PushBackList(fromEntries.MatchingNodes)
 
-	collected, err := collectOperator(d, result, expressionNode)
-	if err != nil {
-		return Context{}, err
 	}
 
 	//from_entries on the result
-	return fromEntriesOperator(d, collected, expressionNode)
+	return context.ChildContext(results), nil
 }

pkg/yqlib/operator_entries_test.go

@@ -52,6 +52,16 @@ var entriesOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!map)::KEY_a: 1\nKEY_b: 2\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `{a: 1, b: 2}`,
+		document2:  `{c: 1, d: 2}`,
+		expression: `with_entries(.key |= "KEY_" + .)`,
+		expected: []string{
+			"D0, P[], (!!map)::KEY_a: 1\nKEY_b: 2\n",
+			"D0, P[], (!!map)::KEY_c: 1\nKEY_d: 2\n",
+		},
+	},
 	{
 		description: "Use with_entries to filter the map",
 		document:    `{a: { b: bird }, c: { d: dog }}`,

pkg/yqlib/operator_equals_test.go

@@ -173,7 +173,7 @@ var equalsOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description: "Two non existant keys are equal",
+		description: "Two non existent keys are equal",
 		document:    "a: frog",
 		expression:  `select(.b == .c)`,
 		expected: []string{

pkg/yqlib/operator_file.go

@@ -15,7 +15,7 @@ func getFilenameOperator(d *dataTreeNavigator, context Context, expressionNode *
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 		node := &yaml.Node{Kind: yaml.ScalarNode, Value: candidate.Filename, Tag: "!!str"}
-		result := candidate.CreateChild(nil, node)
+		result := candidate.CreateReplacement(node)
 		results.PushBack(result)
 	}
 
@@ -30,7 +30,7 @@ func getFileIndexOperator(d *dataTreeNavigator, context Context, expressionNode
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 		node := &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%v", candidate.FileIndex), Tag: "!!int"}
-		result := candidate.CreateChild(nil, node)
+		result := candidate.CreateReplacement(node)
 		results.PushBack(result)
 	}
 

pkg/yqlib/operator_group_by.go

@@ -67,7 +67,7 @@ func groupBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionNo
 			resultNode.Content = append(resultNode.Content, groupResultNode)
 		}
 
-		results.PushBack(candidate.CreateChild(nil, resultNode))
+		results.PushBack(candidate.CreateReplacement(resultNode))
 
 	}
 

pkg/yqlib/operator_keys.go

@@ -7,6 +7,23 @@ import (
 	"gopkg.in/yaml.v3"
 )
 
+func getKeyOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	log.Debugf("-- getKeyOperator")
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+
+		if candidate.Key != nil {
+			results.PushBack(candidate.CreateReplacement(candidate.Key))
+		}
+	}
+
+	return context.ChildContext(results), nil
+
+}
+
 func keysOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
 	log.Debugf("-- keysOperator")
 
@@ -24,7 +41,7 @@ func keysOperator(d *dataTreeNavigator, context Context, expressionNode *Express
 			return Context{}, fmt.Errorf("Cannot get keys of %v, keys only works for maps and arrays", node.Tag)
 		}
 
-		result := candidate.CreateChild(nil, targetNode)
+		result := candidate.CreateReplacement(targetNode)
 		results.PushBack(result)
 	}
 

pkg/yqlib/operator_keys_test.go

@@ -37,6 +37,44 @@ var keysOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::[]\n",
 		},
 	},
+	{
+		description: "Retrieve array key",
+		document:    "[1,2,3]",
+		expression:  `.[1] | key`,
+		expected: []string{
+			"D0, P[1], (!!int)::1\n",
+		},
+	},
+	{
+		description: "Retrieve map key",
+		document:    "a: thing",
+		expression:  `.a | key`,
+		expected: []string{
+			"D0, P[a], (!!str)::a\n",
+		},
+	},
+	{
+		description: "No key",
+		document:    "{}",
+		expression:  `key`,
+		expected:    []string{},
+	},
+	{
+		description: "Update map key",
+		document:    "a:\n  x: 3\n  y: 4",
+		expression:  `(.a.x | key) = "meow"`,
+		expected: []string{
+			"D0, P[], (doc)::a:\n    meow: 3\n    y: 4\n",
+		},
+	},
+	{
+		description: "Get comment from map key",
+		document:    "a: \n  # comment on key\n  x: 3\n  y: 4",
+		expression:  `.a.x | key | headComment`,
+		expected: []string{
+			"D0, P[a x], (!!str)::comment on key\n",
+		},
+	},
 }
 
 func TestKeysOperatorScenarios(t *testing.T) {

pkg/yqlib/operator_length.go

@@ -31,7 +31,7 @@ func lengthOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 		}
 
 		node := &yaml.Node{Kind: yaml.ScalarNode, Value: fmt.Sprintf("%v", length), Tag: "!!int"}
-		result := candidate.CreateChild(nil, node)
+		result := candidate.CreateReplacement(node)
 		results.PushBack(result)
 	}
 

pkg/yqlib/operator_load.go

@@ -15,7 +15,7 @@ type loadPrefs struct {
 }
 
 func loadString(filename string) (*CandidateNode, error) {
-	// ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory,
+	// ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory,
 	// and ensuring that it's not possible to give a path to a file outside that directory.
 
 	filebytes, err := ioutil.ReadFile(filename) // #nosec

pkg/yqlib/operator_map.go

@@ -0,0 +1,64 @@
+package yqlib
+
+import (
+	"container/list"
+)
+
+func mapValuesOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		//run expression against entries
+		// splat toEntries and pipe it into Rhs
+		splatted, err := splat(d, context.SingleChildContext(candidate), traversePreferences{})
+		if err != nil {
+			return Context{}, err
+		}
+
+		assignUpdateExp := &ExpressionNode{
+			Operation: &Operation{OperationType: assignOpType, UpdateAssign: true},
+			Rhs:       expressionNode.Rhs,
+		}
+		_, err = assignUpdateOperator(d, splatted, assignUpdateExp)
+		if err != nil {
+			return Context{}, err
+		}
+
+	}
+
+	return context, nil
+}
+
+func mapOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		//run expression against entries
+		// splat toEntries and pipe it into Rhs
+		splatted, err := splat(d, context.SingleChildContext(candidate), traversePreferences{})
+		if err != nil {
+			return Context{}, err
+		}
+
+		result, err := d.GetMatchingNodes(splatted, expressionNode.Rhs)
+		log.Debug("expressionNode.Rhs %v", expressionNode.Rhs.Operation.OperationType)
+		log.Debug("result %v", result)
+		if err != nil {
+			return Context{}, err
+		}
+
+		selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
+		collected, err := collectTogether(d, result, selfExpression)
+		if err != nil {
+			return Context{}, err
+		}
+		collected.Node.Style = unwrapDoc(candidate.Node).Style
+
+		results.PushBack(collected)
+
+	}
+
+	return context.ChildContext(results), nil
+}

pkg/yqlib/operator_map_test.go

@@ -0,0 +1,51 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var mapOperatorScenarios = []expressionScenario{
+	{
+		skipDoc:    true,
+		document:   `[1,2,3]`,
+		document2:  `[5,6,7]`,
+		expression: `map(. + 1)`,
+		expected: []string{
+			"D0, P[], (!!seq)::[2, 3, 4]\n",
+			"D0, P[], (!!seq)::[6, 7, 8]\n",
+		},
+	},
+	{
+		description: "Map array",
+		document:    `[1,2,3]`,
+		expression:  `map(. + 1)`,
+		expected: []string{
+			"D0, P[], (!!seq)::[2, 3, 4]\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   `{a: 1, b: 2, c: 3}`,
+		document2:  `{x: 10, y: 20, z: 30}`,
+		expression: `map_values(. + 1)`,
+		expected: []string{
+			"D0, P[], (doc)::{a: 2, b: 3, c: 4}\n",
+			"D0, P[], (doc)::{x: 11, y: 21, z: 31}\n",
+		},
+	},
+	{
+		description: "Map object values",
+		document:    `{a: 1, b: 2, c: 3}`,
+		expression:  `map_values(. + 1)`,
+		expected: []string{
+			"D0, P[], (doc)::{a: 2, b: 3, c: 4}\n",
+		},
+	},
+}
+
+func TestMapOperatorScenarios(t *testing.T) {
+	for _, tt := range mapOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "map", mapOperatorScenarios)
+}

pkg/yqlib/operator_multiply.go

@@ -1,11 +1,10 @@
 package yqlib
 
 import (
+	"container/list"
 	"fmt"
 	"strconv"
 
-	"container/list"
-
 	"github.com/jinzhu/copier"
 	yaml "gopkg.in/yaml.v3"
 )
@@ -42,7 +41,7 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
 		leadingContent, headComment, footComment := getComments(lhs, rhs)
 		lhs.Node = unwrapDoc(lhs.Node)
 		rhs.Node = unwrapDoc(rhs.Node)
-		log.Debugf("Multipling LHS: %v", lhs.Node.Tag)
+		log.Debugf("Multiplying LHS: %v", lhs.Node.Tag)
 		log.Debugf("-          RHS: %v", rhs.Node.Tag)
 
 		if lhs.Node.Kind == yaml.MappingNode && rhs.Node.Kind == yaml.MappingNode ||
@@ -67,7 +66,7 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
 }
 
 func multiplyFloats(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-	target := lhs.CreateChild(nil, &yaml.Node{})
+	target := lhs.CreateReplacement(&yaml.Node{})
 	target.Node.Kind = yaml.ScalarNode
 	target.Node.Style = lhs.Node.Style
 	target.Node.Tag = "!!float"
@@ -85,7 +84,7 @@ func multiplyFloats(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, err
 }
 
 func multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
-	target := lhs.CreateChild(nil, &yaml.Node{})
+	target := lhs.CreateReplacement(&yaml.Node{})
 	target.Node.Kind = yaml.ScalarNode
 	target.Node.Style = lhs.Node.Style
 	target.Node.Tag = "!!int"

pkg/yqlib/operator_multiply_test.go

@@ -29,10 +29,22 @@ var mergeArrayWithAnchors = `sample:
 - <<: *a
 `
 
-var mergeArraysObjectKeysText = `It's a complex command, the trickyness comes from needing to have the right context in the expressions.
-First we save the second array into a variable '$two' which lets us reference it later.
-We then need to update the first array. We will use the relative update (|=) because we need to update relative to the current element of the array in the LHS in the RHS expression. 
-We set the current element of the first array as $cur. Now we multiply (merge) $cur with the matching entry in $two, by passing $two through a select filter.
+var mergeArraysObjectKeysText = `There are several parts of the complex expression. 
+The first part is doing the hard work, it creates a map from the arrays keyed by '.a', so that there are no duplicates. 
+Then there's another reduce that converts that map back to an array.
+Finally, we set the result of the merged array back into the first doc.
+
+To use this, you will need to update '.myArray' to be the expression to your array (e.g. .my.array), and '.a' to be the key field of your array (e.g. '.name')
+
+Thanks Kev from [stackoverflow](https://stackoverflow.com/a/70109529/1168223)
+`
+
+var mergeExpression = `
+(
+  ((.myArray[] | {.a: .}) as $item ireduce ({}; . * $item )) as $uniqueMap
+  | ( $uniqueMap  | to_entries | .[]) as $item ireduce([]; . + $item.value)
+) as $mergedArray
+| select(fi == 0) | .myArray = $mergedArray
 `
 
 var docWithHeader = `# here
@@ -371,11 +383,11 @@ var multiplyOperatorScenarios = []expressionScenario{
 	{
 		description:    "Merge arrays of objects together, matching on a key",
 		subdescription: mergeArraysObjectKeysText,
-		document:       `[{a: apple, b: appleB}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB}]`,
-		document2:      `[{a: banana, c: bananaC}, {a: apple, b: appleB2}, {a: dingo, c: dingoC}]`,
-		expression:     `(select(fi==1) | .[]) as $two | select(fi==0) | .[] |= (. as $cur |  $cur * ($two | select(.a == $cur.a)))`,
+		document:       `{myArray: [{a: apple, b: appleB}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB}], something: else}`,
+		document2:      `myArray: [{a: banana, c: bananaC}, {a: apple, b: appleB2}, {a: dingo, c: dingoC}]`,
+		expression:     mergeExpression,
 		expected: []string{
-			"D0, P[], (doc)::[{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC}]\n",
+			"D0, P[], (doc)::{myArray: [{a: apple, b: appleB2}, {a: kiwi, b: kiwiB}, {a: banana, b: bananaB, c: bananaC}, {a: dingo, c: dingoC}], something: else}\n",
 		},
 	},
 	{

pkg/yqlib/operator_parent.go

@@ -0,0 +1,19 @@
+package yqlib
+
+import "container/list"
+
+func getParentOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	log.Debugf("-- getParentOperator")
+
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		if candidate.Parent != nil {
+			results.PushBack(candidate.Parent)
+		}
+	}
+
+	return context.ChildContext(results), nil
+
+}

pkg/yqlib/operator_parent_test.go

@@ -0,0 +1,37 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var parentOperatorScenarios = []expressionScenario{
+	{
+		description: "Simple example",
+		document:    `a: {nested: cat}`,
+		expression:  `.a.nested | parent`,
+		expected: []string{
+			"D0, P[a], (!!map)::{nested: cat}\n",
+		},
+	},
+	{
+		description: "Parent of nested matches",
+		document:    `{a: {fruit: apple, name: bob}, b: {fruit: banana, name: sam}}`,
+		expression:  `.. | select(. == "banana") | parent`,
+		expected: []string{
+			"D0, P[b], (!!map)::{fruit: banana, name: sam}\n",
+		},
+	},
+	{
+		description: "No parent",
+		document:    `{}`,
+		expression:  `parent`,
+		expected:    []string{},
+	},
+}
+
+func TestParentOperatorScenarios(t *testing.T) {
+	for _, tt := range parentOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "parent", parentOperatorScenarios)
+}

pkg/yqlib/operator_path.go

@@ -31,7 +31,7 @@ func getPathOperator(d *dataTreeNavigator, context Context, expressionNode *Expr
 			content[pathIndex] = createPathNodeFor(path)
 		}
 		node.Content = content
-		result := candidate.CreateChild(nil, node)
+		result := candidate.CreateReplacement(node)
 		results.PushBack(result)
 	}
 

pkg/yqlib/operator_path_test.go

@@ -57,16 +57,11 @@ var pathOperatorScenarios = []expressionScenario{
 	{
 		description: "Print path and value",
 		document:    `{a: [cat, dog, frog]}`,
-		expression:  `.a.[] | select(. == "*og") | [{"path":path, "value":.}]`,
-		expected: []string{`D0, P[], (!!seq)::- path:
-    - a
-    - 1
-  value: dog
-- path:
-    - a
-    - 2
-  value: frog
-`},
+		expression:  `.a[] | select(. == "*og") | [{"path":path, "value":.}]`,
+		expected: []string{
+			"D0, P[a 1], (!!seq)::- path:\n    - a\n    - 1\n  value: dog\n",
+			"D0, P[a 2], (!!seq)::- path:\n    - a\n    - 2\n  value: frog\n",
+		},
 	},
 }
 

pkg/yqlib/operator_recursive_descent_test.go

@@ -187,7 +187,7 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
 		document:    mergeDocSample,
 		expression:  `.foobar | [..]`,
 		expected: []string{
-			"D0, P[], (!!seq)::- c: foobar_c\n  !!merge <<: *foo\n  thing: foobar_thing\n- foobar_c\n- *foo\n- foobar_thing\n",
+			"D0, P[foobar], (!!seq)::- c: foobar_c\n  !!merge <<: *foo\n  thing: foobar_thing\n- foobar_c\n- *foo\n- foobar_thing\n",
 		},
 	},
 	{
@@ -195,7 +195,7 @@ var recursiveDescentOperatorScenarios = []expressionScenario{
 		document:   mergeDocSample,
 		expression: `.foobar | [...]`,
 		expected: []string{
-			"D0, P[], (!!seq)::- c: foobar_c\n  !!merge <<: *foo\n  thing: foobar_thing\n- c\n- foobar_c\n- !!merge <<\n- *foo\n- thing\n- foobar_thing\n",
+			"D0, P[foobar], (!!seq)::- c: foobar_c\n  !!merge <<: *foo\n  thing: foobar_thing\n- c\n- foobar_c\n- !!merge <<\n- *foo\n- thing\n- foobar_thing\n",
 		},
 	},
 	{

pkg/yqlib/operator_sort.go

@@ -0,0 +1,131 @@
+package yqlib
+
+import (
+	"container/list"
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+
+	yaml "gopkg.in/yaml.v3"
+)
+
+func sortOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	selfExpression := &ExpressionNode{Operation: &Operation{OperationType: selfReferenceOpType}}
+	expressionNode.Rhs = selfExpression
+	return sortByOperator(d, context, expressionNode)
+}
+
+// context represents the current matching nodes in the expression pipeline
+//expressionNode is your current expression (sort_by)
+func sortByOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+
+	results := list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+
+		candidateNode := unwrapDoc(candidate.Node)
+
+		if candidateNode.Kind != yaml.SequenceNode {
+			return context, fmt.Errorf("%v is not an array", candidate.GetKey())
+		}
+
+		sortableArray := make(sortableNodeArray, len(candidateNode.Content))
+
+		for i, originalNode := range candidateNode.Content {
+
+			childCandidate := candidate.CreateChildInArray(i, originalNode)
+			compareContext, err := d.GetMatchingNodes(context.SingleReadonlyChildContext(childCandidate), expressionNode.Rhs)
+			if err != nil {
+				return Context{}, err
+			}
+
+			nodeToCompare := &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!null"}
+			if compareContext.MatchingNodes.Len() > 0 {
+				nodeToCompare = compareContext.MatchingNodes.Front().Value.(*CandidateNode).Node
+			}
+
+			log.Debug("going to compare %v by %v", NodeToString(candidate.CreateReplacement(originalNode)), NodeToString(candidate.CreateReplacement(nodeToCompare)))
+
+			sortableArray[i] = sortableNode{Node: originalNode, NodeToCompare: nodeToCompare}
+
+			if nodeToCompare.Kind != yaml.ScalarNode {
+				return Context{}, fmt.Errorf("sort only works for scalars, got %v", nodeToCompare.Tag)
+			}
+
+		}
+
+		sort.Stable(sortableArray)
+
+		sortedList := &yaml.Node{Kind: yaml.SequenceNode, Tag: "!!seq", Style: candidateNode.Style}
+		sortedList.Content = make([]*yaml.Node, len(candidateNode.Content))
+
+		for i, sortedNode := range sortableArray {
+			sortedList.Content[i] = sortedNode.Node
+		}
+		results.PushBack(candidate.CreateReplacement(sortedList))
+	}
+	return context.ChildContext(results), nil
+}
+
+type sortableNode struct {
+	Node          *yaml.Node
+	NodeToCompare *yaml.Node
+}
+
+type sortableNodeArray []sortableNode
+
+func (a sortableNodeArray) Len() int      { return len(a) }
+func (a sortableNodeArray) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
+
+func (a sortableNodeArray) Less(i, j int) bool {
+	lhs := a[i].NodeToCompare
+	rhs := a[j].NodeToCompare
+
+	if lhs.Tag == "!!null" && rhs.Tag != "!!null" {
+		return true
+	} else if lhs.Tag != "!!null" && rhs.Tag == "!!null" {
+		return false
+	} else if lhs.Tag == "!!bool" && rhs.Tag != "!!bool" {
+		return true
+	} else if lhs.Tag != "!!bool" && rhs.Tag == "!!bool" {
+		return false
+	} else if lhs.Tag == "!!bool" && rhs.Tag == "!!bool" {
+		lhsTruthy, err := isTruthyNode(lhs)
+		if err != nil {
+			panic(fmt.Errorf("could not parse %v as boolean: %w", lhs.Value, err))
+		}
+
+		rhsTruthy, err := isTruthyNode(rhs)
+		if err != nil {
+			panic(fmt.Errorf("could not parse %v as boolean: %w", rhs.Value, err))
+		}
+
+		return !lhsTruthy && rhsTruthy
+	} else if lhs.Tag != rhs.Tag || lhs.Tag == "!!str" {
+		return strings.Compare(lhs.Value, rhs.Value) < 0
+	} else if lhs.Tag == "!!int" && rhs.Tag == "!!int" {
+		_, lhsNum, err := parseInt(lhs.Value)
+		if err != nil {
+			panic(err)
+		}
+		_, rhsNum, err := parseInt(rhs.Value)
+		if err != nil {
+			panic(err)
+		}
+		return lhsNum < rhsNum
+	} else if (lhs.Tag == "!!int" || lhs.Tag == "!!float") && (rhs.Tag == "!!int" || rhs.Tag == "!!float") {
+		lhsNum, err := strconv.ParseFloat(lhs.Value, 64)
+		if err != nil {
+			panic(err)
+		}
+		rhsNum, err := strconv.ParseFloat(rhs.Value, 64)
+		if err != nil {
+			panic(err)
+		}
+		return lhsNum < rhsNum
+	}
+
+	return true
+}

pkg/yqlib/operator_sort_test.go

@@ -0,0 +1,63 @@
+package yqlib
+
+import "testing"
+
+var sortByOperatorScenarios = []expressionScenario{
+	{
+		description: "Sort by string field",
+		document:    "[{a: banana},{a: cat},{a: apple}]",
+		expression:  `sort_by(.a)`,
+		expected: []string{
+			"D0, P[], (!!seq)::[{a: apple}, {a: banana}, {a: cat}]\n",
+		},
+	},
+	{
+		description:    "Sort is stable",
+		subdescription: "Note the order of the elements in unchanged when equal in sorting.",
+		document:       "[{a: banana, b: 1}, {a: banana, b: 2}, {a: banana, b: 3}, {a: banana, b: 4}]",
+		expression:     `sort_by(.a)`,
+		expected: []string{
+			"D0, P[], (!!seq)::[{a: banana, b: 1}, {a: banana, b: 2}, {a: banana, b: 3}, {a: banana, b: 4}]\n",
+		},
+	},
+	{
+		description: "Sort by numeric field",
+		document:    "[{a: 10},{a: 100},{a: 1}]",
+		expression:  `sort_by(.a)`,
+		expected: []string{
+			"D0, P[], (!!seq)::[{a: 1}, {a: 10}, {a: 100}]\n",
+		},
+	},
+	{
+		skipDoc:    true,
+		document:   "[{a: 1.1},{a: 1.001},{a: 1.01}]",
+		expression: `sort_by(.a)`,
+		expected: []string{
+			"D0, P[], (!!seq)::[{a: 1.001}, {a: 1.01}, {a: 1.1}]\n",
+		},
+	},
+	{
+		description: "Sort, nulls come first",
+		document:    "[8,3,null,6, true, false, cat]",
+		expression:  `sort`,
+		expected: []string{
+			"D0, P[], (!!seq)::[null, false, true, 3, 6, 8, cat]\n",
+		},
+	},
+	{
+		skipDoc:     true,
+		description: "false before true",
+		document:    "[{a: false, b: 1}, {a: true, b: 2}, {a: false, b: 3}]",
+		expression:  `sort_by(.a)`,
+		expected: []string{
+			"D0, P[], (!!seq)::[{a: false, b: 1}, {a: false, b: 3}, {a: true, b: 2}]\n",
+		},
+	},
+}
+
+func TestSortByOperatorScenarios(t *testing.T) {
+	for _, tt := range sortByOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentScenarios(t, "sort", sortByOperatorScenarios)
+}

pkg/yqlib/operator_strings.go

@@ -66,7 +66,7 @@ func substituteStringOperator(d *dataTreeNavigator, context Context, expressionN
 		}
 
 		targetNode := substitute(node.Value, regEx, replacementText)
-		result := candidate.CreateChild(nil, targetNode)
+		result := candidate.CreateReplacement(targetNode)
 		results.PushBack(result)
 	}
 
@@ -150,7 +150,7 @@ func match(matchPrefs matchPreferences, regEx *regexp.Regexp, candidate *Candida
 			createScalarNode("captures", "captures"),
 			capturesNode,
 		)
-		results.PushBack(candidate.CreateChild(nil, node))
+		results.PushBack(candidate.CreateReplacement(node))
 
 	}
 
@@ -187,7 +187,7 @@ func capture(matchPrefs matchPreferences, regEx *regexp.Regexp, candidate *Candi
 			}
 		}
 
-		results.PushBack(candidate.CreateChild(nil, capturesNode))
+		results.PushBack(candidate.CreateReplacement(capturesNode))
 
 	}
 
@@ -321,7 +321,7 @@ func joinStringOperator(d *dataTreeNavigator, context Context, expressionNode *E
 			return Context{}, fmt.Errorf("cannot join with %v, can only join arrays of scalars", node.Tag)
 		}
 		targetNode := join(node.Content, joinStr)
-		result := candidate.CreateChild(nil, targetNode)
+		result := candidate.CreateReplacement(targetNode)
 		results.PushBack(result)
 	}
 
@@ -365,7 +365,7 @@ func splitStringOperator(d *dataTreeNavigator, context Context, expressionNode *
 			return Context{}, fmt.Errorf("Cannot split %v, can only split strings", node.Tag)
 		}
 		targetNode := split(node.Value, splitStr)
-		result := candidate.CreateChild(nil, targetNode)
+		result := candidate.CreateReplacement(targetNode)
 		results.PushBack(result)
 	}
 

pkg/yqlib/operator_strings_test.go

@@ -99,7 +99,7 @@ var stringsOperatorScenarios = []expressionScenario{
 	},
 	{
 		description:    "Test using regex",
-		subdescription: "Like jq'q equivalant, this works like match but only returns true/false instead of full match details",
+		subdescription: "Like jq'q equivalent, this works like match but only returns true/false instead of full match details",
 		document:       `["cat", "dog"]`,
 		expression:     `.[] | test("at")`,
 		expected: []string{

pkg/yqlib/operator_style.go

@@ -100,7 +100,7 @@ func getStyleOperator(d *dataTreeNavigator, context Context, expressionNode *Exp
 			style = "<unknown>"
 		}
 		node := &yaml.Node{Kind: yaml.ScalarNode, Value: style, Tag: "!!str"}
-		result := candidate.CreateChild(nil, node)
+		result := candidate.CreateReplacement(node)
 		results.PushBack(result)
 	}
 

pkg/yqlib/operator_subtract.go

@@ -2,7 +2,6 @@ package yqlib
 
 import (
 	"fmt"
-
 	"strconv"
 
 	"gopkg.in/yaml.v3"
@@ -49,10 +48,10 @@ func subtract(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Ca
 	lhsNode := lhs.Node
 
 	if lhsNode.Tag == "!!null" {
-		return lhs.CreateChild(nil, rhs.Node), nil
+		return lhs.CreateReplacement(rhs.Node), nil
 	}
 
-	target := lhs.CreateChild(nil, &yaml.Node{})
+	target := lhs.CreateReplacement(&yaml.Node{})
 
 	switch lhsNode.Kind {
 	case yaml.MappingNode:

pkg/yqlib/operator_tag.go

@@ -55,7 +55,7 @@ func getTagOperator(d *dataTreeNavigator, context Context, expressionNode *Expre
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 		node := &yaml.Node{Kind: yaml.ScalarNode, Value: unwrapDoc(candidate.Node).Tag, Tag: "!!str"}
-		result := candidate.CreateChild(nil, node)
+		result := candidate.CreateReplacement(node)
 		results.PushBack(result)
 	}
 

pkg/yqlib/operator_traverse_path.go

@@ -42,7 +42,7 @@ func traverse(d *dataTreeNavigator, context Context, matchingNode *CandidateNode
 
 	if value.Tag == "!!null" && operation.Value != "[]" {
 		log.Debugf("Guessing kind")
-		// we must ahve added this automatically, lets guess what it should be now
+		// we must have added this automatically, lets guess what it should be now
 		switch operation.Value.(type) {
 		case int, int64:
 			log.Debugf("probably an array")
@@ -70,7 +70,7 @@ func traverse(d *dataTreeNavigator, context Context, matchingNode *CandidateNode
 	case yaml.DocumentNode:
 		log.Debug("digging into doc node")
 
-		return traverse(d, context, matchingNode.CreateChild(nil, matchingNode.Node.Content[0]), operation)
+		return traverse(d, context, matchingNode.CreateChildInMap(nil, matchingNode.Node.Content[0]), operation)
 	default:
 		return list.New(), nil
 	}
@@ -87,7 +87,7 @@ func traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode
 		return Context{}, err
 	}
 
-	// rhs is a collect expression that will yield indexes to retreive of the arrays
+	// rhs is a collect expression that will yield indexes to retrieve of the arrays
 
 	rhs, err := d.GetMatchingNodes(context.ReadOnlyClone(), expressionNode.Rhs)
 
@@ -96,8 +96,8 @@ func traverseArrayOperator(d *dataTreeNavigator, context Context, expressionNode
 	}
 	prefs := traversePreferences{}
 
-	if expressionNode.Rhs.Rhs != nil && expressionNode.Rhs.Rhs.Operation.Preferences != nil {
-		prefs = expressionNode.Rhs.Rhs.Operation.Preferences.(traversePreferences)
+	if expressionNode.Operation.Preferences != nil {
+		prefs = expressionNode.Operation.Preferences.(traversePreferences)
 	}
 	var indicesToTraverse = rhs.MatchingNodes.Front().Value.(*CandidateNode).Node.Content
 
@@ -147,7 +147,7 @@ func traverseArrayIndices(context Context, matchingNode *CandidateNode, indicesT
 	} else if node.Kind == yaml.MappingNode {
 		return traverseMapWithIndices(context, matchingNode, indicesToTraverse, prefs)
 	} else if node.Kind == yaml.DocumentNode {
-		return traverseArrayIndices(context, matchingNode.CreateChild(nil, matchingNode.Node.Content[0]), indicesToTraverse, prefs)
+		return traverseArrayIndices(context, matchingNode.CreateChildInMap(nil, matchingNode.Node.Content[0]), indicesToTraverse, prefs)
 	}
 	log.Debugf("OperatorArrayTraverse skipping %v as its a %v", matchingNode, node.Tag)
 	return list.New(), nil
@@ -178,10 +178,9 @@ func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node, pr
 	node := unwrapDoc(candidate.Node)
 	if len(indices) == 0 {
 		log.Debug("splatting")
-		var index int64
-		for index = 0; index < int64(len(node.Content)); index = index + 1 {
-
-			newMatches.PushBack(candidate.CreateChild(index, node.Content[index]))
+		var index int
+		for index = 0; index < len(node.Content); index = index + 1 {
+			newMatches.PushBack(candidate.CreateChildInArray(index, node.Content[index]))
 		}
 		return newMatches, nil
 
@@ -194,7 +193,7 @@ func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node, pr
 			continue
 		}
 		if err != nil {
-			return nil, fmt.Errorf("Cannot index array with '%v' (%v)", indexNode.Value, err)
+			return nil, fmt.Errorf("Cannot index array with '%v' (%w)", indexNode.Value, err)
 		}
 		indexToUse := index
 		contentLength := int64(len(node.Content))
@@ -211,7 +210,7 @@ func traverseArrayWithIndices(candidate *CandidateNode, indices []*yaml.Node, pr
 			return nil, fmt.Errorf("Index [%v] out of range, array size is %v", index, contentLength)
 		}
 
-		newMatches.PushBack(candidate.CreateChild(index, node.Content[indexToUse]))
+		newMatches.PushBack(candidate.CreateChildInArray(int(index), node.Content[indexToUse]))
 	}
 	return newMatches, nil
 }
@@ -237,13 +236,13 @@ func traverseMap(context Context, matchingNode *CandidateNode, key string, prefs
 
 		if prefs.IncludeMapKeys {
 			log.Debug("including key")
-			candidateNode := matchingNode.CreateChild(key, keyNode)
+			candidateNode := matchingNode.CreateChildInMap(keyNode, keyNode)
 			candidateNode.IsMapKey = true
 			newMatches.Set(fmt.Sprintf("keyOf-%v", candidateNode.GetKey()), candidateNode)
 		}
 		if !prefs.DontIncludeMapValues {
 			log.Debug("including value")
-			candidateNode := matchingNode.CreateChild(key, valueNode)
+			candidateNode := matchingNode.CreateChildInMap(keyNode, valueNode)
 			newMatches.Set(candidateNode.GetKey(), candidateNode)
 		}
 	}
@@ -282,13 +281,13 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode,
 			log.Debug("MATCHED")
 			if prefs.IncludeMapKeys {
 				log.Debug("including key")
-				candidateNode := candidate.CreateChild(key.Value, key)
+				candidateNode := candidate.CreateChildInMap(key, key)
 				candidateNode.IsMapKey = true
 				newMatches.Set(fmt.Sprintf("keyOf-%v", candidateNode.GetKey()), candidateNode)
 			}
 			if !prefs.DontIncludeMapValues {
 				log.Debug("including value")
-				candidateNode := candidate.CreateChild(key.Value, value)
+				candidateNode := candidate.CreateChildInMap(key, value)
 				newMatches.Set(candidateNode.GetKey(), candidateNode)
 			}
 		}
@@ -300,7 +299,7 @@ func doTraverseMap(newMatches *orderedmap.OrderedMap, candidate *CandidateNode,
 func traverseMergeAnchor(newMatches *orderedmap.OrderedMap, originalCandidate *CandidateNode, value *yaml.Node, wantedKey string, prefs traversePreferences, splat bool) error {
 	switch value.Kind {
 	case yaml.AliasNode:
-		candidateNode := originalCandidate.CreateChild(nil, value.Alias)
+		candidateNode := originalCandidate.CreateReplacement(value.Alias)
 		return doTraverseMap(newMatches, candidateNode, wantedKey, prefs, splat)
 	case yaml.SequenceNode:
 		for _, childValue := range value.Content {

pkg/yqlib/operator_unique.go

@@ -56,7 +56,7 @@ func uniqueBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionN
 			resultNode.Content = append(resultNode.Content, el.Value.(*yaml.Node))
 		}
 
-		results.PushBack(candidate.CreateChild(nil, resultNode))
+		results.PushBack(candidate.CreateReplacement(resultNode))
 	}
 
 	return context.ChildContext(results), nil

pkg/yqlib/operators.go

@@ -144,7 +144,7 @@ func createBooleanCandidate(owner *CandidateNode, value bool) *CandidateNode {
 		valString = "false"
 	}
 	node := &yaml.Node{Kind: yaml.ScalarNode, Value: valString, Tag: "!!bool"}
-	return owner.CreateChild(nil, node)
+	return owner.CreateReplacement(node)
 }
 
 func createTraversalTree(path []interface{}, traversePrefs traversePreferences, targetKey bool) *ExpressionNode {

pkg/yqlib/operators_test.go

@@ -45,7 +45,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
 
 	node, err := NewExpressionParser().ParseExpression(s.expression)
 	if err != nil {
-		t.Error(fmt.Errorf("Error parsing expression %v of %v: %v", s.expression, s.description, err))
+		t.Error(fmt.Errorf("Error parsing expression %v of %v: %w", s.expression, s.description, err))
 		return
 	}
 	inputs := list.New()
@@ -84,7 +84,7 @@ func testScenario(t *testing.T, s *expressionScenario) {
 	context, err := NewDataTreeNavigator().GetMatchingNodes(Context{MatchingNodes: inputs}, node)
 
 	if err != nil {
-		t.Error(fmt.Errorf("%v: %v", err, s.expression))
+		t.Error(fmt.Errorf("%w: %v", err, s.expression))
 		return
 	}
 	test.AssertResultComplexWithContext(t, s.expected, resultsToString(t, context.MatchingNodes), fmt.Sprintf("desc: %v\nexp: %v\ndoc: %v", s.description, s.expression, s.document))
@@ -253,7 +253,7 @@ func documentOutput(t *testing.T, w *bufio.Writer, s expressionScenario, formatt
 
 	node, err := NewExpressionParser().ParseExpression(s.expression)
 	if err != nil {
-		t.Error(fmt.Errorf("Error parsing expression %v of %v: %v", s.expression, s.description, err))
+		t.Error(fmt.Errorf("Error parsing expression %v of %v: %w", s.expression, s.description, err))
 		return
 	}
 

pkg/yqlib/printer.go

@@ -23,6 +23,8 @@ const (
 	YamlOutputFormat = 1 << iota
 	JsonOutputFormat
 	PropsOutputFormat
+	CsvOutputFormat
+	TsvOutputFormat
 )
 
 func OutputFormatFromString(format string) (PrinterOutputFormat, error) {
@@ -33,8 +35,12 @@ func OutputFormatFromString(format string) (PrinterOutputFormat, error) {
 		return JsonOutputFormat, nil
 	case "props", "p":
 		return PropsOutputFormat, nil
+	case "csv", "c":
+		return CsvOutputFormat, nil
+	case "tsv", "t":
+		return TsvOutputFormat, nil
 	default:
-		return 0, fmt.Errorf("Unknown fromat '%v' please use [yaml|json|props]", format)
+		return 0, fmt.Errorf("unknown format '%v' please use [yaml|json|props|csv|tsv]", format)
 	}
 }
 
@@ -87,13 +93,19 @@ func (p *resultsPrinter) printNode(node *yaml.Node, writer io.Writer) error {
 		return writeString(writer, node.Value+"\n")
 	}
 
-	if p.outputFormat == JsonOutputFormat {
+	switch p.outputFormat {
+	case JsonOutputFormat:
 		encoder = NewJsonEncoder(writer, p.indent)
-	} else if p.outputFormat == PropsOutputFormat {
+	case PropsOutputFormat:
 		encoder = NewPropertiesEncoder(writer)
-	} else {
+	case CsvOutputFormat:
+		encoder = NewCsvEncoder(writer, ',')
+	case TsvOutputFormat:
+		encoder = NewCsvEncoder(writer, '\t')
+	case YamlOutputFormat:
 		encoder = NewYamlEncoder(writer, p.indent, p.colorsEnabled)
 	}
+
 	return encoder.Encode(node)
 }
 

pkg/yqlib/stream_evaluator.go

@@ -2,6 +2,7 @@ package yqlib
 
 import (
 	"container/list"
+	"errors"
 	"io"
 	"os"
 
@@ -95,7 +96,7 @@ func (s *streamEvaluator) Evaluate(filename string, reader io.Reader, node *Expr
 		var dataBucket yaml.Node
 		errorReading := decoder.Decode(&dataBucket)
 
-		if errorReading == io.EOF {
+		if errors.Is(errorReading, io.EOF) {
 			s.fileIndex = s.fileIndex + 1
 			return currentIndex, nil
 		} else if errorReading != nil {

pkg/yqlib/utils.go

@@ -3,6 +3,7 @@ package yqlib
 import (
 	"bufio"
 	"container/list"
+	"errors"
 	"io"
 	"os"
 	"regexp"
@@ -16,7 +17,7 @@ func readStream(filename string, leadingContentPreProcessing bool) (io.Reader, s
 	if filename == "-" {
 		reader = bufio.NewReader(os.Stdin)
 	} else {
-		// ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory,
+		// ignore CWE-22 gosec issue - that's more targeted for http based apps that run in a public directory,
 		// and ensuring that it's not possible to give a path to a file outside thar directory.
 		file, err := os.Open(filename) // #nosec
 		if err != nil {
@@ -44,7 +45,7 @@ func processLeadingContent(mappedDoc *CandidateNode, writer io.Writer, printDocS
 	for {
 
 		readline, errReading := reader.ReadString('\n')
-		if errReading != nil && errReading != io.EOF {
+		if errReading != nil && !errors.Is(errReading, io.EOF) {
 			return errReading
 		}
 		if strings.Contains(readline, "$yqDocSeperator$") {
@@ -59,7 +60,7 @@ func processLeadingContent(mappedDoc *CandidateNode, writer io.Writer, printDocS
 			}
 		}
 
-		if errReading == io.EOF {
+		if errors.Is(errReading, io.EOF) {
 			if readline != "" {
 				// the last comment we read didn't have a new line, put one in
 				if err := writeString(writer, "\n"); err != nil {
@@ -78,7 +79,7 @@ func processReadStream(reader *bufio.Reader) (io.Reader, string, error) {
 	var sb strings.Builder
 	for {
 		peekBytes, err := reader.Peek(3)
-		if err == io.EOF {
+		if errors.Is(err, io.EOF) {
 			// EOF are handled else where..
 			return reader, sb.String(), nil
 		} else if err != nil {
@@ -86,7 +87,7 @@ func processReadStream(reader *bufio.Reader) (io.Reader, string, error) {
 		} else if string(peekBytes) == "---" {
 			_, err := reader.ReadString('\n')
 			sb.WriteString("$yqDocSeperator$\n")
-			if err == io.EOF {
+			if errors.Is(err, io.EOF) {
 				return reader, sb.String(), nil
 			} else if err != nil {
 				return reader, sb.String(), err
@@ -94,7 +95,7 @@ func processReadStream(reader *bufio.Reader) (io.Reader, string, error) {
 		} else if commentLineRegEx.MatchString(string(peekBytes)) {
 			line, err := reader.ReadString('\n')
 			sb.WriteString(line)
-			if err == io.EOF {
+			if errors.Is(err, io.EOF) {
 				return reader, sb.String(), nil
 			} else if err != nil {
 				return reader, sb.String(), err
@@ -114,7 +115,7 @@ func readDocuments(reader io.Reader, filename string, fileIndex int) (*list.List
 		var dataBucket yaml.Node
 		errorReading := decoder.Decode(&dataBucket)
 
-		if errorReading == io.EOF {
+		if errors.Is(errorReading, io.EOF) {
 			switch reader := reader.(type) {
 			case *os.File:
 				safelyCloseFile(reader)

release_notes.txt

@@ -1,3 +1,18 @@
+4.16.1:
+  - Added csv, tsv output formats
+  - Added map, map_values operators
+  - Fixed bug in collect
+  - Fixed permissions issue in Dockerfile (#1014)
+  - Fixed assignment operator to no longer overwrite anchor (#1029)
+
+4.15.1:
+  - Added 'load/strload' operators for dynamically loading content from files
+  - Added 'key' operator
+  - Added 'parent' operator
+  - Smarter MAN page installation script (thanks @coolaj86)
+  - Dockerfile improvements (thanks @actualben)
+  - Error handling improvements (thanks @mmorel-35)
+
 4.14.2:
   - Fixed header preprocessing issue (#1000)
   - Bumped version dependencies

scripts/check.sh

@@ -5,8 +5,8 @@ set -o pipefail
 
 if command -v golangci-lint &> /dev/null
 then
-    golangci-lint run --timeout=5m
+    golangci-lint run --verbose
 else
-  ./bin/golangci-lint run --timeout=5m
+  ./bin/golangci-lint run --verbose
 fi
 

scripts/devtools.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
 set -ex
-go get golang.org/x/tools/cmd/goimports
-wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.37.1
+go mod download golang.org/x/tools@latest
+wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.43.0
 wget -O- -nv https://raw.githubusercontent.com/securego/gosec/master/install.sh | sh -s v2.9.1

scripts/install-man-page.sh

@@ -1,3 +1,14 @@
 #!/bin/sh
 
-cp yq.1 /usr/local/share/man/man1/.
+my_path="$(command -v yq)"
+
+if [ -z "$my_path" ]; then
+  echo "'yq' wasn't found in your PATH, so we don't know where to put the man pages."
+  echo "Please update your PATH to include yq, and run this script again."
+  exit 1
+fi
+
+# ex: ~/.local/bin/yq => ~/.local/
+my_prefix="$(dirname "$(dirname "$(command -v yq)")")"
+mkdir -p "$my_prefix/share/man/man1/"
+cp yq.1 "$my_prefix/share/man/man1/"

scripts/test-docker.sh

@@ -0,0 +1,5 @@
+#! /bin/bash
+set -e
+
+docker build . -t temp
+docker run --rm -it --entrypoint sh temp -c 'touch a'

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: '4.14.2'
+version: '4.15.1'
 summary: A lightweight and portable command-line YAML processor
 description: |
   The aim of the project is to be the jq or sed of yaml files.

test.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+# load array into a bash array
+# need to output each entry as a single line
+# readarray identityMappings < <(./yq e -o=j -I=0 '.identitymappings[]' test.yml )
+
+# for identityMapping in "${identityMappings[@]}"; do
+#     # identity mapping is a yaml snippet representing a single entry
+#     roleArn=$(echo "$identityMapping" | yq e '.arn' -)
+#     echo "roleArn: $roleArn"
+# done
+
+
+
+
+while IFS=$'\t' read -r roleArn group user _; do
+  echo "Role:  $roleArn"
+  echo "Group: $group"
+  echo "User:  $user"
+done < <(yq -j read test.yaml \
+         | jq -r '.identitymappings[] | [.arn, .group, .user] | @tsv')