Bumping version

Bumps [github.com/goccy/go-json](https://github.com/goccy/go-json) from 0.10.2 to 0.10.3.
- [Release notes](https://github.com/goccy/go-json/releases)
- [Changelog](https://github.com/goccy/go-json/blob/master/CHANGELOG.md)
- [Commits](https://github.com/goccy/go-json/compare/v0.10.2...v0.10.3)

---
updated-dependencies:
- dependency-name: github.com/goccy/go-json
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

CONTRIBUTING.md

@@ -1,4 +1,4 @@
-# Development
+.# Development
 
 1. Install (Golang)[https://golang.org/]
 1. Run  `scripts/devtools.sh` to install the required devtools
@@ -25,6 +25,8 @@ The pipeline will run the tests and automatically concatenate the files together
 
 The first step is to find if what you want is automatically generated or not - start by looking in the master branch. 
 
+Note that PRs with small changes (e.g. minor typos) may not be merged (see https://joel.net/how-one-guy-ruined-hacktoberfest2020-drama).
+
 ### Updating dynamic documentation from master
 - Search for the documentation you want to update. If you find matches in a `*_test.go` file - update that, as that will automatically update the matching `*.md` file 
 - Assuming you are updating a `*_test.go` file, once updated, run the test to regenerated the docs. E.g. for the 'Add' test generated docs, from the pkg/yqlib folder run:

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.22.1 as builder
+FROM golang:1.22.4 as builder
 
 WORKDIR /go/src/mikefarah/yq
 

Dockerfile.dev

@@ -1,12 +1,15 @@
-FROM golang:1.22.1
+FROM golang:1.22.4
+
+RUN apt-get update && \
+    apt-get install -y npm && \
+    npm install -g npx cspell@latest
 
 COPY scripts/devtools.sh /opt/devtools.sh
 
-RUN set -e -x \
-    && /opt/devtools.sh
-ENV PATH=/go/bin:$PATH
+RUN set -e -x && \
+    /opt/devtools.sh
 
-RUN apt-get update && apt-get install -y npm && npm install -g npx cspell@latest
+ENV PATH=/go/bin:$PATH
 
 ENV CGO_ENABLED 0
 ENV GOPATH /go:/yq

Makefile.variables

@@ -26,6 +26,7 @@ ifeq ($(CYG_CHECK),1)
 else
 	# all non-windows environments
 	ROOT := $(shell pwd)
+	SELINUX := $(shell which getenforce 2>&1 >/dev/null && echo :z)
 endif
 
 DEV_IMAGE := ${PROJECT}_dev
@@ -33,7 +34,7 @@ DEV_IMAGE := ${PROJECT}_dev
 ENGINERUN := ${ENGINE} run --rm \
 	-e LDFLAGS="${LDFLAGS}" \
 	-e GITHUB_TOKEN="${GITHUB_TOKEN}" \
-	-v ${ROOT}/vendor:/go/src \
-	-v ${ROOT}:/${PROJECT}/src/${IMPORT_PATH} \
+	-v ${ROOT}/vendor:/go/src${SELINUX} \
+	-v ${ROOT}:/${PROJECT}/src/${IMPORT_PATH}${SELINUX} \
 	-w /${PROJECT}/src/${IMPORT_PATH} \
 	${DEV_IMAGE}

README.md

@@ -295,12 +295,18 @@ sudo port install yq
 Supported by @herbygillot (https://ports.macports.org/maintainer/github/herbygillot)
 
 ### Alpine Linux
-- Enable edge/community repo by adding ```$MIRROR/alpine/edge/community``` to ```/etc/apk/repositories```
-- Update database index with ```apk update```
-- Install yq with ```apk add yq```
 
-Supported by Tuan Hoang
-https://pkgs.alpinelinux.org/package/edge/community/x86/yq
+Alpine Linux v3.20+ (and Edge):
+```
+apk add yq-go
+```
+
+Alpine Linux up to v3.19:
+```
+apk add yq
+```
+
+Supported by Tuan Hoang (https://pkgs.alpinelinux.org/packages?name=yq-go)
 
 
 ## Features
@@ -349,7 +355,6 @@ Available Commands:
   eval             (default) Apply the expression to each document in each yaml file in sequence
   eval-all         Loads _all_ yaml documents of _all_ yaml files and runs expression once
   help             Help about any command
-  shell-completion Generate completion script
 
 Flags:
   -C, --colors                        force print with colors

cmd/completion.go

@@ -7,19 +7,20 @@ import (
 )
 
 var completionCmd = &cobra.Command{
-	Use:   "shell-completion [bash|zsh|fish|powershell]",
-	Short: "Generate completion script",
+	Use:     "completion [bash|zsh|fish|powershell]",
+	Aliases: []string{"shell-completion"},
+	Short:   "Generate the autocompletion script for the specified shell",
 	Long: `To load completions:
 
 Bash:
 
-$ source <(yq shell-completion bash)
+$ source <(yq completion bash)
 
 # To load completions for each session, execute once:
 Linux:
-  $ yq shell-completion bash > /etc/bash_completion.d/yq
+  $ yq completion bash > /etc/bash_completion.d/yq
 MacOS:
-  $ yq shell-completion bash > /usr/local/etc/bash_completion.d/yq
+  $ yq completion bash > /usr/local/etc/bash_completion.d/yq
 
 Zsh:
 
@@ -29,16 +30,16 @@ Zsh:
 $ echo "autoload -U compinit; compinit" >> ~/.zshrc
 
 # To load completions for each session, execute once:
-$ yq shell-completion zsh > "${fpath[1]}/_yq"
+$ yq completion zsh > "${fpath[1]}/_yq"
 
 # You will need to start a new shell for this setup to take effect.
 
 Fish:
 
-$ yq shell-completion fish | source
+$ yq completion fish | source
 
 # To load completions for each session, execute once:
-$ yq shell-completion fish > ~/.config/fish/completions/yq.fish
+$ yq completion fish > ~/.config/fish/completions/yq.fish
 `,
 	DisableFlagsInUseLine: true,
 	ValidArgs:             []string{"bash", "zsh", "fish", "powershell"},

cmd/root.go

@@ -5,8 +5,6 @@ import (
 	"os"
 	"strings"
 
-	"github.com/mikefarah/yq/v4/pkg/properties"
-	"github.com/mikefarah/yq/v4/pkg/xml"
 	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"github.com/spf13/cobra"
 	logging "gopkg.in/op/go-logging.v1"
@@ -119,27 +117,27 @@ yq -P -oy sample.json
 		panic(err)
 	}
 
-	rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", xml.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
+	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.AttributePrefix, "xml-attribute-prefix", yqlib.ConfiguredXMLPreferences.AttributePrefix, "prefix for xml attributes")
 	if err = rootCmd.RegisterFlagCompletionFunc("xml-attribute-prefix", cobra.NoFileCompletions); err != nil {
 		panic(err)
 	}
-	rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.ContentName, "xml-content-name", xml.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")
+	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ContentName, "xml-content-name", yqlib.ConfiguredXMLPreferences.ContentName, "name for xml content (if no attribute name is present).")
 	if err = rootCmd.RegisterFlagCompletionFunc("xml-content-name", cobra.NoFileCompletions); err != nil {
 		panic(err)
 	}
-	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.StrictMode, "xml-strict-mode", xml.ConfiguredXMLPreferences.StrictMode, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
-	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.KeepNamespace, "xml-keep-namespace", xml.ConfiguredXMLPreferences.KeepNamespace, "enables keeping namespace after parsing attributes")
-	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.UseRawToken, "xml-raw-token", xml.ConfiguredXMLPreferences.UseRawToken, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")
-	rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.ProcInstPrefix, "xml-proc-inst-prefix", xml.ConfiguredXMLPreferences.ProcInstPrefix, "prefix for xml processing instructions (e.g. <?xml version=\"1\"?>)")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.StrictMode, "xml-strict-mode", yqlib.ConfiguredXMLPreferences.StrictMode, "enables strict parsing of XML. See https://pkg.go.dev/encoding/xml for more details.")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.KeepNamespace, "xml-keep-namespace", yqlib.ConfiguredXMLPreferences.KeepNamespace, "enables keeping namespace after parsing attributes")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.UseRawToken, "xml-raw-token", yqlib.ConfiguredXMLPreferences.UseRawToken, "enables using RawToken method instead Token. Commonly disables namespace translations. See https://pkg.go.dev/encoding/xml#Decoder.RawToken for details.")
+	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.ProcInstPrefix, "xml-proc-inst-prefix", yqlib.ConfiguredXMLPreferences.ProcInstPrefix, "prefix for xml processing instructions (e.g. <?xml version=\"1\"?>)")
 	if err = rootCmd.RegisterFlagCompletionFunc("xml-proc-inst-prefix", cobra.NoFileCompletions); err != nil {
 		panic(err)
 	}
-	rootCmd.PersistentFlags().StringVar(&xml.ConfiguredXMLPreferences.DirectiveName, "xml-directive-name", xml.ConfiguredXMLPreferences.DirectiveName, "name for xml directives (e.g. <!DOCTYPE thing cat>)")
+	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredXMLPreferences.DirectiveName, "xml-directive-name", yqlib.ConfiguredXMLPreferences.DirectiveName, "name for xml directives (e.g. <!DOCTYPE thing cat>)")
 	if err = rootCmd.RegisterFlagCompletionFunc("xml-directive-name", cobra.NoFileCompletions); err != nil {
 		panic(err)
 	}
-	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", xml.ConfiguredXMLPreferences.SkipProcInst, "skip over process instructions (e.g. <?xml version=\"1\"?>)")
-	rootCmd.PersistentFlags().BoolVar(&xml.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", xml.ConfiguredXMLPreferences.SkipDirectives, "skip over directives (e.g. <!DOCTYPE thing cat>)")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipProcInst, "xml-skip-proc-inst", yqlib.ConfiguredXMLPreferences.SkipProcInst, "skip over process instructions (e.g. <?xml version=\"1\"?>)")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredXMLPreferences.SkipDirectives, "xml-skip-directives", yqlib.ConfiguredXMLPreferences.SkipDirectives, "skip over directives (e.g. <!DOCTYPE thing cat>)")
 
 	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredCsvPreferences.AutoParse, "csv-auto-parse", yqlib.ConfiguredCsvPreferences.AutoParse, "parse CSV YAML/JSON values")
 	rootCmd.PersistentFlags().Var(newRuneVar(&yqlib.ConfiguredCsvPreferences.Separator), "csv-separator", "CSV Separator character")
@@ -157,8 +155,8 @@ yq -P -oy sample.json
 	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.UnquotedKeys, "lua-unquoted", yqlib.ConfiguredLuaPreferences.UnquotedKeys, "output unquoted string keys (e.g. {foo=\"bar\"})")
 	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredLuaPreferences.Globals, "lua-globals", yqlib.ConfiguredLuaPreferences.Globals, "output keys as top-level global variables")
 
-	rootCmd.PersistentFlags().StringVar(&properties.ConfiguredPropertiesPreferences.KeyValueSeparator, "properties-separator", properties.ConfiguredPropertiesPreferences.KeyValueSeparator, "separator to use between keys and values")
-	rootCmd.PersistentFlags().BoolVar(&properties.ConfiguredPropertiesPreferences.UseArrayBrackets, "properties-array-brackets", properties.ConfiguredPropertiesPreferences.UseArrayBrackets, "use [x] in array paths (e.g. for SpringBoot)")
+	rootCmd.PersistentFlags().StringVar(&yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, "properties-separator", yqlib.ConfiguredPropertiesPreferences.KeyValueSeparator, "separator to use between keys and values")
+	rootCmd.PersistentFlags().BoolVar(&yqlib.ConfiguredPropertiesPreferences.UseArrayBrackets, "properties-array-brackets", yqlib.ConfiguredPropertiesPreferences.UseArrayBrackets, "use [x] in array paths (e.g. for SpringBoot)")
 
 	rootCmd.PersistentFlags().BoolVar(&yqlib.StringInterpolationEnabled, "string-interpolation", yqlib.StringInterpolationEnabled, "Toggles strings interpolation of \\(exp)")
 

cmd/utils.go

@@ -6,8 +6,6 @@ import (
 	"os"
 	"strings"
 
-	"github.com/mikefarah/yq/v4/pkg/properties"
-	"github.com/mikefarah/yq/v4/pkg/xml"
 	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"github.com/spf13/cobra"
 	"gopkg.in/op/go-logging.v1"
@@ -77,10 +75,6 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 				outputFormat = "yaml"
 			}
 		} else if isAutomaticOutputFormat() {
-			// automatic input worked, we can do it for output too unless specified
-			if inputFormat == "json" {
-				yqlib.GetLogger().Warning("JSON file output is now JSON by default (instead of yaml). Use '-oy' or '--output-format=yaml' for yaml output")
-			}
 			outputFormat = inputFormat
 		}
 	} else if isAutomaticOutputFormat() {
@@ -104,7 +98,7 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 	yqlib.GetLogger().Debug("Using output format %v", outputFormat)
 
 	if outputFormatType == yqlib.YamlFormat ||
-		outputFormatType == properties.PropertiesFormat {
+		outputFormatType == yqlib.PropertiesFormat {
 		unwrapScalar = true
 	}
 	if unwrapScalarFlag.IsExplicitlySet() {
@@ -150,12 +144,12 @@ func configureEncoder() (yqlib.Encoder, error) {
 	if err != nil {
 		return nil, err
 	}
-	xml.ConfiguredXMLPreferences.Indent = indent
+	yqlib.ConfiguredXMLPreferences.Indent = indent
 	yqlib.ConfiguredYamlPreferences.Indent = indent
 	yqlib.ConfiguredJSONPreferences.Indent = indent
 
 	yqlib.ConfiguredYamlPreferences.UnwrapScalar = unwrapScalar
-	properties.ConfiguredPropertiesPreferences.UnwrapScalar = unwrapScalar
+	yqlib.ConfiguredPropertiesPreferences.UnwrapScalar = unwrapScalar
 	yqlib.ConfiguredJSONPreferences.UnwrapScalar = unwrapScalar
 
 	yqlib.ConfiguredYamlPreferences.ColorsEnabled = colorsEnabled

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "v4.42.1"
+	Version = "v4.44.2"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

debian/changelog

@@ -1,210 +0,0 @@
-yq (4.16.2) focal; urgency=medium
-
-  * Fixed with semicolon space issue
-  * Updating with documentation
-  * Added STDIN example to the top
-  * minor readme cleanup
-  * Help text tweak
-  * Fixed docker timeout - simplify docker builds
-  * New release with docker build fixes
-  * Updating to go 1.17 to fix CVE #944
-  * Fix a typo in root.go
-  * Skip the tests if the nocheck Debian build option is specified
-  * Fixed select bug (#958)
-  * Sped up explode operator
-  * Slight performance improvement to context.ChildContext
-  * Speed up multiply
-  * Update README with recently added / changed options
-  * Make deepMatch report in linear time
-  * Removed leadingContentPreProcessing flag - header preprocessing is stable
-  * Revert "Removed leadingContentPreProcessing flag - header preprocessing is stable"
-  * Keep flag, it is needed in corner cases
-  * Updated Readme
-  * Man page
-  * Fixed expression parsing bug #970
-  * Bumping go-lang, docker versions
-  * Added test release flow
-  * Updated github action release to generate man page
-  * Bumping version
-  * Removing no longer needed github action
-  * Added decoder op
-  * Fixed newline handling when decoding/encoding
-  * Fixed newline handling in encoder/decoder
-  * Can specify indent in encode ops
-  * Added group_by operator
-  * Added flatten operator
-  * Fixed flatten error message
-  * Improving docs
-  * Split printer
-  * Refactored command logic
-  * Fix JSON encoding removing null #985
-  * Fixed acceptance tests
-  * gitbook
-  * Update document generation script
-  * Updating README
-  * Updating release instructions
-  * github action no longer uses data1.yml
-  * Create dependabot.yml
-  * Bump actions/create-release from 1.0.0 to 1.1.4
-  * Bump actions/setup-go from 1 to 2.1.4
-  * Bump github.com/goccy/go-yaml from 1.8.9 to 1.9.4
-  * Bump github.com/jinzhu/copier from 0.2.8 to 0.3.2
-  * Bump github.com/fatih/color from 1.10.0 to 1.13.0
-  * Bump github.com/spf13/cobra from 1.1.3 to 1.2.1
-  * Update dependabot.yml
-  * Update go.yml
-  * add build check to PRs
-  * Include secure as part of build process
-  * Fixing bad label in github action
-  * fixed printer test
-  * remove leading content indicator
-  * Fixed header preprocessing!
-  * lint : define golangci configuration file
-  * Update check.sh
-  * Load file acceptance test
-  * Minor improvement on handling front matter
-  * Improved load doc
-  * feature: detect MANPATh and install there
-  * Update install-man-page.sh
-  * simplify prod stage, move version label to action
-  * add labels, quote some values
-  * enable errorlint linter
-  * Added errorlint to devtools
-  * Added key operator
-  * Added more tests
-  * Fixing comments
-  * Attempt to fix golint problem
-  * Include version query for tools
-  * Clean up errored file?
-  * enable misspell linter
-  * updated readme
-  * update Golangci version to v1.43.0
-  * gci linter
-  * Better merge array by key example
-  * Added credit for merge by array example
-  * Better formatting of merge arrays example
-  * Better merge example
-  * Add accessor for the yq logger instance (#1013)
-  * Fixed collect op when working with multiple nodes
-  * Added map, map_values
-  * Add support for Podman as well as Docker (#1026)
-  * Bump github.com/jinzhu/copier from 0.3.2 to 0.3.4 (#1027)
-  * Added csv, tsv output formats
-  * Added encoder tests
-  * Cleanup test
-  * Fixed docker permission issue #1014
-  * Recording release notes for next release
-  * Assignment op no longer clobbers anchor (#1029)
-  * Added sort_by operator
-  * Improved error message
-  * Improved tips and tricks
-  * Report while filename failed to parse #1030
-  * Added script for extracting checksums
-  * Improved extract-checksum.sh
-  * Bump github.com/spf13/cobra from 1.2.1 to 1.3.0 (#1039)
-  * enable more linters (#1043)
-  * Bump Golang compiler #1037
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Tue, 21 Dec 2021 09:41:44 +0000
-
-yq (4.13.0) focal; urgency=medium
-
-  * New `with` operator for making multiple changes to a given path
-  * New `contains` operator, works like the `jq` equivalent
-  * Subtract operator now supports subtracting elements from arrays!
-  * Fixed Swapping values using variables #934
-  * Github Action now properly supports multiline output #936, thanks @pjxiao
-  * Fixed missing closing bracket validation #932
-  * Fix processing of hex numbers #929
-  * Fixed alternative and union operator issues #930
-  * Can now convert yaml to properties properties format (`-o=props`), See [docs](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties) for more info.
-  * Fixed document header/footer comment handling when merging (https://github.com/mikefarah/yq/issues/919)
-  * pretty print yaml 1.1 compatibility  (https://github.com/mikefarah/yq/issues/914)
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Thu, 16 Sep 2021 20:58:30 +0200
-
-yq (4.9.6) focal; urgency=medium
-
-  * Added darwin/arm64 build, thanks @alecthomas
-  * Incremented docker alpine base version, thanks @da6d6i7-bronga
-  * Bug fix: multiline expression
-  * Bug fix: special character
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Tue, 29 Jun 2021 21:32:14 +0200
-
-yq (3.3.2) focal; urgency=medium
-
-  * Bug fix: existStatus bug (#459)
-  * Automatically makes an os temp directory if it does not exist (#461)
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Fri, 07 Aug 2020 18:53:01 +0200
-
-yq (3.3-0) focal; urgency=medium
-
-  * You can control string styles (quotes) using the new --style flag
-  * String values now always have quotes when outputting to json
-  * Negative array indices now traverse the array backwards
-  * Added a --stripComments flag to print yaml without any comments
-  * Bumped go to version 1.14
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Thu, 30 Apr 2020 20:45:44 +0200
-
-yq (3.1-2) eoan; urgency=medium
-
-  * Bug fix: yq 3 was removing empty inline-style objects and arrays (#355)
-  * Bug fix: Merge option returned different output when switching order of 
-    merging files(#347)
-  * Bug fix: Add new object to existing array object was failing in 3.1.1 (#361)
-  * Bug fix: yq 3 empty keys did not allow merging of values (#356)
-  * Bug fix: keys quoted during merge (#363)
-  * Bug fix: Correct length with wc -l (#362)
-  * Bug fix: Write to empty document removed path (#359)
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Mon, 24 Feb 2020 20:31:58 +0100
-
-yq (3.1-1) eoan; urgency=medium
-
-  * Keeps yaml comments and formatting, can specify yaml tags when updating.
-  * Handles anchors
-  * Can print out matching paths and values when splatting
-  * JSON output works for all commands
-  * Yaml files with multiple documents are printed out as one JSON 
-    document per line.
-  * Deep splat (**) to match arbitrary paths
-  * Update scripts file format has changed to be more powerful
-  * Reading and splatting, matching results are printed once per line
-  * Bugfixing
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Tue, 11 Feb 2020 22:18:24 +0100
-
-yq (2.2-1) bionic; urgency=medium
-
-  * Added Windows support for the "--inplace" command flag
-  * Prefix now supports arrays
-  * Add prefix command
-  * Bump Alpine version to 3.8
-  * Improved docker build process
-  * Lint fixes
-  * Build support for all linux architectures supported by gox
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Sat, 19 Jan 2019 15:50:47 +0100
-
-yq (2.1-0) bionic; urgency=medium
-
-  * Ability to read multiple documents in a single file
-  * Ability to append list items instead of overwriting
-
- -- Roberto Mier Escandón <rmescandon@gmail.com>  Tue, 10 Jul 2018 14:02:42 +0200
-
-yq (2.0-0) bionic; urgency=medium
-
-  * Release 2.0.0
-
- -- Roberto Mier Escandón <rmescandon@gmail.com>  Wed, 20 Jun 2018 10:29:53 +0200
-
-yq (1.15-0) bionic; urgency=medium
-
-  * Release 1.15
-
- -- Roberto Mier Escandón <rmescandon@gmail.com>  Wed, 06 Jun 2018 11:32:03 +0200
-

debian/compat

@@ -1 +0,0 @@
-10

debian/control

@@ -1,22 +0,0 @@
-Source: yq
-Section: devel
-Priority: optional
-Maintainer: Roberto Mier Escandón <rmescandon@gmail.com>
-Build-Depends: debhelper (>=10),
-               golang-1.17-go,
-               pandoc,
-               rsync
-Standards-Version: 4.1.4
-Homepage: https://github.com/mikefarah/yq.git
-Vcs-Browser: https://github.com/mikefarah/yq.git
-Vcs-Git: https://github.com/mikefarah/yq.git
-XS-Go-Import-Path: github.com/mikefarah/yq
-XSBC-Original-Maintainer: Roberto Mier Escandón <rmescandon@gmail.com>
-
-Package: yq
-Architecture: any
-Depends: ${shlibs:Depends}, ${misc:Depends}
-Description: lightweight and portable command-line data file processor
- .
- The aim of the project is to be the 
- [jq](https://github.com/stedolan/jq) or sed of yaml files.

debian/copyright

@@ -1,24 +0,0 @@
-Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-Upstream-Name: yq
-Source: https://github.com/mikefarah/yq.git
-
-Files: *
-Copyright: 2017 Mike Farah
-License: Expat
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to deal
-    in the Software without restriction, including without limitation the rights
-    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-    copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
-    .
-    The above copyright notice and this permission notice shall be included in all
-    copies or substantial portions of the Software.
-    .
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-    SOFTWARE.

debian/gbp.conf

@@ -1,2 +0,0 @@
-[DEFAULT]
-pristine-tar = True

debian/rules

@@ -1,74 +0,0 @@
-#!/usr/bin/make -f
-#
-# Copyright (C) 2018-2021 Roberto Mier Escandón <rmescandon@gmail.com>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-PROJECT := yq
-OWNER := mikefarah
-REPO := github.com
-
-export DH_OPTIONS
-export DH_GOPKG := ${REPO}/${OWNER}/${PROJECT}
-export GOROOT := /usr/local/go
-export GOPATH := ${CURDIR}/_build
-export GOBIN := ${GOPATH}/bin
-export PATH := ${GOROOT}/bin:${GOBIN}:${PATH}
-export GOCACHE := /tmp/gocache
-export GOFLAGS := -mod=vendor
-export GO111MODULE := on
-
-SRCDIR := ${GOPATH}/src/${DH_GOPKG}
-DESTDIR := ${CURDIR}/debian/${PROJECT}
-BINDIR := /usr/bin
-MANDIR := /usr/share/man/man1/
-ASSETSDIR := /usr/share/${PROJECT}
-
-%:
-	dh $@
-
-override_dh_auto_build:
-	mkdir -p ${SRCDIR}
-	mkdir -p ${GOBIN}
-	# copy project to local srcdir to build from there
-	rsync -avz --progress --exclude=_build --exclude=debian --exclude=tmp. --exclude=go.mod --exclude=docs . $(SRCDIR)
-	# build go code
-	( \
-		cd ${SRCDIR} && \
-		go install -buildmode=pie ./... \
-	)
-
-	# build man page
-	( \
-		cd ${SRCDIR} && \
-		./scripts/generate-man-page-md.sh && \
-		./scripts/generate-man-page.sh \
-	)
-	
-override_dh_auto_test:
-ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
-	(cd ${SRCDIR} && go test -v ./...)
-endif
-
-override_dh_auto_install:
-	cp ${GOBIN}/yq ${DESTDIR}/${BINDIR}
-	cp -f ${SRCDIR}/LICENSE ${DESTDIR}/${ASSETSDIR}
-	chmod a+x ${DESTDIR}/${BINDIR}/yq
-
-	# man
-	mkdir -p "${DESTDIR}"/"${MANDIR}"
-	cp "${SRCDIR}"/yq.1 "${DESTDIR}"/"${MANDIR}" \
-
-override_dh_auto_clean:
-	dh_clean
-	rm -rf ${CURDIR}/_build

debian/source/format

@@ -1 +0,0 @@
-3.0 (native)

debian/yq.dirs

@@ -1,3 +0,0 @@
-usr/bin
-usr/share/yq
-usr/share/man/man1

go.mod

@@ -5,19 +5,19 @@ require (
 	github.com/alecthomas/participle/v2 v2.1.1
 	github.com/alecthomas/repr v0.4.0
 	github.com/dimchansky/utfbom v1.1.1
-	github.com/elliotchance/orderedmap v1.5.1
-	github.com/fatih/color v1.16.0
-	github.com/goccy/go-json v0.10.2
+	github.com/elliotchance/orderedmap v1.6.0
+	github.com/fatih/color v1.17.0
+	github.com/goccy/go-json v0.10.3
 	github.com/goccy/go-yaml v1.11.3
 	github.com/jinzhu/copier v0.4.0
 	github.com/magiconair/properties v1.8.7
-	github.com/pelletier/go-toml/v2 v2.1.1
+	github.com/pelletier/go-toml/v2 v2.2.2
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
 	github.com/yuin/gopher-lua v1.1.1
-	golang.org/x/net v0.22.0
-	golang.org/x/text v0.14.0
+	golang.org/x/net v0.26.0
+	golang.org/x/text v0.16.0
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -26,7 +26,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 )
 

go.sum

@@ -12,18 +12,18 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
 github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
-github.com/elliotchance/orderedmap v1.5.1 h1:G1X4PYlljzimbdQ3RXmtIZiQ9d6aRQ3sH1nzjq5mECE=
-github.com/elliotchance/orderedmap v1.5.1/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
-github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
-github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
+github.com/elliotchance/orderedmap v1.6.0 h1:xjn+kbbKXeDq6v9RVE+WYwRbYfAZKvlWfcJNxM8pvEw=
+github.com/elliotchance/orderedmap v1.6.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys=
+github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4=
+github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI=
 github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no=
 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA=
 github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE=
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
-github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
-github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
+github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
 github.com/goccy/go-yaml v1.11.3 h1:B3W9IdWbvrUu2OYQGwvU1nZtvMQJPBKgBUuweJjLj6I=
 github.com/goccy/go-yaml v1.11.3/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
@@ -43,8 +43,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
-github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
+github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
+github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -57,23 +57,25 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
 github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
-golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
-golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
+golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
+golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
+golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

pkg/properties/properties.go

@@ -1,45 +0,0 @@
-package properties
-
-import "github.com/mikefarah/yq/v4/pkg/yqlib"
-
-type PropertiesPreferences struct {
-	UnwrapScalar      bool
-	KeyValueSeparator string
-	UseArrayBrackets  bool
-}
-
-func NewDefaultPropertiesPreferences() PropertiesPreferences {
-	return PropertiesPreferences{
-		UnwrapScalar:      true,
-		KeyValueSeparator: " = ",
-		UseArrayBrackets:  false,
-	}
-}
-
-func (p *PropertiesPreferences) Copy() PropertiesPreferences {
-	return PropertiesPreferences{
-		UnwrapScalar:      p.UnwrapScalar,
-		KeyValueSeparator: p.KeyValueSeparator,
-		UseArrayBrackets:  p.UseArrayBrackets,
-	}
-}
-
-var PropertiesFormat = &yqlib.Format{"props", []string{"p", "properties"}, "properties",
-	func() yqlib.Encoder { return NewPropertiesEncoder(ConfiguredPropertiesPreferences) },
-	func() yqlib.Decoder { return NewPropertiesDecoder() },
-	nil,
-}
-
-var propertyYqRules = []*yqlib.ParticipleYqRule{
-	{"PropertiesDecode", `from_?props|@propsd`, yqlib.CreateDecodeOpYqAction(PropertiesFormat), 0},
-	{"PropsEncode", `to_?props|@props`, yqlib.CreateEncodeOpYqAction(PropertiesFormat, 2), 0},
-	{"LoadProperties", `load_?props`, yqlib.CreateLoadOpYqAction(NewPropertiesDecoder()), 0},
-}
-
-func RegisterPropertiesFormat() {
-	yqlib.RegisterFormat(PropertiesFormat)
-	yqlib.RegisterRules(propertyYqRules)
-
-}
-
-var ConfiguredPropertiesPreferences = NewDefaultPropertiesPreferences()

pkg/yqlib/all_at_once_evaluator.go

@@ -63,7 +63,7 @@ func (e *allAtOnceEvaluator) EvaluateFiles(expression string, filenames []string
 	}
 
 	if allDocuments.Len() == 0 {
-		candidateNode := CreateScalarNode(nil, "")
+		candidateNode := createScalarNode(nil, "")
 		allDocuments.PushBack(candidateNode)
 	}
 

pkg/yqlib/candidate_node.go

@@ -27,14 +27,14 @@ const (
 	FlowStyle
 )
 
-func CreateStringScalarNode(stringValue string) *CandidateNode {
+func createStringScalarNode(stringValue string) *CandidateNode {
 	var node = &CandidateNode{Kind: ScalarNode}
 	node.Value = stringValue
 	node.Tag = "!!str"
 	return node
 }
 
-func CreateScalarNode(value interface{}, stringValue string) *CandidateNode {
+func createScalarNode(value interface{}, stringValue string) *CandidateNode {
 	var node = &CandidateNode{Kind: ScalarNode}
 	node.Value = stringValue
 
@@ -219,7 +219,7 @@ func (n *CandidateNode) AddChild(rawChild *CandidateNode) {
 		value.Key.SetParent(n)
 	} else {
 		index := len(n.Content)
-		keyNode := CreateScalarNode(index, fmt.Sprintf("%v", index))
+		keyNode := createScalarNode(index, fmt.Sprintf("%v", index))
 		keyNode.SetParent(n)
 		value.Key = keyNode
 	}
@@ -243,7 +243,7 @@ func (n *CandidateNode) AddChildren(children []*CandidateNode) {
 
 func (n *CandidateNode) GetValueRep() (interface{}, error) {
 	log.Debugf("GetValueRep for %v value: %v", n.GetNicePath(), n.Value)
-	realTag := n.GuessTagFromCustomType()
+	realTag := n.guessTagFromCustomType()
 
 	switch realTag {
 	case "!!int":
@@ -261,17 +261,17 @@ func (n *CandidateNode) GetValueRep() (interface{}, error) {
 	return n.Value, nil
 }
 
-func (n *CandidateNode) GuessTagFromCustomType() string {
+func (n *CandidateNode) guessTagFromCustomType() string {
 	if strings.HasPrefix(n.Tag, "!!") {
 		return n.Tag
 	} else if n.Value == "" {
-		log.Debug("GuessTagFromCustomType: node has no value to guess the type with")
+		log.Debug("guessTagFromCustomType: node has no value to guess the type with")
 		return n.Tag
 	}
 	dataBucket, errorReading := parseSnippet(n.Value)
 
 	if errorReading != nil {
-		log.Debug("GuessTagFromCustomType: could not guess underlying tag type %v", errorReading)
+		log.Debug("guessTagFromCustomType: could not guess underlying tag type %v", errorReading)
 		return n.Tag
 	}
 	guessedTag := dataBucket.Tag
@@ -376,7 +376,7 @@ func (n *CandidateNode) UpdateFrom(other *CandidateNode, prefs assignPreferences
 	// if this is an empty map or empty array, use the style of other node.
 	if (n.Kind != ScalarNode && len(n.Content) == 0) ||
 		// if the tag has changed (e.g. from str to bool)
-		(n.GuessTagFromCustomType() != other.GuessTagFromCustomType()) {
+		(n.guessTagFromCustomType() != other.guessTagFromCustomType()) {
 		n.Style = other.Style
 	}
 
@@ -428,34 +428,3 @@ func (n *CandidateNode) UpdateAttributesFrom(other *CandidateNode, prefs assignP
 		n.LineComment = other.LineComment
 	}
 }
-
-func (n *CandidateNode) CleanHeadAndLineComment() string {
-	return n.CleanHeadComment() + n.CleanLineComment()
-}
-
-func (n *CandidateNode) CleanHeadComment() string {
-	return strings.Replace(n.HeadComment, "#", "", 1)
-}
-
-func (n *CandidateNode) CleanLineComment() string {
-	return strings.Replace(n.LineComment, "#", "", 1)
-}
-
-func (n *CandidateNode) CleanFootComment() string {
-	return strings.Replace(n.FootComment, "#", "", 1)
-}
-
-func (n *CandidateNode) ConvertKeysToStrings() {
-
-	if n.Kind == MappingNode {
-		for index, child := range n.Content {
-			if index%2 == 0 { // its a map key
-				child.Tag = "!!str"
-			}
-		}
-	}
-
-	for _, child := range n.Content {
-		child.ConvertKeysToStrings()
-	}
-}

pkg/yqlib/candidate_node_test.go

@@ -64,13 +64,13 @@ func TestCandidateNodeChildWhenParentUpdated(t *testing.T) {
 	test.AssertResultWithContext(t, uint(1), child.GetDocument(), "document index")
 }
 
-type CreateScalarNodeScenario struct {
+type createScalarNodeScenario struct {
 	value       interface{}
 	stringValue string
 	expectedTag string
 }
 
-var createScalarScenarios = []CreateScalarNodeScenario{
+var createScalarScenarios = []createScalarNodeScenario{
 	{
 		value:       "mike",
 		stringValue: "mike",
@@ -100,20 +100,20 @@ var createScalarScenarios = []CreateScalarNodeScenario{
 
 func TestCreateScalarNodeScenarios(t *testing.T) {
 	for _, tt := range createScalarScenarios {
-		actual := CreateScalarNode(tt.value, tt.stringValue)
+		actual := createScalarNode(tt.value, tt.stringValue)
 		test.AssertResultWithContext(t, tt.stringValue, actual.Value, fmt.Sprintf("Value for: Value: [%v], String: %v", tt.value, tt.stringValue))
 		test.AssertResultWithContext(t, tt.expectedTag, actual.Tag, fmt.Sprintf("Value for: Value: [%v], String: %v", tt.value, tt.stringValue))
 	}
 }
 
 func TestGetKeyForMapValue(t *testing.T) {
-	key := CreateStringScalarNode("yourKey")
+	key := createStringScalarNode("yourKey")
 	n := CandidateNode{Key: key, Value: "meow", document: 3}
 	test.AssertResult(t, "3 - yourKey", n.GetKey())
 }
 
 func TestGetKeyForMapKey(t *testing.T) {
-	key := CreateStringScalarNode("yourKey")
+	key := createStringScalarNode("yourKey")
 	key.IsMapKey = true
 	key.document = 3
 	test.AssertResult(t, "key-yourKey-3 - ", key.GetKey())
@@ -125,7 +125,7 @@ func TestGetKeyForValue(t *testing.T) {
 }
 
 func TestGetParsedKeyForMapKey(t *testing.T) {
-	key := CreateStringScalarNode("yourKey")
+	key := createStringScalarNode("yourKey")
 	key.IsMapKey = true
 	key.document = 3
 	test.AssertResult(t, "yourKey", key.getParsedKey())
@@ -137,13 +137,13 @@ func TestGetParsedKeyForLooseValue(t *testing.T) {
 }
 
 func TestGetParsedKeyForMapValue(t *testing.T) {
-	key := CreateStringScalarNode("yourKey")
+	key := createStringScalarNode("yourKey")
 	n := CandidateNode{Key: key, Value: "meow", document: 3}
 	test.AssertResult(t, "yourKey", n.getParsedKey())
 }
 
 func TestGetParsedKeyForArrayValue(t *testing.T) {
-	key := CreateScalarNode(4, "4")
+	key := createScalarNode(4, "4")
 	n := CandidateNode{Key: key, Value: "meow", document: 3}
 	test.AssertResult(t, 4, n.getParsedKey())
 }

pkg/yqlib/candidiate_node_json.go

@@ -96,7 +96,7 @@ func (o *CandidateNode) UnmarshalJSON(data []byte) error {
 
 			if child == nil {
 				// need to represent it as a null scalar
-				child = CreateScalarNode(nil, "null")
+				child = createScalarNode(nil, "null")
 			}
 			childKey := o.CreateChild()
 			childKey.Kind = ScalarNode

pkg/yqlib/chown_not_linux_os.go

@@ -7,6 +7,6 @@ import (
 	"os"
 )
 
-func changeOwner(info fs.FileInfo, file *os.File) error {
+func changeOwner(_ fs.FileInfo, _ *os.File) error {
 	return nil
 }

pkg/yqlib/csv_test.go

@@ -94,7 +94,7 @@ const tsvTestExpectedSimpleCsv = `i	like	csv
 because	excel	is	cool
 `
 
-var csvScenarios = []FormatScenario{
+var csvScenarios = []formatScenario{
 	{
 		description:  "Encode CSV simple",
 		input:        csvTestSimpleYaml,
@@ -210,7 +210,7 @@ var csvScenarios = []FormatScenario{
 	},
 }
 
-func testCSVScenario(t *testing.T, s FormatScenario) {
+func testCSVScenario(t *testing.T, s formatScenario) {
 	switch s.scenarioType {
 	case "encode-csv":
 		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewCsvEncoder(ConfiguredCsvPreferences)), s.description)
@@ -229,7 +229,7 @@ func testCSVScenario(t *testing.T, s FormatScenario) {
 	}
 }
 
-func documentCSVDecodeObjectScenario(w *bufio.Writer, s FormatScenario, formatType string) {
+func documentCSVDecodeObjectScenario(w *bufio.Writer, s formatScenario, formatType string) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -254,7 +254,7 @@ func documentCSVDecodeObjectScenario(w *bufio.Writer, s FormatScenario, formatTy
 	)
 }
 
-func documentCSVDecodeObjectNoAutoScenario(w *bufio.Writer, s FormatScenario, formatType string) {
+func documentCSVDecodeObjectNoAutoScenario(w *bufio.Writer, s formatScenario, formatType string) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -279,7 +279,7 @@ func documentCSVDecodeObjectNoAutoScenario(w *bufio.Writer, s FormatScenario, fo
 	)
 }
 
-func documentCSVEncodeScenario(w *bufio.Writer, s FormatScenario, formatType string) {
+func documentCSVEncodeScenario(w *bufio.Writer, s formatScenario, formatType string) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -312,7 +312,7 @@ func documentCSVEncodeScenario(w *bufio.Writer, s FormatScenario, formatType str
 	)
 }
 
-func documentCSVRoundTripScenario(w *bufio.Writer, s FormatScenario, formatType string) {
+func documentCSVRoundTripScenario(w *bufio.Writer, s formatScenario, formatType string) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -348,7 +348,7 @@ func documentCSVRoundTripScenario(w *bufio.Writer, s FormatScenario, formatType
 }
 
 func documentCSVScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(FormatScenario)
+	s := i.(formatScenario)
 	if s.skipDoc {
 		return
 	}

pkg/yqlib/data_tree_navigator.go

@@ -13,7 +13,6 @@ type DataTreeNavigator interface {
 	GetMatchingNodes(context Context, expressionNode *ExpressionNode) (Context, error)
 
 	DeeplyAssign(context Context, path []interface{}, rhsNode *CandidateNode) error
-	DeeplyAssignKey(context Context, path []interface{}, rhsNode *CandidateNode) error
 }
 
 type dataTreeNavigator struct {
@@ -24,14 +23,6 @@ func NewDataTreeNavigator() DataTreeNavigator {
 }
 
 func (d *dataTreeNavigator) DeeplyAssign(context Context, path []interface{}, rhsCandidateNode *CandidateNode) error {
-	return d.deeplyAssign(context, path, rhsCandidateNode, false)
-}
-
-func (d *dataTreeNavigator) DeeplyAssignKey(context Context, path []interface{}, rhsCandidateNode *CandidateNode) error {
-	return d.deeplyAssign(context, path, rhsCandidateNode, true)
-}
-
-func (d *dataTreeNavigator) deeplyAssign(context Context, path []interface{}, rhsCandidateNode *CandidateNode, targetKey bool) error {
 
 	assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
 
@@ -50,7 +41,7 @@ func (d *dataTreeNavigator) deeplyAssign(context Context, path []interface{}, rh
 
 	assignmentOpNode := &ExpressionNode{
 		Operation: assignmentOp,
-		LHS:       createTraversalTree(path, traversePreferences{}, targetKey),
+		LHS:       createTraversalTree(path, traversePreferences{}, false),
 		RHS:       &ExpressionNode{Operation: rhsOp},
 	}
 

pkg/yqlib/decoder_base64.go

@@ -68,5 +68,5 @@ func (dec *base64Decoder) Decode() (*CandidateNode, error) {
 		}
 	}
 	dec.readAnything = true
-	return CreateStringScalarNode(buf.String()), nil
+	return createStringScalarNode(buf.String()), nil
 }

pkg/yqlib/decoder_csv_object.go

@@ -32,7 +32,7 @@ func (dec *csvObjectDecoder) convertToNode(content string) *CandidateNode {
 	// if we're not auto-parsing, then we wont put in parsed objects or arrays
 	// but we still parse scalars
 	if err != nil || (!dec.prefs.AutoParse && node.Kind != ScalarNode) {
-		return CreateScalarNode(content, content)
+		return createScalarNode(content, content)
 	}
 	return node
 }
@@ -41,7 +41,7 @@ func (dec *csvObjectDecoder) createObject(headerRow []string, contentRow []strin
 	objectNode := &CandidateNode{Kind: MappingNode, Tag: "!!map"}
 
 	for i, header := range headerRow {
-		objectNode.AddKeyValueChild(CreateScalarNode(header, header), dec.convertToNode(contentRow[i]))
+		objectNode.AddKeyValueChild(createScalarNode(header, header), dec.convertToNode(contentRow[i]))
 	}
 	return objectNode
 }

pkg/yqlib/decoder_properties.go

@@ -1,4 +1,4 @@
-package properties
+package yqlib
 
 import (
 	"bytes"
@@ -8,17 +8,16 @@ import (
 	"strings"
 
 	"github.com/magiconair/properties"
-	"github.com/mikefarah/yq/v4/pkg/yqlib"
 )
 
 type propertiesDecoder struct {
 	reader   io.Reader
 	finished bool
-	d        yqlib.DataTreeNavigator
+	d        DataTreeNavigator
 }
 
-func NewPropertiesDecoder() yqlib.Decoder {
-	return &propertiesDecoder{d: yqlib.NewDataTreeNavigator(), finished: false}
+func NewPropertiesDecoder() Decoder {
+	return &propertiesDecoder{d: NewDataTreeNavigator(), finished: false}
 }
 
 func (dec *propertiesDecoder) Init(reader io.Reader) error {
@@ -48,18 +47,31 @@ func (dec *propertiesDecoder) processComment(c string) string {
 	return "# " + c
 }
 
-func (dec *propertiesDecoder) applyPropertyComments(context yqlib.Context, path []interface{}, comments []string) error {
-	rhsCandidateNode := &yqlib.CandidateNode{
+func (dec *propertiesDecoder) applyPropertyComments(context Context, path []interface{}, comments []string) error {
+	assignmentOp := &Operation{OperationType: assignOpType, Preferences: assignPreferences{}}
+
+	rhsCandidateNode := &CandidateNode{
 		Tag:         "!!str",
 		Value:       fmt.Sprintf("%v", path[len(path)-1]),
 		HeadComment: dec.processComment(strings.Join(comments, "\n")),
-		Kind:        yqlib.ScalarNode,
+		Kind:        ScalarNode,
 	}
-	rhsCandidateNode.Tag = rhsCandidateNode.GuessTagFromCustomType()
-	return dec.d.DeeplyAssignKey(context, path, rhsCandidateNode)
+
+	rhsCandidateNode.Tag = rhsCandidateNode.guessTagFromCustomType()
+
+	rhsOp := &Operation{OperationType: referenceOpType, CandidateNode: rhsCandidateNode}
+
+	assignmentOpNode := &ExpressionNode{
+		Operation: assignmentOp,
+		LHS:       createTraversalTree(path, traversePreferences{}, true),
+		RHS:       &ExpressionNode{Operation: rhsOp},
+	}
+
+	_, err := dec.d.GetMatchingNodes(context, assignmentOpNode)
+	return err
 }
 
-func (dec *propertiesDecoder) applyProperty(context yqlib.Context, properties *properties.Properties, key string) error {
+func (dec *propertiesDecoder) applyProperty(context Context, properties *properties.Properties, key string) error {
 	value, _ := properties.Get(key)
 	path := parsePropKey(key)
 
@@ -71,13 +83,13 @@ func (dec *propertiesDecoder) applyProperty(context yqlib.Context, properties *p
 		}
 	}
 
-	rhsNode := yqlib.CreateStringScalarNode(value)
-	rhsNode.Tag = rhsNode.GuessTagFromCustomType()
+	rhsNode := createStringScalarNode(value)
+	rhsNode.Tag = rhsNode.guessTagFromCustomType()
 
 	return dec.d.DeeplyAssign(context, path, rhsNode)
 }
 
-func (dec *propertiesDecoder) Decode() (*yqlib.CandidateNode, error) {
+func (dec *propertiesDecoder) Decode() (*CandidateNode, error) {
 	if dec.finished {
 		return nil, io.EOF
 	}
@@ -96,12 +108,12 @@ func (dec *propertiesDecoder) Decode() (*yqlib.CandidateNode, error) {
 	}
 	properties.DisableExpansion = true
 
-	rootMap := &yqlib.CandidateNode{
-		Kind: yqlib.MappingNode,
+	rootMap := &CandidateNode{
+		Kind: MappingNode,
 		Tag:  "!!map",
 	}
 
-	context := yqlib.Context{}
+	context := Context{}
 	context = context.SingleChildContext(rootMap)
 
 	for _, key := range properties.Keys() {

pkg/yqlib/decoder_test.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 )
 
-type FormatScenario struct {
+type formatScenario struct {
 	input          string
 	indent         int
 	expression     string
@@ -19,7 +19,7 @@ type FormatScenario struct {
 	expectedError  string
 }
 
-func processFormatScenario(s FormatScenario, decoder Decoder, encoder Encoder) (string, error) {
+func processFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) (string, error) {
 	var output bytes.Buffer
 	writer := bufio.NewWriter(&output)
 
@@ -64,7 +64,7 @@ func processFormatScenario(s FormatScenario, decoder Decoder, encoder Encoder) (
 	return output.String(), nil
 }
 
-func mustProcessFormatScenario(s FormatScenario, decoder Decoder, encoder Encoder) string {
+func mustProcessFormatScenario(s formatScenario, decoder Decoder, encoder Encoder) string {
 
 	result, err := processFormatScenario(s, decoder, encoder)
 	if err != nil {

pkg/yqlib/decoder_toml.go

@@ -145,30 +145,30 @@ func (dec *tomlDecoder) createArray(tomlNode *toml.Node) (*CandidateNode, error)
 
 func (dec *tomlDecoder) createStringScalar(tomlNode *toml.Node) (*CandidateNode, error) {
 	content := string(tomlNode.Data)
-	return CreateScalarNode(content, content), nil
+	return createScalarNode(content, content), nil
 }
 
 func (dec *tomlDecoder) createBoolScalar(tomlNode *toml.Node) (*CandidateNode, error) {
 	content := string(tomlNode.Data)
-	return CreateScalarNode(content == "true", content), nil
+	return createScalarNode(content == "true", content), nil
 }
 
 func (dec *tomlDecoder) createIntegerScalar(tomlNode *toml.Node) (*CandidateNode, error) {
 	content := string(tomlNode.Data)
 	_, num, err := parseInt64(content)
-	return CreateScalarNode(num, content), err
+	return createScalarNode(num, content), err
 }
 
 func (dec *tomlDecoder) createDateTimeScalar(tomlNode *toml.Node) (*CandidateNode, error) {
 	content := string(tomlNode.Data)
 	val, err := parseDateTime(time.RFC3339, content)
-	return CreateScalarNode(val, content), err
+	return createScalarNode(val, content), err
 }
 
 func (dec *tomlDecoder) createFloatScalar(tomlNode *toml.Node) (*CandidateNode, error) {
 	content := string(tomlNode.Data)
 	num, err := strconv.ParseFloat(content, 64)
-	return CreateScalarNode(num, content), err
+	return createScalarNode(num, content), err
 }
 
 func (dec *tomlDecoder) decodeNode(tomlNode *toml.Node) (*CandidateNode, error) {

pkg/yqlib/decoder_uri.go

@@ -48,5 +48,5 @@ func (dec *uriDecoder) Decode() (*CandidateNode, error) {
 		return nil, err
 	}
 	dec.readAnything = true
-	return CreateStringScalarNode(newValue), nil
+	return createStringScalarNode(newValue), nil
 }

pkg/yqlib/decoder_xml.go

@@ -1,6 +1,6 @@
 //go:build !yq_noxml
 
-package xml
+package yqlib
 
 import (
 	"encoding/xml"
@@ -11,7 +11,6 @@ import (
 	"strings"
 	"unicode"
 
-	"github.com/mikefarah/yq/v4/pkg/yqlib"
 	"golang.org/x/net/html/charset"
 )
 
@@ -22,7 +21,7 @@ type xmlDecoder struct {
 	prefs        XmlPreferences
 }
 
-func NewXMLDecoder(prefs XmlPreferences) yqlib.Decoder {
+func NewXMLDecoder(prefs XmlPreferences) Decoder {
 	return &xmlDecoder{
 		finished: false,
 		prefs:    prefs,
@@ -36,8 +35,8 @@ func (dec *xmlDecoder) Init(reader io.Reader) error {
 	return nil
 }
 
-func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*yqlib.CandidateNode, error) {
-	yamlNode := &yqlib.CandidateNode{Kind: yqlib.SequenceNode, Tag: "!!seq"}
+func (dec *xmlDecoder) createSequence(nodes []*xmlNode) (*CandidateNode, error) {
+	yamlNode := &CandidateNode{Kind: SequenceNode, Tag: "!!seq"}
 	for _, child := range nodes {
 		yamlChild, err := dec.convertToYamlNode(child)
 		if err != nil {
@@ -64,14 +63,14 @@ func (dec *xmlDecoder) processComment(c string) string {
 	return replacement
 }
 
-func (dec *xmlDecoder) createMap(n *xmlNode) (*yqlib.CandidateNode, error) {
-	yqlib.GetLogger().Debug("createMap: headC: %v, lineC: %v, footC: %v", n.HeadComment, n.LineComment, n.FootComment)
-	yamlNode := &yqlib.CandidateNode{Kind: yqlib.MappingNode, Tag: "!!map"}
+func (dec *xmlDecoder) createMap(n *xmlNode) (*CandidateNode, error) {
+	log.Debug("createMap: headC: %v, lineC: %v, footC: %v", n.HeadComment, n.LineComment, n.FootComment)
+	yamlNode := &CandidateNode{Kind: MappingNode, Tag: "!!map"}
 
 	if len(n.Data) > 0 {
-		yqlib.GetLogger().Debugf("creating content node for map: %v", dec.prefs.ContentName)
+		log.Debugf("creating content node for map: %v", dec.prefs.ContentName)
 		label := dec.prefs.ContentName
-		labelNode := yqlib.CreateScalarNode(label, label)
+		labelNode := createScalarNode(label, label)
 		labelNode.HeadComment = dec.processComment(n.HeadComment)
 		labelNode.LineComment = dec.processComment(n.LineComment)
 		labelNode.FootComment = dec.processComment(n.FootComment)
@@ -81,19 +80,19 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yqlib.CandidateNode, error) {
 	for i, keyValuePair := range n.Children {
 		label := keyValuePair.K
 		children := keyValuePair.V
-		labelNode := yqlib.CreateScalarNode(label, label)
-		var valueNode *yqlib.CandidateNode
+		labelNode := createScalarNode(label, label)
+		var valueNode *CandidateNode
 		var err error
 
 		if i == 0 {
-			yqlib.GetLogger().Debugf("head comment here")
+			log.Debugf("head comment here")
 			labelNode.HeadComment = dec.processComment(n.HeadComment)
 
 		}
-		yqlib.GetLogger().Debugf("label=%v, i=%v, keyValuePair.FootComment: %v", label, i, keyValuePair.FootComment)
+		log.Debugf("label=%v, i=%v, keyValuePair.FootComment: %v", label, i, keyValuePair.FootComment)
 		labelNode.FootComment = dec.processComment(keyValuePair.FootComment)
 
-		yqlib.GetLogger().Debug("len of children in %v is %v", label, len(children))
+		log.Debug("len of children in %v is %v", label, len(children))
 		if len(children) > 1 {
 			valueNode, err = dec.createSequence(children)
 			if err != nil {
@@ -106,7 +105,7 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yqlib.CandidateNode, error) {
 			if len(children[0].Children) == 0 && children[0].HeadComment != "" {
 				if len(children[0].Data) > 0 {
 
-					yqlib.GetLogger().Debug("scalar comment hack, currentlabel [%v]", labelNode.HeadComment)
+					log.Debug("scalar comment hack, currentlabel [%v]", labelNode.HeadComment)
 					labelNode.HeadComment = joinComments([]string{labelNode.HeadComment, strings.TrimSpace(children[0].HeadComment)}, "\n")
 					children[0].HeadComment = ""
 				} else {
@@ -126,33 +125,33 @@ func (dec *xmlDecoder) createMap(n *xmlNode) (*yqlib.CandidateNode, error) {
 	return yamlNode, nil
 }
 
-func (dec *xmlDecoder) createValueNodeFromData(values []string) *yqlib.CandidateNode {
+func (dec *xmlDecoder) createValueNodeFromData(values []string) *CandidateNode {
 	switch len(values) {
 	case 0:
-		return yqlib.CreateScalarNode(nil, "")
+		return createScalarNode(nil, "")
 	case 1:
-		return yqlib.CreateScalarNode(values[0], values[0])
+		return createScalarNode(values[0], values[0])
 	default:
-		content := make([]*yqlib.CandidateNode, 0)
+		content := make([]*CandidateNode, 0)
 		for _, value := range values {
-			content = append(content, yqlib.CreateScalarNode(value, value))
+			content = append(content, createScalarNode(value, value))
 		}
-		return &yqlib.CandidateNode{
-			Kind:    yqlib.SequenceNode,
+		return &CandidateNode{
+			Kind:    SequenceNode,
 			Tag:     "!!seq",
 			Content: content,
 		}
 	}
 }
 
-func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yqlib.CandidateNode, error) {
+func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*CandidateNode, error) {
 	if len(n.Children) > 0 {
 		return dec.createMap(n)
 	}
 
 	scalar := dec.createValueNodeFromData(n.Data)
 
-	yqlib.GetLogger().Debug("scalar (%v), headC: %v, lineC: %v, footC: %v", scalar.Tag, n.HeadComment, n.LineComment, n.FootComment)
+	log.Debug("scalar (%v), headC: %v, lineC: %v, footC: %v", scalar.Tag, n.HeadComment, n.LineComment, n.FootComment)
 	scalar.HeadComment = dec.processComment(n.HeadComment)
 	scalar.LineComment = dec.processComment(n.LineComment)
 	if scalar.Tag == "!!seq" {
@@ -165,7 +164,7 @@ func (dec *xmlDecoder) convertToYamlNode(n *xmlNode) (*yqlib.CandidateNode, erro
 	return scalar, nil
 }
 
-func (dec *xmlDecoder) Decode() (*yqlib.CandidateNode, error) {
+func (dec *xmlDecoder) Decode() (*CandidateNode, error) {
 	if dec.finished {
 		return nil, io.EOF
 	}
@@ -212,17 +211,17 @@ func (n *xmlNode) AddChild(s string, c *xmlNode) {
 	if n.Children == nil {
 		n.Children = make([]*xmlChildrenKv, 0)
 	}
-	yqlib.GetLogger().Debug("looking for %s", s)
+	log.Debug("looking for %s", s)
 	// see if we can find an existing entry to add to
 	for _, childEntry := range n.Children {
 		if childEntry.K == s {
-			yqlib.GetLogger().Debug("found it, appending an entry%s", s)
+			log.Debug("found it, appending an entry%s", s)
 			childEntry.V = append(childEntry.V, c)
-			yqlib.GetLogger().Debug("yay len of children in %v is %v", s, len(childEntry.V))
+			log.Debug("yay len of children in %v is %v", s, len(childEntry.V))
 			return
 		}
 	}
-	yqlib.GetLogger().Debug("not there, making a new one %s", s)
+	log.Debug("not there, making a new one %s", s)
 	n.Children = append(n.Children, &xmlChildrenKv{K: s, V: []*xmlNode{c}})
 }
 
@@ -268,7 +267,7 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
 
 		switch se := t.(type) {
 		case xml.StartElement:
-			yqlib.GetLogger().Debug("start element %v", se.Name.Local)
+			log.Debug("start element %v", se.Name.Local)
 			elem.state = "started"
 			// Build new a new current element and link it to its parent
 			elem = &element{
@@ -297,14 +296,14 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
 			if len(newBit) > 0 {
 				elem.n.Data = append(elem.n.Data, newBit)
 				elem.state = "chardata"
-				yqlib.GetLogger().Debug("chardata [%v] for %v", elem.n.Data, elem.label)
+				log.Debug("chardata [%v] for %v", elem.n.Data, elem.label)
 			}
 		case xml.EndElement:
 			if elem == nil {
-				yqlib.GetLogger().Debug("no element, probably bad xml")
+				log.Debug("no element, probably bad xml")
 				continue
 			}
-			yqlib.GetLogger().Debug("end element %v", elem.label)
+			log.Debug("end element %v", elem.label)
 			elem.state = "finished"
 			// And add it to its parent list
 			if elem.parent != nil {
@@ -320,10 +319,10 @@ func (dec *xmlDecoder) decodeXML(root *xmlNode) error {
 				applyFootComment(elem, commentStr)
 
 			} else if elem.state == "chardata" {
-				yqlib.GetLogger().Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
+				log.Debug("got a line comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
 				elem.n.LineComment = joinComments([]string{elem.n.LineComment, commentStr}, " ")
 			} else {
-				yqlib.GetLogger().Debug("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
+				log.Debug("got a head comment for (%v) %v: [%v]", elem.state, elem.label, commentStr)
 				elem.n.HeadComment = joinComments([]string{elem.n.HeadComment, commentStr}, " ")
 			}
 
@@ -348,7 +347,7 @@ func applyFootComment(elem *element, commentStr string) {
 	if len(elem.n.Children) > 0 {
 		lastChildIndex := len(elem.n.Children) - 1
 		childKv := elem.n.Children[lastChildIndex]
-		yqlib.GetLogger().Debug("got a foot comment, putting on last child for %v: [%v]", childKv.K, commentStr)
+		log.Debug("got a foot comment, putting on last child for %v: [%v]", childKv.K, commentStr)
 		// if it's an array of scalars, put the foot comment on the scalar itself
 		if len(childKv.V) > 0 && len(childKv.V[0].Children) == 0 {
 			nodeToUpdate := childKv.V[len(childKv.V)-1]
@@ -357,7 +356,7 @@ func applyFootComment(elem *element, commentStr string) {
 			childKv.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ")
 		}
 	} else {
-		yqlib.GetLogger().Debug("got a foot comment for %v: [%v]", elem.label, commentStr)
+		log.Debug("got a foot comment for %v: [%v]", elem.label, commentStr)
 		elem.n.FootComment = joinComments([]string{elem.n.FootComment, commentStr}, " ")
 	}
 }

pkg/yqlib/decoder_yaml.go

@@ -151,7 +151,7 @@ func (dec *yamlDecoder) Decode() (*CandidateNode, error) {
 }
 
 func (dec *yamlDecoder) blankNodeWithComment() *CandidateNode {
-	node := CreateScalarNode(nil, "")
+	node := createScalarNode(nil, "")
 	node.LeadingContent = dec.leadingContent
 	return node
 }

pkg/yqlib/doc/operators/headers/max.md

@@ -0,0 +1,3 @@
+# Max
+
+Computes the maximum among an incoming sequence of scalar values.

pkg/yqlib/doc/operators/headers/min.md

@@ -0,0 +1,3 @@
+# Min
+
+Computes the minimum among an incoming sequence of scalar values.

pkg/yqlib/doc/operators/headers/omit.md

@@ -0,0 +1,3 @@
+# Omit
+
+Works like `pick`, but instead you specify the keys/indices that you _don't_ want included.

pkg/yqlib/doc/operators/headers/pivot.md

@@ -0,0 +1,3 @@
+# Pivot
+
+Emulates the `PIVOT` function supported by several popular RDBMS systems.

pkg/yqlib/doc/operators/headers/slice-array.md

@@ -2,4 +2,4 @@
 
 The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.
 
-You may leave out the first or second number, which will will refer to the start or end of the array respectively.
+You may leave out the first or second number, which will refer to the start or end of the array respectively.

pkg/yqlib/doc/operators/max.md

@@ -0,0 +1,51 @@
+# Max
+
+Computes the maximum among an incoming sequence of scalar values.
+
+## Maximum int
+Given a sample.yml file of:
+```yaml
+- 99
+- 16
+- 12
+- 6
+- 66
+```
+then
+```bash
+yq 'max' sample.yml
+```
+will output
+```yaml
+99
+```
+
+## Maximum string
+Given a sample.yml file of:
+```yaml
+- foo
+- bar
+- baz
+```
+then
+```bash
+yq 'max' sample.yml
+```
+will output
+```yaml
+foo
+```
+
+## Maximum of empty
+Given a sample.yml file of:
+```yaml
+[]
+```
+then
+```bash
+yq 'max' sample.yml
+```
+will output
+```yaml
+```
+

pkg/yqlib/doc/operators/min.md

@@ -0,0 +1,51 @@
+# Min
+
+Computes the minimum among an incoming sequence of scalar values.
+
+## Minimum int
+Given a sample.yml file of:
+```yaml
+- 99
+- 16
+- 12
+- 6
+- 66
+```
+then
+```bash
+yq 'min' sample.yml
+```
+will output
+```yaml
+6
+```
+
+## Minimum string
+Given a sample.yml file of:
+```yaml
+- foo
+- bar
+- baz
+```
+then
+```bash
+yq 'min' sample.yml
+```
+will output
+```yaml
+bar
+```
+
+## Minimum of empty
+Given a sample.yml file of:
+```yaml
+[]
+```
+then
+```bash
+yq 'min' sample.yml
+```
+will output
+```yaml
+```
+

pkg/yqlib/doc/operators/multiply-merge.md

@@ -55,6 +55,62 @@ a: 12
 b: 4
 ```
 
+## Multiply string node X int
+Given a sample.yml file of:
+```yaml
+b: banana
+```
+then
+```bash
+yq '.b * 4' sample.yml
+```
+will output
+```yaml
+bananabananabananabanana
+```
+
+## Multiply int X string node
+Given a sample.yml file of:
+```yaml
+b: banana
+```
+then
+```bash
+yq '4 * .b' sample.yml
+```
+will output
+```yaml
+bananabananabananabanana
+```
+
+## Multiply string X int node
+Given a sample.yml file of:
+```yaml
+n: 4
+```
+then
+```bash
+yq '"banana" * .n' sample.yml
+```
+will output
+```yaml
+bananabananabananabanana
+```
+
+## Multiply int node X string
+Given a sample.yml file of:
+```yaml
+n: 4
+```
+then
+```bash
+yq '.n * "banana"' sample.yml
+```
+will output
+```yaml
+bananabananabananabanana
+```
+
 ## Merge objects together, returning merged result only
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/operators/omit.md

@@ -0,0 +1,44 @@
+# Omit
+
+Works like `pick`, but instead you specify the keys/indices that you _don't_ want included.
+
+## Omit keys from map
+Note that non existent keys are skipped.
+
+Given a sample.yml file of:
+```yaml
+myMap:
+  cat: meow
+  dog: bark
+  thing: hamster
+  hamster: squeak
+```
+then
+```bash
+yq '.myMap |= omit(["hamster", "cat", "goat"])' sample.yml
+```
+will output
+```yaml
+myMap:
+  dog: bark
+  thing: hamster
+```
+
+## Omit indices from array
+Note that non existent indices are skipped.
+
+Given a sample.yml file of:
+```yaml
+- cat
+- leopard
+- lion
+```
+then
+```bash
+yq 'omit([2, 0, 734, -5])' sample.yml
+```
+will output
+```yaml
+- leopard
+```
+

pkg/yqlib/doc/operators/pivot.md

@@ -0,0 +1,117 @@
+# Pivot
+
+Emulates the `PIVOT` function supported by several popular RDBMS systems.
+
+## Pivot a sequence of sequences
+Given a sample.yml file of:
+```yaml
+- - foo
+  - bar
+  - baz
+- - sis
+  - boom
+  - bah
+```
+then
+```bash
+yq 'pivot' sample.yml
+```
+will output
+```yaml
+- - foo
+  - sis
+- - bar
+  - boom
+- - baz
+  - bah
+```
+
+## Pivot sequence of heterogeneous sequences
+Missing values are "padded" to null.
+
+Given a sample.yml file of:
+```yaml
+- - foo
+  - bar
+  - baz
+- - sis
+  - boom
+  - bah
+  - blah
+```
+then
+```bash
+yq 'pivot' sample.yml
+```
+will output
+```yaml
+- - foo
+  - sis
+- - bar
+  - boom
+- - baz
+  - bah
+- -
+  - blah
+```
+
+## Pivot sequence of maps
+Given a sample.yml file of:
+```yaml
+- foo: a
+  bar: b
+  baz: c
+- foo: x
+  bar: y
+  baz: z
+```
+then
+```bash
+yq 'pivot' sample.yml
+```
+will output
+```yaml
+foo:
+  - a
+  - x
+bar:
+  - b
+  - y
+baz:
+  - c
+  - z
+```
+
+## Pivot sequence of heterogeneous maps
+Missing values are "padded" to null.
+
+Given a sample.yml file of:
+```yaml
+- foo: a
+  bar: b
+  baz: c
+- foo: x
+  bar: y
+  baz: z
+  what: ever
+```
+then
+```bash
+yq 'pivot' sample.yml
+```
+will output
+```yaml
+foo:
+  - a
+  - x
+bar:
+  - b
+  - y
+baz:
+  - c
+  - z
+what:
+  -
+  - ever
+```
+

pkg/yqlib/doc/operators/slice-array.md

@@ -2,7 +2,7 @@
 
 The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.
 
-You may leave out the first or second number, which will will refer to the start or end of the array respectively.
+You may leave out the first or second number, which will refer to the start or end of the array respectively.
 
 ## Slicing arrays
 Given a sample.yml file of:

pkg/yqlib/doc/operators/unique.md

@@ -63,7 +63,29 @@ will output
 - ~
 ```
 
-## Unique array object fields
+## Unique array objects
+Given a sample.yml file of:
+```yaml
+- name: harry
+  pet: cat
+- name: billy
+  pet: dog
+- name: harry
+  pet: cat
+```
+then
+```bash
+yq 'unique' sample.yml
+```
+will output
+```yaml
+- name: harry
+  pet: cat
+- name: billy
+  pet: dog
+```
+
+## Unique array of objects by a field
 Given a sample.yml file of:
 ```yaml
 - name: harry
@@ -85,3 +107,25 @@ will output
   pet: dog
 ```
 
+## Unique array of arrays
+Given a sample.yml file of:
+```yaml
+- - cat
+  - dog
+- - cat
+  - sheep
+- - cat
+  - dog
+```
+then
+```bash
+yq 'unique' sample.yml
+```
+will output
+```yaml
+- - cat
+  - dog
+- - cat
+  - sheep
+```
+

pkg/yqlib/doc/usage/properties.md

@@ -230,6 +230,23 @@ person:
     - pizza
 ```
 
+## Decode properties: numbers
+All values are assumed to be strings when parsing properties, but you can use the `from_yaml` operator on all the strings values to autoparse into the correct type.
+
+Given a sample.properties file of:
+```properties
+a.b = 10
+```
+then
+```bash
+yq -p=props ' (.. | select(tag == "!!str")) |= from_yaml' sample.properties
+```
+will output
+```yaml
+a:
+  b: 10
+```
+
 ## Decode properties - array should be a map
 If you have a numeric map key in your property files, use array_to_map to convert them to maps.
 

pkg/yqlib/encoder.go

@@ -10,3 +10,18 @@ type Encoder interface {
 	PrintLeadingContent(writer io.Writer, content string) error
 	CanHandleAliases() bool
 }
+
+func mapKeysToStrings(node *CandidateNode) {
+
+	if node.Kind == MappingNode {
+		for index, child := range node.Content {
+			if index%2 == 0 { // its a map key
+				child.Tag = "!!str"
+			}
+		}
+	}
+
+	for _, child := range node.Content {
+		mapKeysToStrings(child)
+	}
+}

pkg/yqlib/encoder_base64.go

@@ -27,7 +27,7 @@ func (e *base64Encoder) PrintLeadingContent(_ io.Writer, _ string) error {
 }
 
 func (e *base64Encoder) Encode(writer io.Writer, node *CandidateNode) error {
-	if node.GuessTagFromCustomType() != "!!str" {
+	if node.guessTagFromCustomType() != "!!str" {
 		return fmt.Errorf("cannot encode %v as base64, can only operate on strings", node.Tag)
 	}
 	_, err := writer.Write([]byte(e.encoding.EncodeToString([]byte(node.Value))))

pkg/yqlib/encoder_csv.go

@@ -65,7 +65,7 @@ func (e *csvEncoder) createChildRow(child *CandidateNode, headers []*CandidateNo
 	childRow := make([]*CandidateNode, 0)
 	for _, header := range headers {
 		keyIndex := findKeyInMap(child, header)
-		value := CreateScalarNode(nil, "")
+		value := createScalarNode(nil, "")
 		if keyIndex != -1 {
 			value = child.Content[keyIndex+1]
 		}
@@ -102,7 +102,7 @@ func (e *csvEncoder) encodeObjects(csvWriter *csv.Writer, content []*CandidateNo
 
 func (e *csvEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 	if node.Kind == ScalarNode {
-		return WriteString(writer, node.Value+"\n")
+		return writeString(writer, node.Value+"\n")
 	}
 
 	csvWriter := csv.NewWriter(writer)

pkg/yqlib/encoder_json.go

@@ -41,7 +41,7 @@ func (je *jsonEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 	log.Debugf("kids %v", len(node.Content))
 
 	if node.Kind == ScalarNode && je.prefs.UnwrapScalar {
-		return WriteString(writer, node.Value+"\n")
+		return writeString(writer, node.Value+"\n")
 	}
 
 	destination := writer

pkg/yqlib/encoder_lua.go

@@ -86,15 +86,15 @@ func (le *luaEncoder) encodeString(writer io.Writer, node *CandidateNode) error
 	case LiteralStyle, FoldedStyle, FlowStyle:
 		for i := 0; i < 10; i++ {
 			if !strings.Contains(node.Value, "]"+strings.Repeat("=", i)+"]") {
-				err := WriteString(writer, "["+strings.Repeat("=", i)+"[\n")
+				err := writeString(writer, "["+strings.Repeat("=", i)+"[\n")
 				if err != nil {
 					return err
 				}
-				err = WriteString(writer, node.Value)
+				err = writeString(writer, node.Value)
 				if err != nil {
 					return err
 				}
-				return WriteString(writer, "]"+strings.Repeat("=", i)+"]")
+				return writeString(writer, "]"+strings.Repeat("=", i)+"]")
 			}
 		}
 	case SingleQuotedStyle:
@@ -102,22 +102,22 @@ func (le *luaEncoder) encodeString(writer io.Writer, node *CandidateNode) error
 
 		// fallthrough to regular ol' string
 	}
-	return WriteString(writer, quote+le.escape.Replace(node.Value)+quote)
+	return writeString(writer, quote+le.escape.Replace(node.Value)+quote)
 }
 
 func (le *luaEncoder) writeIndent(writer io.Writer) error {
 	if le.indentStr == "" {
 		return nil
 	}
-	err := WriteString(writer, "\n")
+	err := writeString(writer, "\n")
 	if err != nil {
 		return err
 	}
-	return WriteString(writer, strings.Repeat(le.indentStr, le.indent))
+	return writeString(writer, strings.Repeat(le.indentStr, le.indent))
 }
 
 func (le *luaEncoder) encodeArray(writer io.Writer, node *CandidateNode) error {
-	err := WriteString(writer, "{")
+	err := writeString(writer, "{")
 	if err != nil {
 		return err
 	}
@@ -131,13 +131,13 @@ func (le *luaEncoder) encodeArray(writer io.Writer, node *CandidateNode) error {
 		if err != nil {
 			return err
 		}
-		err = WriteString(writer, ",")
+		err = writeString(writer, ",")
 		if err != nil {
 			return err
 		}
 		if child.LineComment != "" {
 			sansPrefix, _ := strings.CutPrefix(child.LineComment, "#")
-			err = WriteString(writer, " --"+sansPrefix)
+			err = writeString(writer, " --"+sansPrefix)
 			if err != nil {
 				return err
 			}
@@ -150,7 +150,7 @@ func (le *luaEncoder) encodeArray(writer io.Writer, node *CandidateNode) error {
 			return err
 		}
 	}
-	return WriteString(writer, "}")
+	return writeString(writer, "}")
 }
 
 func needsQuoting(s string) bool {
@@ -181,7 +181,7 @@ func needsQuoting(s string) bool {
 
 func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bool) error {
 	if !global {
-		err := WriteString(writer, "{")
+		err := writeString(writer, "{")
 		if err != nil {
 			return err
 		}
@@ -194,7 +194,7 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo
 			if err != nil {
 				return err
 			}
-			err = WriteString(writer, ";")
+			err = writeString(writer, ";")
 			if err != nil {
 				return err
 			}
@@ -207,19 +207,19 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo
 				}
 			}
 			if (le.unquoted || global) && child.Tag == "!!str" && !needsQuoting(child.Value) {
-				err := WriteString(writer, child.Value+" = ")
+				err := writeString(writer, child.Value+" = ")
 				if err != nil {
 					return err
 				}
 			} else {
 				if global {
 					// This only works in Lua 5.2+
-					err := WriteString(writer, "_ENV")
+					err := writeString(writer, "_ENV")
 					if err != nil {
 						return err
 					}
 				}
-				err := WriteString(writer, "[")
+				err := writeString(writer, "[")
 				if err != nil {
 					return err
 				}
@@ -227,7 +227,7 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo
 				if err != nil {
 					return err
 				}
-				err = WriteString(writer, "] = ")
+				err = writeString(writer, "] = ")
 				if err != nil {
 					return err
 				}
@@ -235,7 +235,7 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo
 		}
 		if child.LineComment != "" {
 			sansPrefix, _ := strings.CutPrefix(child.LineComment, "#")
-			err := WriteString(writer, strings.Repeat(" ", i%2)+"--"+sansPrefix)
+			err := writeString(writer, strings.Repeat(" ", i%2)+"--"+sansPrefix)
 			if err != nil {
 				return err
 			}
@@ -249,7 +249,7 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo
 		}
 	}
 	if global {
-		return WriteString(writer, "\n")
+		return writeString(writer, "\n")
 	}
 	le.indent--
 	if len(node.Content) != 0 {
@@ -258,7 +258,7 @@ func (le *luaEncoder) encodeMap(writer io.Writer, node *CandidateNode, global bo
 			return err
 		}
 	}
-	return WriteString(writer, "}")
+	return writeString(writer, "}")
 }
 
 func (le *luaEncoder) encodeAny(writer io.Writer, node *CandidateNode) error {
@@ -273,30 +273,30 @@ func (le *luaEncoder) encodeAny(writer io.Writer, node *CandidateNode) error {
 			return le.encodeString(writer, node)
 		case "!!null":
 			// TODO reject invalid use as a table key
-			return WriteString(writer, "nil")
+			return writeString(writer, "nil")
 		case "!!bool":
 			// Yaml 1.2 has case variation e.g. True, FALSE etc but Lua only has
 			// lower case
-			return WriteString(writer, strings.ToLower(node.Value))
+			return writeString(writer, strings.ToLower(node.Value))
 		case "!!int":
 			if strings.HasPrefix(node.Value, "0o") {
 				_, octalValue, err := parseInt64(node.Value)
 				if err != nil {
 					return err
 				}
-				return WriteString(writer, fmt.Sprintf("%d", octalValue))
+				return writeString(writer, fmt.Sprintf("%d", octalValue))
 			}
-			return WriteString(writer, strings.ToLower(node.Value))
+			return writeString(writer, strings.ToLower(node.Value))
 		case "!!float":
 			switch strings.ToLower(node.Value) {
 			case ".inf", "+.inf":
-				return WriteString(writer, "(1/0)")
+				return writeString(writer, "(1/0)")
 			case "-.inf":
-				return WriteString(writer, "(-1/0)")
+				return writeString(writer, "(-1/0)")
 			case ".nan":
-				return WriteString(writer, "(0/0)")
+				return writeString(writer, "(0/0)")
 			default:
-				return WriteString(writer, node.Value)
+				return writeString(writer, node.Value)
 			}
 		default:
 			return fmt.Errorf("Lua encoder NYI -- %s", node.Tag)
@@ -307,7 +307,7 @@ func (le *luaEncoder) encodeAny(writer io.Writer, node *CandidateNode) error {
 }
 
 func (le *luaEncoder) encodeTopLevel(writer io.Writer, node *CandidateNode) error {
-	err := WriteString(writer, le.docPrefix)
+	err := writeString(writer, le.docPrefix)
 	if err != nil {
 		return err
 	}
@@ -315,7 +315,7 @@ func (le *luaEncoder) encodeTopLevel(writer io.Writer, node *CandidateNode) erro
 	if err != nil {
 		return err
 	}
-	return WriteString(writer, le.docSuffix)
+	return writeString(writer, le.docSuffix)
 }
 
 func (le *luaEncoder) Encode(writer io.Writer, node *CandidateNode) error {

pkg/yqlib/encoder_properties.go

@@ -1,4 +1,4 @@
-package properties
+package yqlib
 
 import (
 	"bufio"
@@ -8,14 +8,13 @@ import (
 	"strings"
 
 	"github.com/magiconair/properties"
-	"github.com/mikefarah/yq/v4/pkg/yqlib"
 )
 
 type propertiesEncoder struct {
 	prefs PropertiesPreferences
 }
 
-func NewPropertiesEncoder(prefs PropertiesPreferences) yqlib.Encoder {
+func NewPropertiesEncoder(prefs PropertiesPreferences) Encoder {
 	return &propertiesEncoder{
 		prefs: prefs,
 	}
@@ -44,7 +43,7 @@ func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content strin
 			}
 
 		} else {
-			if err := yqlib.WriteString(writer, readline); err != nil {
+			if err := writeString(writer, readline); err != nil {
 				return err
 			}
 		}
@@ -52,7 +51,7 @@ func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content strin
 		if errors.Is(errReading, io.EOF) {
 			if readline != "" {
 				// the last comment we read didn't have a newline, put one in
-				if err := yqlib.WriteString(writer, "\n"); err != nil {
+				if err := writeString(writer, "\n"); err != nil {
 					return err
 				}
 			}
@@ -62,13 +61,13 @@ func (pe *propertiesEncoder) PrintLeadingContent(writer io.Writer, content strin
 	return nil
 }
 
-func (pe *propertiesEncoder) Encode(writer io.Writer, node *yqlib.CandidateNode) error {
+func (pe *propertiesEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 
-	if node.Kind == yqlib.ScalarNode {
-		return yqlib.WriteString(writer, node.Value+"\n")
+	if node.Kind == ScalarNode {
+		return writeString(writer, node.Value+"\n")
 	}
 
-	node.ConvertKeysToStrings()
+	mapKeysToStrings(node)
 	p := properties.NewProperties()
 	p.WriteSeparator = pe.prefs.KeyValueSeparator
 	err := pe.doEncode(p, node, "", nil)
@@ -80,19 +79,19 @@ func (pe *propertiesEncoder) Encode(writer io.Writer, node *yqlib.CandidateNode)
 	return err
 }
 
-func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yqlib.CandidateNode, path string, keyNode *yqlib.CandidateNode) error {
+func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *CandidateNode, path string, keyNode *CandidateNode) error {
 
 	comments := ""
 	if keyNode != nil {
 		// include the key node comments if present
-		comments = keyNode.CleanHeadAndLineComment()
+		comments = headAndLineComment(keyNode)
 	}
-	comments = comments + node.CleanHeadAndLineComment()
+	comments = comments + headAndLineComment(node)
 	commentsWithSpaces := strings.ReplaceAll(comments, "\n", "\n ")
 	p.SetComments(path, strings.Split(commentsWithSpaces, "\n"))
 
 	switch node.Kind {
-	case yqlib.ScalarNode:
+	case ScalarNode:
 		var nodeValue string
 		if pe.prefs.UnwrapScalar || !strings.Contains(node.Value, " ") {
 			nodeValue = node.Value
@@ -101,11 +100,11 @@ func (pe *propertiesEncoder) doEncode(p *properties.Properties, node *yqlib.Cand
 		}
 		_, _, err := p.Set(path, nodeValue)
 		return err
-	case yqlib.SequenceNode:
+	case SequenceNode:
 		return pe.encodeArray(p, node.Content, path)
-	case yqlib.MappingNode:
+	case MappingNode:
 		return pe.encodeMap(p, node.Content, path)
-	case yqlib.AliasNode:
+	case AliasNode:
 		return pe.doEncode(p, node.Alias, path, nil)
 	default:
 		return fmt.Errorf("Unsupported node %v", node.Tag)
@@ -126,7 +125,7 @@ func (pe *propertiesEncoder) appendPath(path string, key interface{}) string {
 	return fmt.Sprintf("%v.%v", path, key)
 }
 
-func (pe *propertiesEncoder) encodeArray(p *properties.Properties, kids []*yqlib.CandidateNode, path string) error {
+func (pe *propertiesEncoder) encodeArray(p *properties.Properties, kids []*CandidateNode, path string) error {
 	for index, child := range kids {
 		err := pe.doEncode(p, child, pe.appendPath(path, index), nil)
 		if err != nil {
@@ -136,7 +135,7 @@ func (pe *propertiesEncoder) encodeArray(p *properties.Properties, kids []*yqlib
 	return nil
 }
 
-func (pe *propertiesEncoder) encodeMap(p *properties.Properties, kids []*yqlib.CandidateNode, path string) error {
+func (pe *propertiesEncoder) encodeMap(p *properties.Properties, kids []*CandidateNode, path string) error {
 	for index := 0; index < len(kids); index = index + 2 {
 		key := kids[index]
 		value := kids[index+1]

pkg/yqlib/encoder_properties_test.go

@@ -1,4 +1,4 @@
-package properties
+package yqlib
 
 import (
 	"bufio"

pkg/yqlib/encoder_sh.go

@@ -30,11 +30,11 @@ func (e *shEncoder) PrintLeadingContent(_ io.Writer, _ string) error {
 }
 
 func (e *shEncoder) Encode(writer io.Writer, node *CandidateNode) error {
-	if node.GuessTagFromCustomType() != "!!str" {
+	if node.guessTagFromCustomType() != "!!str" {
 		return fmt.Errorf("cannot encode %v as URI, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag)
 	}
 
-	return WriteString(writer, e.encode(node.Value))
+	return writeString(writer, e.encode(node.Value))
 }
 
 // put any (shell-unsafe) characters into a single-quoted block, close the block lazily

pkg/yqlib/encoder_shellvariables.go

@@ -30,7 +30,7 @@ func (pe *shellVariablesEncoder) PrintLeadingContent(_ io.Writer, _ string) erro
 
 func (pe *shellVariablesEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 
-	node.ConvertKeysToStrings()
+	mapKeysToStrings(node)
 	err := pe.doEncode(&writer, node, "")
 	if err != nil {
 		return err

pkg/yqlib/encoder_toml.go

@@ -14,7 +14,7 @@ func NewTomlEncoder() Encoder {
 
 func (te *tomlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 	if node.Kind == ScalarNode {
-		return WriteString(writer, node.Value+"\n")
+		return writeString(writer, node.Value+"\n")
 	}
 	return fmt.Errorf("only scalars (e.g. strings, numbers, booleans) are supported for TOML output at the moment. Please use yaml output format (-oy) until the encoder has been fully implemented")
 }

pkg/yqlib/encoder_uri.go

@@ -26,7 +26,7 @@ func (e *uriEncoder) PrintLeadingContent(_ io.Writer, _ string) error {
 }
 
 func (e *uriEncoder) Encode(writer io.Writer, node *CandidateNode) error {
-	if node.GuessTagFromCustomType() != "!!str" {
+	if node.guessTagFromCustomType() != "!!str" {
 		return fmt.Errorf("cannot encode %v as URI, can only operate on strings. Please first pipe through another encoding operator to convert the value to a string", node.Tag)
 	}
 	_, err := writer.Write([]byte(url.QueryEscape(node.Value)))

pkg/yqlib/encoder_xml.go

@@ -1,6 +1,6 @@
 //go:build !yq_noxml
 
-package xml
+package yqlib
 
 import (
 	"encoding/xml"
@@ -8,8 +8,6 @@ import (
 	"io"
 	"regexp"
 	"strings"
-
-	"github.com/mikefarah/yq/v4/pkg/yqlib"
 )
 
 type xmlEncoder struct {
@@ -19,7 +17,7 @@ type xmlEncoder struct {
 	leadingContent string
 }
 
-func NewXMLEncoder(prefs XmlPreferences) yqlib.Encoder {
+func NewXMLEncoder(prefs XmlPreferences) Encoder {
 	var indentString = ""
 
 	for index := 0; index < prefs.Indent; index++ {
@@ -41,7 +39,7 @@ func (e *xmlEncoder) PrintLeadingContent(_ io.Writer, content string) error {
 	return nil
 }
 
-func (e *xmlEncoder) Encode(writer io.Writer, node *yqlib.CandidateNode) error {
+func (e *xmlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 	encoder := xml.NewEncoder(writer)
 	// hack so we can manually add newlines to procInst and directives
 	e.writer = writer
@@ -61,7 +59,7 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yqlib.CandidateNode) error {
 					return err
 				}
 				if _, err := e.writer.Write([]byte("\n")); err != nil {
-					yqlib.GetLogger().Warning("Unable to write newline, skipping: %w", err)
+					log.Warning("Unable to write newline, skipping: %w", err)
 				}
 			}
 		}
@@ -81,12 +79,12 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yqlib.CandidateNode) error {
 	}
 
 	switch node.Kind {
-	case yqlib.MappingNode:
+	case MappingNode:
 		err := e.encodeTopLevelMap(encoder, node)
 		if err != nil {
 			return err
 		}
-	case yqlib.ScalarNode:
+	case ScalarNode:
 		var charData xml.CharData = []byte(node.Value)
 		err := encoder.EncodeToken(charData)
 		if err != nil {
@@ -101,8 +99,8 @@ func (e *xmlEncoder) Encode(writer io.Writer, node *yqlib.CandidateNode) error {
 
 }
 
-func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yqlib.CandidateNode) error {
-	err := e.encodeComment(encoder, node.CleanHeadAndLineComment())
+func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *CandidateNode) error {
+	err := e.encodeComment(encoder, headAndLineComment(node))
 	if err != nil {
 		return err
 	}
@@ -111,12 +109,12 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yqlib.Candida
 		value := node.Content[i+1]
 
 		start := xml.StartElement{Name: xml.Name{Local: key.Value}}
-		yqlib.GetLogger().Debugf("comments of key %v", key.Value)
-		err := e.encodeComment(encoder, key.CleanHeadAndLineComment())
+		log.Debugf("comments of key %v", key.Value)
+		err := e.encodeComment(encoder, headAndLineComment(key))
 		if err != nil {
 			return err
 		}
-		if key.CleanHeadAndLineComment() != "" {
+		if headAndLineComment(key) != "" {
 			var newLine xml.CharData = []byte("\n")
 			err = encoder.EncodeToken(newLine)
 			if err != nil {
@@ -133,7 +131,7 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yqlib.Candida
 				return err
 			}
 			if _, err := e.writer.Write([]byte("\n")); err != nil {
-				yqlib.GetLogger().Warning("Unable to write newline, skipping: %w", err)
+				log.Warning("Unable to write newline, skipping: %w", err)
 			}
 		} else if key.Value == e.prefs.DirectiveName {
 			var directive xml.Directive = []byte(value.Value)
@@ -141,48 +139,48 @@ func (e *xmlEncoder) encodeTopLevelMap(encoder *xml.Encoder, node *yqlib.Candida
 				return err
 			}
 			if _, err := e.writer.Write([]byte("\n")); err != nil {
-				yqlib.GetLogger().Warning("Unable to write newline, skipping: %w", err)
+				log.Warning("Unable to write newline, skipping: %w", err)
 			}
 		} else {
 
-			yqlib.GetLogger().Debugf("recursing")
+			log.Debugf("recursing")
 
 			err = e.doEncode(encoder, value, start)
 			if err != nil {
 				return err
 			}
 		}
-		err = e.encodeComment(encoder, key.CleanFootComment())
+		err = e.encodeComment(encoder, footComment(key))
 		if err != nil {
 			return err
 		}
 	}
-	return e.encodeComment(encoder, node.CleanFootComment())
+	return e.encodeComment(encoder, footComment(node))
 }
 
-func (e *xmlEncoder) encodeStart(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
+func (e *xmlEncoder) encodeStart(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error {
 	err := encoder.EncodeToken(start)
 	if err != nil {
 		return err
 	}
-	return e.encodeComment(encoder, node.CleanHeadComment())
+	return e.encodeComment(encoder, headComment(node))
 }
 
-func (e *xmlEncoder) encodeEnd(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
+func (e *xmlEncoder) encodeEnd(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error {
 	err := encoder.EncodeToken(start.End())
 	if err != nil {
 		return err
 	}
-	return e.encodeComment(encoder, node.CleanFootComment())
+	return e.encodeComment(encoder, footComment(node))
 }
 
-func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
+func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error {
 	switch node.Kind {
-	case yqlib.MappingNode:
+	case MappingNode:
 		return e.encodeMap(encoder, node, start)
-	case yqlib.SequenceNode:
+	case SequenceNode:
 		return e.encodeArray(encoder, node, start)
-	case yqlib.ScalarNode:
+	case ScalarNode:
 		err := e.encodeStart(encoder, node, start)
 		if err != nil {
 			return err
@@ -194,7 +192,7 @@ func (e *xmlEncoder) doEncode(encoder *xml.Encoder, node *yqlib.CandidateNode, s
 			return err
 		}
 
-		if err = e.encodeComment(encoder, node.CleanLineComment()); err != nil {
+		if err = e.encodeComment(encoder, lineComment(node)); err != nil {
 			return err
 		}
 
@@ -209,13 +207,13 @@ var chompRegexp = regexp.MustCompile(`\n$`)
 
 func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) error {
 	if commentStr != "" {
-		yqlib.GetLogger().Debugf("got comment [%v]", commentStr)
+		log.Debugf("got comment [%v]", commentStr)
 		// multi line string
 		if len(commentStr) > 2 && strings.Contains(commentStr[1:len(commentStr)-1], "\n") {
 			commentStr = chompRegexp.ReplaceAllString(commentStr, "")
-			yqlib.GetLogger().Debugf("chompRegexp [%v]", commentStr)
+			log.Debugf("chompRegexp [%v]", commentStr)
 			commentStr = xmlEncodeMultilineCommentRegex.ReplaceAllString(commentStr, "$1$2")
-			yqlib.GetLogger().Debugf("processed multiline [%v]", commentStr)
+			log.Debugf("processed multiline [%v]", commentStr)
 			// if the first line is non blank, add a space
 			if commentStr[0] != '\n' && commentStr[0] != ' ' {
 				commentStr = " " + commentStr
@@ -227,9 +225,9 @@ func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) erro
 
 		if !strings.HasSuffix(commentStr, " ") && !strings.HasSuffix(commentStr, "\n") {
 			commentStr = commentStr + " "
-			yqlib.GetLogger().Debugf("added suffix [%v]", commentStr)
+			log.Debugf("added suffix [%v]", commentStr)
 		}
-		yqlib.GetLogger().Debugf("encoding comment [%v]", commentStr)
+		log.Debugf("encoding comment [%v]", commentStr)
 
 		var comment xml.Comment = []byte(commentStr)
 		err := encoder.EncodeToken(comment)
@@ -240,9 +238,9 @@ func (e *xmlEncoder) encodeComment(encoder *xml.Encoder, commentStr string) erro
 	return nil
 }
 
-func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
+func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error {
 
-	if err := e.encodeComment(encoder, node.CleanHeadAndLineComment()); err != nil {
+	if err := e.encodeComment(encoder, headAndLineComment(node)); err != nil {
 		return err
 	}
 
@@ -252,7 +250,7 @@ func (e *xmlEncoder) encodeArray(encoder *xml.Encoder, node *yqlib.CandidateNode
 			return err
 		}
 	}
-	return e.encodeComment(encoder, node.CleanFootComment())
+	return e.encodeComment(encoder, footComment(node))
 }
 
 func (e *xmlEncoder) isAttribute(name string) bool {
@@ -262,8 +260,8 @@ func (e *xmlEncoder) isAttribute(name string) bool {
 		!strings.HasPrefix(name, e.prefs.ProcInstPrefix)
 }
 
-func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yqlib.CandidateNode, start xml.StartElement) error {
-	yqlib.GetLogger().Debug("its a map")
+func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *CandidateNode, start xml.StartElement) error {
+	log.Debug("its a map")
 
 	//first find all the attributes and put them on the start token
 	for i := 0; i < len(node.Content); i += 2 {
@@ -271,7 +269,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yqlib.CandidateNode,
 		value := node.Content[i+1]
 
 		if e.isAttribute(key.Value) {
-			if value.Kind == yqlib.ScalarNode {
+			if value.Kind == ScalarNode {
 				attributeName := strings.Replace(key.Value, e.prefs.AttributePrefix, "", 1)
 				start.Attr = append(start.Attr, xml.Attr{Name: xml.Name{Local: attributeName}, Value: value.Value})
 			} else {
@@ -290,7 +288,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yqlib.CandidateNode,
 		key := node.Content[i]
 		value := node.Content[i+1]
 
-		err := e.encodeComment(encoder, key.CleanHeadAndLineComment())
+		err := e.encodeComment(encoder, headAndLineComment(key))
 		if err != nil {
 			return err
 		}
@@ -307,7 +305,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yqlib.CandidateNode,
 			}
 		} else if key.Value == e.prefs.ContentName {
 			// directly encode the contents
-			err = e.encodeComment(encoder, value.CleanHeadAndLineComment())
+			err = e.encodeComment(encoder, headAndLineComment(value))
 			if err != nil {
 				return err
 			}
@@ -316,7 +314,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yqlib.CandidateNode,
 			if err != nil {
 				return err
 			}
-			err = e.encodeComment(encoder, value.CleanFootComment())
+			err = e.encodeComment(encoder, footComment(value))
 			if err != nil {
 				return err
 			}
@@ -327,7 +325,7 @@ func (e *xmlEncoder) encodeMap(encoder *xml.Encoder, node *yqlib.CandidateNode,
 				return err
 			}
 		}
-		err = e.encodeComment(encoder, key.CleanFootComment())
+		err = e.encodeComment(encoder, footComment(key))
 		if err != nil {
 			return err
 		}

pkg/yqlib/encoder_yaml.go

@@ -26,7 +26,7 @@ func (ye *yamlEncoder) CanHandleAliases() bool {
 func (ye *yamlEncoder) PrintDocumentSeparator(writer io.Writer) error {
 	if ye.prefs.PrintDocSeparators {
 		log.Debug("writing doc sep")
-		if err := WriteString(writer, "---\n"); err != nil {
+		if err := writeString(writer, "---\n"); err != nil {
 			return err
 		}
 	}
@@ -54,7 +54,7 @@ func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) err
 			if len(readline) > 0 && readline != "\n" && readline[0] != '%' && !commentLineRegEx.MatchString(readline) {
 				readline = "# " + readline
 			}
-			if err := WriteString(writer, readline); err != nil {
+			if err := writeString(writer, readline); err != nil {
 				return err
 			}
 		}
@@ -62,7 +62,7 @@ func (ye *yamlEncoder) PrintLeadingContent(writer io.Writer, content string) err
 		if errors.Is(errReading, io.EOF) {
 			if readline != "" {
 				// the last comment we read didn't have a newline, put one in
-				if err := WriteString(writer, "\n"); err != nil {
+				if err := writeString(writer, "\n"); err != nil {
 					return err
 				}
 			}
@@ -80,7 +80,7 @@ func (ye *yamlEncoder) Encode(writer io.Writer, node *CandidateNode) error {
 		if node.LeadingContent == "" || valueToPrint != "" {
 			valueToPrint = valueToPrint + "\n"
 		}
-		return WriteString(writer, valueToPrint)
+		return writeString(writer, valueToPrint)
 	}
 
 	destination := writer

pkg/yqlib/format.go

@@ -6,100 +6,82 @@ import (
 )
 
 type EncoderFactoryFunction func() Encoder
-type InDocumentEncoderFactoryFunction func(indent int) Encoder
 type DecoderFactoryFunction func() Decoder
 
 type Format struct {
-	FormalName       string
-	Names            []string
-	DefaultExtension string
-	EncoderFactory   EncoderFactoryFunction
-	DecoderFactory   DecoderFactoryFunction
-
-	/**
-	* Like the Encoder Factory, but for encoding content within the document itself.
-	* Should turn off colors and other settings to ensure the content comes out right.
-	* If this function is not configured, it will default to the EncoderFactory.
-	**/
-	InDocumentEncoderFactory InDocumentEncoderFactoryFunction
+	FormalName     string
+	Names          []string
+	EncoderFactory EncoderFactoryFunction
+	DecoderFactory DecoderFactoryFunction
 }
 
-var YamlFormat = &Format{"yaml", []string{"y", "yml"}, "yml",
+var YamlFormat = &Format{"yaml", []string{"y", "yml"},
 	func() Encoder { return NewYamlEncoder(ConfiguredYamlPreferences) },
 	func() Decoder { return NewYamlDecoder(ConfiguredYamlPreferences) },
-	func(indent int) Encoder {
-		prefs := ConfiguredYamlPreferences.Copy()
-		prefs.Indent = indent
-		prefs.ColorsEnabled = false
-		return NewYamlEncoder(prefs)
-	},
 }
 
-var JSONFormat = &Format{"json", []string{"j"}, "json",
+var JSONFormat = &Format{"json", []string{"j"},
 	func() Encoder { return NewJSONEncoder(ConfiguredJSONPreferences) },
 	func() Decoder { return NewJSONDecoder() },
-	func(indent int) Encoder {
-		prefs := ConfiguredJSONPreferences.Copy()
-		prefs.Indent = indent
-		prefs.ColorsEnabled = false
-		prefs.UnwrapScalar = false
-		return NewJSONEncoder(prefs)
-	},
 }
 
-var CSVFormat = &Format{"csv", []string{"c"}, "csv",
+var PropertiesFormat = &Format{"props", []string{"p", "properties"},
+	func() Encoder { return NewPropertiesEncoder(ConfiguredPropertiesPreferences) },
+	func() Decoder { return NewPropertiesDecoder() },
+}
+
+var CSVFormat = &Format{"csv", []string{"c"},
 	func() Encoder { return NewCsvEncoder(ConfiguredCsvPreferences) },
 	func() Decoder { return NewCSVObjectDecoder(ConfiguredCsvPreferences) },
-	nil,
 }
 
-var TSVFormat = &Format{"tsv", []string{"t"}, "tsv",
+var TSVFormat = &Format{"tsv", []string{"t"},
 	func() Encoder { return NewCsvEncoder(ConfiguredTsvPreferences) },
 	func() Decoder { return NewCSVObjectDecoder(ConfiguredTsvPreferences) },
-	nil,
 }
 
-var Base64Format = &Format{"base64", []string{}, "txt",
+var XMLFormat = &Format{"xml", []string{"x"},
+	func() Encoder { return NewXMLEncoder(ConfiguredXMLPreferences) },
+	func() Decoder { return NewXMLDecoder(ConfiguredXMLPreferences) },
+}
+
+var Base64Format = &Format{"base64", []string{},
 	func() Encoder { return NewBase64Encoder() },
 	func() Decoder { return NewBase64Decoder() },
-	nil,
 }
 
-var UriFormat = &Format{"uri", []string{}, "txt",
+var UriFormat = &Format{"uri", []string{},
 	func() Encoder { return NewUriEncoder() },
 	func() Decoder { return NewUriDecoder() },
-	nil,
 }
 
-var ShFormat = &Format{"", nil, "sh",
+var ShFormat = &Format{"", nil,
 	func() Encoder { return NewShEncoder() },
 	nil,
-	nil,
 }
 
-var TomlFormat = &Format{"toml", []string{}, "toml",
+var TomlFormat = &Format{"toml", []string{},
 	func() Encoder { return NewTomlEncoder() },
 	func() Decoder { return NewTomlDecoder() },
-	nil,
 }
 
-var ShellVariablesFormat = &Format{"shell", []string{"s", "sh"}, "sh",
+var ShellVariablesFormat = &Format{"shell", []string{"s", "sh"},
 	func() Encoder { return NewShellVariablesEncoder() },
 	nil,
-	nil,
 }
 
-var LuaFormat = &Format{"lua", []string{"l"}, "lua",
+var LuaFormat = &Format{"lua", []string{"l"},
 	func() Encoder { return NewLuaEncoder(ConfiguredLuaPreferences) },
 	func() Decoder { return NewLuaDecoder(ConfiguredLuaPreferences) },
-	nil,
 }
 
 var Formats = []*Format{
 	YamlFormat,
 	JSONFormat,
+	PropertiesFormat,
 	CSVFormat,
 	TSVFormat,
+	XMLFormat,
 	Base64Format,
 	UriFormat,
 	ShFormat,
@@ -108,10 +90,6 @@ var Formats = []*Format{
 	LuaFormat,
 }
 
-func RegisterFormat(f *Format) {
-	Formats = append(Formats, f)
-}
-
 func (f *Format) MatchesName(name string) bool {
 	if f.FormalName == name {
 		return true
@@ -124,10 +102,7 @@ func (f *Format) MatchesName(name string) bool {
 	return false
 }
 
-func (f *Format) GetInDocumentEncoder(indent int) Encoder {
-	if f.InDocumentEncoderFactory != nil {
-		return f.InDocumentEncoderFactory(indent)
-	}
+func (f *Format) GetConfiguredEncoder() Encoder {
 	return f.EncoderFactory()
 }
 
@@ -148,9 +123,11 @@ func FormatStringFromFilename(filename string) string {
 }
 
 func FormatFromString(format string) (*Format, error) {
-	for _, printerFormat := range Formats {
-		if printerFormat.MatchesName(format) {
-			return printerFormat, nil
+	if format != "" {
+		for _, printerFormat := range Formats {
+			if printerFormat.MatchesName(format) {
+				return printerFormat, nil
+			}
 		}
 	}
 	return nil, fmt.Errorf("unknown format '%v' please use [%v]", format, GetAvailableOutputFormatString())

pkg/yqlib/format_test.go

@@ -0,0 +1,52 @@
+package yqlib
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+type formatStringScenario struct {
+	description    string
+	input          string
+	expectedFormat *Format
+	expectedError  string
+}
+
+var formatStringScenarios = []formatStringScenario{
+	{
+		description:    "yaml",
+		input:          "yaml",
+		expectedFormat: YamlFormat,
+	},
+	{
+		description:   "Unknown format type",
+		input:         "doc",
+		expectedError: "unknown format 'doc' please use",
+	},
+	{
+		description:   "blank should error",
+		input:         "",
+		expectedError: "unknown format '' please use",
+	},
+}
+
+func TestFormatFromString(t *testing.T) {
+	for _, tt := range formatStringScenarios {
+		actualFormat, actualError := FormatFromString(tt.input)
+
+		if tt.expectedError != "" {
+			if actualError == nil {
+				t.Errorf("Expected [%v] error but found none", tt.expectedError)
+			} else {
+				test.AssertResultWithContext(t, true, strings.Contains(actualError.Error(), tt.expectedError),
+					fmt.Sprintf("Expected [%v] to contain [%v]", actualError.Error(), tt.expectedError),
+				)
+			}
+		} else {
+			test.AssertResult(t, tt.expectedFormat, actualFormat)
+		}
+	}
+}

pkg/yqlib/formatting_expressions_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
-var formattingExpressionScenarios = []FormatScenario{
+var formattingExpressionScenarios = []formatScenario{
 	{
 		description: "Using expression files and comments",
 		skipDoc:     true,
@@ -43,7 +43,7 @@ var formattingExpressionScenarios = []FormatScenario{
 }
 
 func documentExpressionScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(FormatScenario)
+	s := i.(formatScenario)
 
 	if s.skipDoc {
 		return

pkg/yqlib/goccy_yaml_test.go

@@ -6,7 +6,7 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
-var goccyYamlFormatScenarios = []FormatScenario{
+var goccyYamlFormatScenarios = []formatScenario{
 	{
 		description: "basic - 3",
 		skipDoc:     true,
@@ -159,7 +159,7 @@ var goccyYamlFormatScenarios = []FormatScenario{
 	// },
 }
 
-func testGoccyYamlScenario(t *testing.T, s FormatScenario) {
+func testGoccyYamlScenario(t *testing.T, s formatScenario) {
 	test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewGoccyYAMLDecoder(), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
 }
 

pkg/yqlib/json_test.go

@@ -79,7 +79,7 @@ const roundTripMultiLineJson = `{
 }
 `
 
-var jsonScenarios = []FormatScenario{
+var jsonScenarios = []formatScenario{
 	{
 		description:  "array empty",
 		skipDoc:      true,
@@ -307,7 +307,7 @@ var jsonScenarios = []FormatScenario{
 	},
 }
 
-func documentRoundtripNdJsonScenario(w *bufio.Writer, s FormatScenario, indent int) {
+func documentRoundtripNdJsonScenario(w *bufio.Writer, s formatScenario, indent int) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -335,7 +335,7 @@ func documentRoundtripNdJsonScenario(w *bufio.Writer, s FormatScenario, indent i
 	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewJSONDecoder(), NewJSONEncoder(prefs))))
 }
 
-func documentDecodeNdJsonScenario(w *bufio.Writer, s FormatScenario) {
+func documentDecodeNdJsonScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -385,7 +385,7 @@ func decodeJSON(t *testing.T, jsonString string) *CandidateNode {
 	return context.MatchingNodes.Front().Value.(*CandidateNode)
 }
 
-func testJSONScenario(t *testing.T, s FormatScenario) {
+func testJSONScenario(t *testing.T, s formatScenario) {
 	prefs := ConfiguredJSONPreferences.Copy()
 	prefs.Indent = s.indent
 	prefs.UnwrapScalar = false
@@ -413,7 +413,7 @@ func testJSONScenario(t *testing.T, s FormatScenario) {
 	}
 }
 
-func documentJSONDecodeScenario(t *testing.T, w *bufio.Writer, s FormatScenario) {
+func documentJSONDecodeScenario(t *testing.T, w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -443,7 +443,7 @@ func documentJSONDecodeScenario(t *testing.T, w *bufio.Writer, s FormatScenario)
 }
 
 func documentJSONScenario(t *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(FormatScenario)
+	s := i.(formatScenario)
 	if s.skipDoc {
 		return
 	}
@@ -464,7 +464,7 @@ func documentJSONScenario(t *testing.T, w *bufio.Writer, i interface{}) {
 	}
 }
 
-func documentJSONEncodeScenario(w *bufio.Writer, s FormatScenario) {
+func documentJSONEncodeScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {

pkg/yqlib/lexer_participle.go

@@ -7,7 +7,7 @@ import (
 	"github.com/alecthomas/participle/v2/lexer"
 )
 
-var participleYqRules = []*ParticipleYqRule{
+var participleYqRules = []*participleYqRule{
 	{"LINE_COMMENT", `line_?comment|lineComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{LineComment: true}), 0},
 	{"HEAD_COMMENT", `head_?comment|headComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{HeadComment: true}), 0},
 	{"FOOT_COMMENT", `foot_?comment|footComment`, opTokenWithPrefs(getCommentOpType, assignCommentOpType, commentOpPreferences{FootComment: true}), 0},
@@ -40,6 +40,7 @@ var participleYqRules = []*ParticipleYqRule{
 	simpleOp("map", mapOpType),
 	simpleOp("filter", filterOpType),
 	simpleOp("pick", pickOpType),
+	simpleOp("omit", omitOpType),
 
 	{"FlattenWithDepth", `flatten\([0-9]+\)`, flattenWithDepth(), 0},
 	{"Flatten", `flatten`, opTokenWithPrefs(flattenOpType, nil, flattenPreferences{depth: -1}), 0},
@@ -57,34 +58,43 @@ var participleYqRules = []*ParticipleYqRule{
 
 	{"ArrayToMap", "array_?to_?map", expressionOpToken(`(.[] | select(. != null) ) as $i ireduce({}; .[$i | key] = $i)`), 0},
 
-	{"YamlEncodeWithIndent", `to_?yaml\([0-9]+\)`, CreateEncodeYqActionParsingIndent(YamlFormat), 0},
+	{"YamlEncodeWithIndent", `to_?yaml\([0-9]+\)`, encodeParseIndent(YamlFormat), 0},
+	{"XMLEncodeWithIndent", `to_?xml\([0-9]+\)`, encodeParseIndent(XMLFormat), 0},
+	{"JSONEncodeWithIndent", `to_?json\([0-9]+\)`, encodeParseIndent(JSONFormat), 0},
 
-	{"JSONEncodeWithIndent", `to_?json\([0-9]+\)`, CreateEncodeYqActionParsingIndent(JSONFormat), 0},
+	{"YamlDecode", `from_?yaml|@yamld|from_?json|@jsond`, decodeOp(YamlFormat), 0},
+	{"YamlEncode", `to_?yaml|@yaml`, encodeWithIndent(YamlFormat, 2), 0},
 
-	{"YamlDecode", `from_?yaml|@yamld|from_?json|@jsond`, CreateDecodeOpYqAction(YamlFormat), 0},
-	{"YamlEncode", `to_?yaml|@yaml`, CreateEncodeOpYqAction(YamlFormat, 2), 0},
+	{"JSONEncode", `to_?json`, encodeWithIndent(JSONFormat, 2), 0},
+	{"JSONEncodeNoIndent", `@json`, encodeWithIndent(JSONFormat, 0), 0},
 
-	{"JSONEncode", `to_?json`, CreateEncodeOpYqAction(JSONFormat, 2), 0},
-	{"JSONEncodeNoIndent", `@json`, CreateEncodeOpYqAction(JSONFormat, 0), 0},
+	{"PropertiesDecode", `from_?props|@propsd`, decodeOp(PropertiesFormat), 0},
+	{"PropsEncode", `to_?props|@props`, encodeWithIndent(PropertiesFormat, 2), 0},
 
-	{"CSVDecode", `from_?csv|@csvd`, CreateDecodeOpYqAction(CSVFormat), 0},
-	{"CSVEncode", `to_?csv|@csv`, CreateEncodeOpYqAction(CSVFormat, 0), 0},
+	{"XmlDecode", `from_?xml|@xmld`, decodeOp(XMLFormat), 0},
+	{"XMLEncode", `to_?xml`, encodeWithIndent(XMLFormat, 2), 0},
+	{"XMLEncodeNoIndent", `@xml`, encodeWithIndent(XMLFormat, 0), 0},
 
-	{"TSVDecode", `from_?tsv|@tsvd`, CreateDecodeOpYqAction(TSVFormat), 0},
-	{"TSVEncode", `to_?tsv|@tsv`, CreateEncodeOpYqAction(TSVFormat, 0), 0},
+	{"CSVDecode", `from_?csv|@csvd`, decodeOp(CSVFormat), 0},
+	{"CSVEncode", `to_?csv|@csv`, encodeWithIndent(CSVFormat, 0), 0},
 
-	{"Base64d", `@base64d`, CreateDecodeOpYqAction(Base64Format), 0},
-	{"Base64", `@base64`, CreateEncodeOpYqAction(Base64Format, 0), 0},
+	{"TSVDecode", `from_?tsv|@tsvd`, decodeOp(TSVFormat), 0},
+	{"TSVEncode", `to_?tsv|@tsv`, encodeWithIndent(TSVFormat, 0), 0},
 
-	{"Urid", `@urid`, CreateDecodeOpYqAction(UriFormat), 0},
-	{"Uri", `@uri`, CreateEncodeOpYqAction(UriFormat, 0), 0},
-	{"SH", `@sh`, CreateEncodeOpYqAction(ShFormat, 0), 0},
+	{"Base64d", `@base64d`, decodeOp(Base64Format), 0},
+	{"Base64", `@base64`, encodeWithIndent(Base64Format, 0), 0},
 
-	{"LoadBase64", `load_?base64`, CreateLoadOpYqAction(NewBase64Decoder()), 0},
+	{"Urid", `@urid`, decodeOp(UriFormat), 0},
+	{"Uri", `@uri`, encodeWithIndent(UriFormat, 0), 0},
+	{"SH", `@sh`, encodeWithIndent(ShFormat, 0), 0},
 
-	simpleOp(`load_?str|str_?load`, loadStringOpType),
+	{"LoadXML", `load_?xml|xml_?load`, loadOp(NewXMLDecoder(ConfiguredXMLPreferences)), 0},
 
-	{"LoadYaml", `load`, CreateLoadOpYqAction(NewYamlDecoder(LoadYamlPreferences)), 0},
+	{"LoadBase64", `load_?base64`, loadOp(NewBase64Decoder()), 0},
+
+	{"LoadProperties", `load_?props`, loadOp(NewPropertiesDecoder()), 0},
+	simpleOp("load_?str|str_?load", loadStringOpType),
+	{"LoadYaml", `load`, loadOp(NewYamlDecoder(LoadYamlPreferences)), 0},
 
 	{"SplitDocument", `splitDoc|split_?doc`, opToken(splitDocumentOpType), 0},
 
@@ -189,6 +199,9 @@ var participleYqRules = []*ParticipleYqRule{
 	{"GreaterThan", `\s*>\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: false, Greater: true}), 0},
 	{"LessThan", `\s*<\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: false, Greater: false}), 0},
 
+	simpleOp("min", minOpType),
+	simpleOp("max", maxOpType),
+
 	{"AssignRelative", `\|=[c]*`, assignOpToken(true), 0},
 	{"Assign", `=[c]*`, assignOpToken(false), 0},
 
@@ -214,11 +227,13 @@ var participleYqRules = []*ParticipleYqRule{
 	{"SubtractAssign", `\-=`, opToken(subtractAssignOpType), 0},
 	{"Subtract", `\-`, opToken(subtractOpType), 0},
 	{"Comment", `#.*`, nil, 0},
+
+	simpleOp("pivot", pivotOpType),
 }
 
 type yqAction func(lexer.Token) (*token, error)
 
-type ParticipleYqRule struct {
+type participleYqRule struct {
 	Name                string
 	Pattern             string
 	CreateYqToken       yqAction
@@ -229,16 +244,12 @@ type participleLexer struct {
 	lexerDefinition lexer.StringDefinition
 }
 
-func simpleOp(name string, opType *operationType) *ParticipleYqRule {
-	return &ParticipleYqRule{strings.ToUpper(string(name[1])) + name[1:], name, opToken(opType), 0}
+func simpleOp(name string, opType *operationType) *participleYqRule {
+	return &participleYqRule{strings.ToUpper(string(name[1])) + name[1:], name, opToken(opType), 0}
 }
 
-func assignableOp(name string, opType *operationType, assignOpType *operationType) *ParticipleYqRule {
-	return &ParticipleYqRule{strings.ToUpper(string(name[1])) + name[1:], name, opTokenWithPrefs(opType, assignOpType, nil), 0}
-}
-
-func RegisterRules(rules []*ParticipleYqRule) {
-	participleYqRules = append(participleYqRules, rules...)
+func assignableOp(name string, opType *operationType, assignOpType *operationType) *participleYqRule {
+	return &participleYqRule{strings.ToUpper(string(name[1])) + name[1:], name, opTokenWithPrefs(opType, assignOpType, nil), 0}
 }
 
 func newParticipleLexer() expressionTokeniser {
@@ -317,7 +328,7 @@ func flattenWithDepth() yqAction {
 
 		prefs := flattenPreferences{depth: depth}
 		op := &Operation{OperationType: flattenOpType, Value: flattenOpType.Type, StringValue: value, Preferences: prefs}
-		return &token{TokenType: operationToken, Operation: op}, nil
+		return &token{TokenType: operationToken, Operation: op, CheckForPostTraverse: flattenOpType.CheckForPostTraverse}, nil
 	}
 }
 
@@ -395,7 +406,7 @@ func envOp(strenv bool) yqAction {
 		envOperation.OperationType = envOpType
 		envOperation.Preferences = preferences
 
-		return &token{TokenType: operationToken, Operation: envOperation}, nil
+		return &token{TokenType: operationToken, Operation: envOperation, CheckForPostTraverse: envOpType.CheckForPostTraverse}, nil
 	}
 }
 
@@ -518,7 +529,7 @@ func parentWithDefaultLevel() yqAction {
 	}
 }
 
-func CreateEncodeYqActionParsingIndent(outputFormat *Format) yqAction {
+func encodeParseIndent(outputFormat *Format) yqAction {
 	return func(rawToken lexer.Token) (*token, error) {
 		value := rawToken.Value
 		var indent, errParsingInt = extractNumberParameter(value)
@@ -532,17 +543,17 @@ func CreateEncodeYqActionParsingIndent(outputFormat *Format) yqAction {
 	}
 }
 
-func CreateEncodeOpYqAction(outputFormat *Format, indent int) yqAction {
+func encodeWithIndent(outputFormat *Format, indent int) yqAction {
 	prefs := encoderPreferences{format: outputFormat, indent: indent}
 	return opTokenWithPrefs(encodeOpType, nil, prefs)
 }
 
-func CreateDecodeOpYqAction(format *Format) yqAction {
+func decodeOp(format *Format) yqAction {
 	prefs := decoderPreferences{format: format}
 	return opTokenWithPrefs(decodeOpType, nil, prefs)
 }
 
-func CreateLoadOpYqAction(decoder Decoder) yqAction {
+func loadOp(decoder Decoder) yqAction {
 	prefs := loadPrefs{decoder}
 	return opTokenWithPrefs(loadOpType, nil, prefs)
 }
@@ -557,13 +568,13 @@ func literalToken(tt tokenType, checkForPost bool) yqAction {
 	}
 }
 
-func (p *participleLexer) getYqDefinition(rawToken lexer.Token) *ParticipleYqRule {
+func (p *participleLexer) getYqDefinition(rawToken lexer.Token) *participleYqRule {
 	for _, yqRule := range participleYqRules {
 		if yqRule.ParticipleTokenType == rawToken.Type {
 			return yqRule
 		}
 	}
-	return &ParticipleYqRule{}
+	return &participleYqRule{}
 }
 
 func (p *participleLexer) Tokenise(expression string) ([]*token, error) {

pkg/yqlib/lib.go

@@ -98,7 +98,7 @@ func parseSnippet(value string) (*CandidateNode, error) {
 	if result.Tag == "!!str" {
 		// use the original string value, as
 		// decoding drops new lines
-		return CreateScalarNode(value, value), nil
+		return createScalarNode(value, value), nil
 	}
 	result.Line = 0
 	result.Column = 0
@@ -114,8 +114,8 @@ func recursiveNodeEqual(lhs *CandidateNode, rhs *CandidateNode) bool {
 		//process custom tags of scalar nodes.
 		//dont worry about matching tags of maps or arrays.
 
-		lhsTag := lhs.GuessTagFromCustomType()
-		rhsTag := rhs.GuessTagFromCustomType()
+		lhsTag := lhs.guessTagFromCustomType()
+		rhsTag := rhs.guessTagFromCustomType()
 
 		if lhsTag != rhsTag {
 			return false
@@ -135,8 +135,12 @@ func recursiveNodeEqual(lhs *CandidateNode, rhs *CandidateNode) bool {
 	return false
 }
 
-// yaml numbers can be hex and octal encoded...
+// yaml numbers can have underscores, be hex and octal encoded...
 func parseInt64(numberString string) (string, int64, error) {
+	if strings.Contains(numberString, "_") {
+		numberString = strings.ReplaceAll(numberString, "_", "")
+	}
+
 	if strings.HasPrefix(numberString, "0x") ||
 		strings.HasPrefix(numberString, "0X") {
 		num, err := strconv.ParseInt(numberString[2:], 16, 64)
@@ -161,6 +165,22 @@ func parseInt(numberString string) (int, error) {
 	return int(parsed), err
 }
 
+func headAndLineComment(node *CandidateNode) string {
+	return headComment(node) + lineComment(node)
+}
+
+func headComment(node *CandidateNode) string {
+	return strings.Replace(node.HeadComment, "#", "", 1)
+}
+
+func lineComment(node *CandidateNode) string {
+	return strings.Replace(node.LineComment, "#", "", 1)
+}
+
+func footComment(node *CandidateNode) string {
+	return strings.Replace(node.FootComment, "#", "", 1)
+}
+
 // use for debugging only
 func NodesToString(collection *list.List) string {
 	if !log.IsEnabledFor(logging.DEBUG) {

pkg/yqlib/lib_test.go

@@ -106,6 +106,7 @@ func TestParseSnippet(t *testing.T) {
 type parseInt64Scenario struct {
 	numberString         string
 	expectedParsedNumber int64
+	expectedFormatString string
 }
 
 var parseInt64Scenarios = []parseInt64Scenario{
@@ -113,10 +114,20 @@ var parseInt64Scenarios = []parseInt64Scenario{
 		numberString:         "34",
 		expectedParsedNumber: 34,
 	},
+	{
+		numberString:         "10_000",
+		expectedParsedNumber: 10000,
+		expectedFormatString: "10000",
+	},
 	{
 		numberString:         "0x10",
 		expectedParsedNumber: 16,
 	},
+	{
+		numberString:         "0x10_000",
+		expectedParsedNumber: 65536,
+		expectedFormatString: "0x10000",
+	},
 	{
 		numberString:         "0o10",
 		expectedParsedNumber: 8,
@@ -132,7 +143,10 @@ func TestParseInt64(t *testing.T) {
 			t.Error(err)
 		}
 		test.AssertResultComplexWithContext(t, tt.expectedParsedNumber, actualNumber, tt.numberString)
+		if tt.expectedFormatString == "" {
+			tt.expectedFormatString = tt.numberString
+		}
 
-		test.AssertResultComplexWithContext(t, tt.numberString, fmt.Sprintf(format, actualNumber), fmt.Sprintf("Formatting of: %v", tt.numberString))
+		test.AssertResultComplexWithContext(t, tt.expectedFormatString, fmt.Sprintf(format, actualNumber), fmt.Sprintf("Formatting of: %v", tt.numberString))
 	}
 }

pkg/yqlib/lua_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/mikefarah/yq/v4/test"
 )
 
-var luaScenarios = []FormatScenario{
+var luaScenarios = []formatScenario{
 	{
 		description: "Basic input example",
 		input: `return {
@@ -250,7 +250,7 @@ numbers:
 	},
 }
 
-func testLuaScenario(t *testing.T, s FormatScenario) {
+func testLuaScenario(t *testing.T, s formatScenario) {
 	switch s.scenarioType {
 	case "", "decode":
 		test.AssertResultWithContext(t, s.expected, mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewYamlEncoder(ConfiguredYamlPreferences)), s.description)
@@ -278,7 +278,7 @@ func testLuaScenario(t *testing.T, s FormatScenario) {
 }
 
 func documentLuaScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
-	s := i.(FormatScenario)
+	s := i.(formatScenario)
 
 	if s.skipDoc {
 		return
@@ -295,7 +295,7 @@ func documentLuaScenario(_ *testing.T, w *bufio.Writer, i interface{}) {
 	}
 }
 
-func documentLuaDecodeScenario(w *bufio.Writer, s FormatScenario) {
+func documentLuaDecodeScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -317,7 +317,7 @@ func documentLuaDecodeScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("```yaml\n%v```\n\n", mustProcessFormatScenario(s, NewLuaDecoder(ConfiguredLuaPreferences), NewYamlEncoder(ConfiguredYamlPreferences))))
 }
 
-func documentLuaEncodeScenario(w *bufio.Writer, s FormatScenario) {
+func documentLuaEncodeScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {
@@ -359,7 +359,7 @@ func documentLuaEncodeScenario(w *bufio.Writer, s FormatScenario) {
 	writeOrPanic(w, fmt.Sprintf("```lua\n%v```\n\n", mustProcessFormatScenario(s, NewYamlDecoder(ConfiguredYamlPreferences), NewLuaEncoder(prefs))))
 }
 
-func documentLuaRoundTripScenario(w *bufio.Writer, s FormatScenario) {
+func documentLuaRoundTripScenario(w *bufio.Writer, s formatScenario) {
 	writeOrPanic(w, fmt.Sprintf("## %v\n", s.description))
 
 	if s.subdescription != "" {

pkg/yqlib/operation.go

@@ -26,7 +26,7 @@ var valueToStringFunc = func(p *Operation) string {
 
 func createValueOperation(value interface{}, stringValue string) *Operation {
 	log.Debug("creating value op for string %v", stringValue)
-	var node = CreateScalarNode(value, stringValue)
+	var node = createScalarNode(value, stringValue)
 
 	return &Operation{
 		OperationType: valueOpType,
@@ -73,6 +73,8 @@ var equalsOpType = &operationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Ha
 var notEqualsOpType = &operationType{Type: "NOT_EQUALS", NumArgs: 2, Precedence: 40, Handler: notEqualsOperator}
 
 var compareOpType = &operationType{Type: "COMPARE", NumArgs: 2, Precedence: 40, Handler: compareOperator}
+var minOpType = &operationType{Type: "MIN", NumArgs: 0, Precedence: 40, Handler: minOperator}
+var maxOpType = &operationType{Type: "MAX", NumArgs: 0, Precedence: 40, Handler: maxOperator}
 
 // createmap needs to be above union, as we use union to build the components of the objects
 var createMapOpType = &operationType{Type: "CREATE_MAP", NumArgs: 2, Precedence: 15, Handler: createMapOperator}
@@ -83,15 +85,17 @@ var lengthOpType = &operationType{Type: "LENGTH", NumArgs: 0, Precedence: 50, Ha
 var lineOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Handler: lineOperator}
 var columnOpType = &operationType{Type: "LINE", NumArgs: 0, Precedence: 50, Handler: columnOperator}
 
+// Use this expression to create alias/syntactic sugar expressions (in lexer_participle).
 var expressionOpType = &operationType{Type: "EXP", NumArgs: 0, Precedence: 50, Handler: expressionOperator}
 
 var collectOpType = &operationType{Type: "COLLECT", NumArgs: 1, Precedence: 50, Handler: collectOperator}
-var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler: mapOperator}
-var filterOpType = &operationType{Type: "FILTER", NumArgs: 1, Precedence: 50, Handler: filterOperator}
+var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 52, Handler: mapOperator, CheckForPostTraverse: true}
+var filterOpType = &operationType{Type: "FILTER", NumArgs: 1, Precedence: 52, Handler: filterOperator, CheckForPostTraverse: true}
 var errorOpType = &operationType{Type: "ERROR", NumArgs: 1, Precedence: 50, Handler: errorOperator}
-var pickOpType = &operationType{Type: "PICK", NumArgs: 1, Precedence: 50, Handler: pickOperator}
-var evalOpType = &operationType{Type: "EVAL", NumArgs: 1, Precedence: 50, Handler: evalOperator}
-var mapValuesOpType = &operationType{Type: "MAP_VALUES", NumArgs: 1, Precedence: 50, Handler: mapValuesOperator}
+var pickOpType = &operationType{Type: "PICK", NumArgs: 1, Precedence: 52, Handler: pickOperator, CheckForPostTraverse: true}
+var omitOpType = &operationType{Type: "OMIT", NumArgs: 1, Precedence: 52, Handler: omitOperator, CheckForPostTraverse: true}
+var evalOpType = &operationType{Type: "EVAL", NumArgs: 1, Precedence: 52, Handler: evalOperator, CheckForPostTraverse: true}
+var mapValuesOpType = &operationType{Type: "MAP_VALUES", NumArgs: 1, Precedence: 52, Handler: mapValuesOperator, CheckForPostTraverse: true}
 
 var formatDateTimeOpType = &operationType{Type: "FORMAT_DATE_TIME", NumArgs: 1, Precedence: 50, Handler: formatDateTime}
 var withDtFormatOpType = &operationType{Type: "WITH_DATE_TIME_FORMAT", NumArgs: 1, Precedence: 50, Handler: withDateTimeFormat}
@@ -113,9 +117,9 @@ var toEntriesOpType = &operationType{Type: "TO_ENTRIES", NumArgs: 0, Precedence:
 var fromEntriesOpType = &operationType{Type: "FROM_ENTRIES", NumArgs: 0, Precedence: 50, Handler: fromEntriesOperator}
 var withEntriesOpType = &operationType{Type: "WITH_ENTRIES", NumArgs: 1, Precedence: 50, Handler: withEntriesOperator}
 
-var withOpType = &operationType{Type: "WITH", NumArgs: 1, Precedence: 50, Handler: withOperator}
+var withOpType = &operationType{Type: "WITH", NumArgs: 1, Precedence: 52, Handler: withOperator, CheckForPostTraverse: true}
 
-var splitDocumentOpType = &operationType{Type: "SPLIT_DOC", NumArgs: 0, Precedence: 50, Handler: splitDocumentOperator}
+var splitDocumentOpType = &operationType{Type: "SPLIT_DOC", NumArgs: 0, Precedence: 52, Handler: splitDocumentOperator, CheckForPostTraverse: true}
 var getVariableOpType = &operationType{Type: "GET_VARIABLE", NumArgs: 0, Precedence: 55, Handler: getVariableOperator}
 var getStyleOpType = &operationType{Type: "GET_STYLE", NumArgs: 0, Precedence: 50, Handler: getStyleOperator}
 var getTagOpType = &operationType{Type: "GET_TAG", NumArgs: 0, Precedence: 50, Handler: getTagOperator}
@@ -132,33 +136,33 @@ var getDocumentIndexOpType = &operationType{Type: "GET_DOCUMENT_INDEX", NumArgs:
 var getFilenameOpType = &operationType{Type: "GET_FILENAME", NumArgs: 0, Precedence: 50, Handler: getFilenameOperator}
 var getFileIndexOpType = &operationType{Type: "GET_FILE_INDEX", NumArgs: 0, Precedence: 50, Handler: getFileIndexOperator}
 
-var getPathOpType = &operationType{Type: "GET_PATH", NumArgs: 0, Precedence: 50, Handler: getPathOperator}
+var getPathOpType = &operationType{Type: "GET_PATH", NumArgs: 0, Precedence: 52, Handler: getPathOperator, CheckForPostTraverse: true}
 var setPathOpType = &operationType{Type: "SET_PATH", NumArgs: 1, Precedence: 50, Handler: setPathOperator}
-var delPathsOpType = &operationType{Type: "DEL_PATHS", NumArgs: 1, Precedence: 50, Handler: delPathsOperator}
+var delPathsOpType = &operationType{Type: "DEL_PATHS", NumArgs: 1, Precedence: 52, Handler: delPathsOperator, CheckForPostTraverse: true}
 
-var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 50, Handler: explodeOperator}
-var sortByOpType = &operationType{Type: "SORT_BY", NumArgs: 1, Precedence: 50, Handler: sortByOperator}
-var reverseOpType = &operationType{Type: "REVERSE", NumArgs: 0, Precedence: 50, Handler: reverseOperator}
-var sortOpType = &operationType{Type: "SORT", NumArgs: 0, Precedence: 50, Handler: sortOperator}
-var shuffleOpType = &operationType{Type: "SHUFFLE", NumArgs: 0, Precedence: 50, Handler: shuffleOperator}
+var explodeOpType = &operationType{Type: "EXPLODE", NumArgs: 1, Precedence: 52, Handler: explodeOperator, CheckForPostTraverse: true}
+var sortByOpType = &operationType{Type: "SORT_BY", NumArgs: 1, Precedence: 52, Handler: sortByOperator, CheckForPostTraverse: true}
+var reverseOpType = &operationType{Type: "REVERSE", NumArgs: 0, Precedence: 52, Handler: reverseOperator, CheckForPostTraverse: true}
+var sortOpType = &operationType{Type: "SORT", NumArgs: 0, Precedence: 52, Handler: sortOperator, CheckForPostTraverse: true}
+var shuffleOpType = &operationType{Type: "SHUFFLE", NumArgs: 0, Precedence: 52, Handler: shuffleOperator, CheckForPostTraverse: true}
 
-var sortKeysOpType = &operationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 50, Handler: sortKeysOperator}
+var sortKeysOpType = &operationType{Type: "SORT_KEYS", NumArgs: 1, Precedence: 52, Handler: sortKeysOperator, CheckForPostTraverse: true}
 
 var joinStringOpType = &operationType{Type: "JOIN", NumArgs: 1, Precedence: 50, Handler: joinStringOperator}
 var subStringOpType = &operationType{Type: "SUBSTR", NumArgs: 1, Precedence: 50, Handler: substituteStringOperator}
 var matchOpType = &operationType{Type: "MATCH", NumArgs: 1, Precedence: 50, Handler: matchOperator}
 var captureOpType = &operationType{Type: "CAPTURE", NumArgs: 1, Precedence: 50, Handler: captureOperator}
 var testOpType = &operationType{Type: "TEST", NumArgs: 1, Precedence: 50, Handler: testOperator}
-var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 50, Handler: splitStringOperator}
+var splitStringOpType = &operationType{Type: "SPLIT", NumArgs: 1, Precedence: 52, Handler: splitStringOperator, CheckForPostTraverse: true}
 var changeCaseOpType = &operationType{Type: "CHANGE_CASE", NumArgs: 0, Precedence: 50, Handler: changeCaseOperator}
 var trimOpType = &operationType{Type: "TRIM", NumArgs: 0, Precedence: 50, Handler: trimSpaceOperator}
 var toStringOpType = &operationType{Type: "TO_STRING", NumArgs: 0, Precedence: 50, Handler: toStringOperator}
 var stringInterpolationOpType = &operationType{Type: "STRING_INT", NumArgs: 0, Precedence: 50, Handler: stringInterpolationOperator, ToString: valueToStringFunc}
 
-var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 52, Handler: loadOperator}
+var loadOpType = &operationType{Type: "LOAD", NumArgs: 1, Precedence: 52, Handler: loadOperator, CheckForPostTraverse: true}
 var loadStringOpType = &operationType{Type: "LOAD_STRING", NumArgs: 1, Precedence: 52, Handler: loadStringOperator}
 
-var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 50, Handler: keysOperator}
+var keysOpType = &operationType{Type: "KEYS", NumArgs: 0, Precedence: 52, Handler: keysOperator, CheckForPostTraverse: true}
 
 var collectObjectOpType = &operationType{Type: "COLLECT_OBJECT", NumArgs: 0, Precedence: 50, Handler: collectObjectOperator}
 
@@ -172,7 +176,7 @@ var traverseArrayOpType = &operationType{Type: "TRAVERSE_ARRAY", NumArgs: 2, Pre
 var selfReferenceOpType = &operationType{Type: "SELF", NumArgs: 0, Precedence: 55, Handler: selfOperator}
 var valueOpType = &operationType{Type: "VALUE", NumArgs: 0, Precedence: 50, Handler: valueOperator, ToString: valueToStringFunc}
 var referenceOpType = &operationType{Type: "REF", NumArgs: 0, Precedence: 50, Handler: referenceOperator}
-var envOpType = &operationType{Type: "ENV", NumArgs: 0, Precedence: 50, Handler: envOperator}
+var envOpType = &operationType{Type: "ENV", NumArgs: 0, Precedence: 52, Handler: envOperator, CheckForPostTraverse: true}
 var notOpType = &operationType{Type: "NOT", NumArgs: 0, Precedence: 50, Handler: notOperator}
 var toNumberOpType = &operationType{Type: "TO_NUMBER", NumArgs: 0, Precedence: 50, Handler: toNumberOperator}
 var emptyOpType = &operationType{Type: "EMPTY", Precedence: 50, Handler: emptyOperator}
@@ -181,14 +185,16 @@ var envsubstOpType = &operationType{Type: "ENVSUBST", NumArgs: 0, Precedence: 50
 
 var recursiveDescentOpType = &operationType{Type: "RECURSIVE_DESCENT", NumArgs: 0, Precedence: 50, Handler: recursiveDescentOperator}
 
-var selectOpType = &operationType{Type: "SELECT", NumArgs: 1, Precedence: 50, Handler: selectOperator}
+var selectOpType = &operationType{Type: "SELECT", NumArgs: 1, Precedence: 52, Handler: selectOperator, CheckForPostTraverse: true}
 var hasOpType = &operationType{Type: "HAS", NumArgs: 1, Precedence: 50, Handler: hasOperator}
-var uniqueOpType = &operationType{Type: "UNIQUE", NumArgs: 0, Precedence: 50, Handler: unique}
-var uniqueByOpType = &operationType{Type: "UNIQUE_BY", NumArgs: 1, Precedence: 50, Handler: uniqueBy}
-var groupByOpType = &operationType{Type: "GROUP_BY", NumArgs: 1, Precedence: 50, Handler: groupBy}
-var flattenOpType = &operationType{Type: "FLATTEN_BY", NumArgs: 0, Precedence: 50, Handler: flattenOp}
+var uniqueOpType = &operationType{Type: "UNIQUE", NumArgs: 0, Precedence: 52, Handler: unique, CheckForPostTraverse: true}
+var uniqueByOpType = &operationType{Type: "UNIQUE_BY", NumArgs: 1, Precedence: 52, Handler: uniqueBy, CheckForPostTraverse: true}
+var groupByOpType = &operationType{Type: "GROUP_BY", NumArgs: 1, Precedence: 52, Handler: groupBy, CheckForPostTraverse: true}
+var flattenOpType = &operationType{Type: "FLATTEN_BY", NumArgs: 0, Precedence: 52, Handler: flattenOp, CheckForPostTraverse: true}
 var deleteChildOpType = &operationType{Type: "DELETE", NumArgs: 1, Precedence: 40, Handler: deleteChildOperator}
 
+var pivotOpType = &operationType{Type: "PIVOT", NumArgs: 0, Precedence: 52, Handler: pivotOperator, CheckForPostTraverse: true}
+
 // debugging purposes only
 func (p *Operation) toString() string {
 	if p == nil {

pkg/yqlib/operator_add.go

@@ -74,11 +74,11 @@ func add(_ *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *Candida
 
 func addScalars(context Context, target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error {
 	lhsTag := lhs.Tag
-	rhsTag := rhs.GuessTagFromCustomType()
+	rhsTag := rhs.guessTagFromCustomType()
 	lhsIsCustom := false
 	if !strings.HasPrefix(lhsTag, "!!") {
 		// custom tag - we have to have a guess
-		lhsTag = lhs.GuessTagFromCustomType()
+		lhsTag = lhs.guessTagFromCustomType()
 		lhsIsCustom = true
 	}
 

pkg/yqlib/operator_anchors_aliases_test.go

@@ -188,6 +188,16 @@ var anchorOperatorScenarios = []expressionScenario{
 			"D0, P[b], (!!str)::cat\n",
 		},
 	},
+	{
+		description: "Explode splat",
+		skipDoc:     true,
+		document:    `{a: &a cat, b: *a}`,
+		expression:  `explode(.)[]`,
+		expected: []string{
+			"D0, P[a], (!!str)::cat\n",
+			"D0, P[b], (!!str)::cat\n",
+		},
+	},
 	{
 		description: "Explode alias and anchor - check original parent",
 		skipDoc:     true,

pkg/yqlib/operator_assign.go

@@ -6,10 +6,6 @@ type assignPreferences struct {
 	ClobberCustomTags   bool
 }
 
-func PerformAssignment(context Context, path []interface{}) {
-
-}
-
 func assignUpdateFunc(prefs assignPreferences) crossFunctionCalculation {
 	return func(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 		if !prefs.OnlyWriteNull || lhs.Tag == "!!null" {

pkg/yqlib/operator_collect_object_test.go

@@ -28,6 +28,16 @@ var collectObjectOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!map)::dog: great\n",
 		},
 	},
+	{
+		description: "collect splat",
+		skipDoc:     true,
+		document:    `[{name: cat}, {name: dog}]`,
+		expression:  `.[] | {.name: "great"}[]`,
+		expected: []string{
+			"D0, P[cat], (!!str)::great\n",
+			"D0, P[dog], (!!str)::great\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		expression: `({} + {}) | (.b = 3)`,

pkg/yqlib/operator_collect_test.go

@@ -109,6 +109,14 @@ var collectOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::- 1\n- 2\n- 3\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		expression: `[1,2][]`,
+		expected: []string{
+			"D0, P[0], (!!int)::1\n",
+			"D0, P[1], (!!int)::2\n",
+		},
+	},
 	{
 		document:   `a: {b: [1,2,3]}`,
 		expression: `[.a.b.[]]`,

pkg/yqlib/operator_compare.go

@@ -1,6 +1,7 @@
 package yqlib
 
 import (
+	"container/list"
 	"fmt"
 	"strconv"
 )
@@ -69,8 +70,8 @@ func compareDateTime(layout string, prefs compareTypePref, lhs *CandidateNode, r
 }
 
 func compareScalars(context Context, prefs compareTypePref, lhs *CandidateNode, rhs *CandidateNode) (bool, error) {
-	lhsTag := lhs.GuessTagFromCustomType()
-	rhsTag := rhs.GuessTagFromCustomType()
+	lhsTag := lhs.guessTagFromCustomType()
+	rhsTag := rhs.guessTagFromCustomType()
 
 	isDateTime := lhs.Tag == "!!timestamp"
 	// if the lhs is a string, it might be a timestamp in a custom format.
@@ -129,3 +130,40 @@ func compareScalars(context Context, prefs compareTypePref, lhs *CandidateNode,
 
 	return false, fmt.Errorf("%v not yet supported for comparison", lhs.Tag)
 }
+
+func superlativeByComparison(d *dataTreeNavigator, context Context, prefs compareTypePref) (Context, error) {
+	fn := compare(prefs)
+
+	var results = list.New()
+
+	for seq := context.MatchingNodes.Front(); seq != nil; seq = seq.Next() {
+		splatted, err := splat(context.SingleChildContext(seq.Value.(*CandidateNode)), traversePreferences{})
+		if err != nil {
+			return Context{}, err
+		}
+		result := splatted.MatchingNodes.Front()
+		if result != nil {
+			for el := result.Next(); el != nil; el = el.Next() {
+				cmp, err := fn(d, context, el.Value.(*CandidateNode), result.Value.(*CandidateNode))
+				if err != nil {
+					return Context{}, err
+				}
+				if isTruthyNode(cmp) {
+					result = el
+				}
+			}
+			results.PushBack(result.Value)
+		}
+	}
+	return context.ChildContext(results), nil
+}
+
+func minOperator(d *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {
+	log.Debug(("Min"))
+	return superlativeByComparison(d, context, compareTypePref{Greater: false})
+}
+
+func maxOperator(d *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {
+	log.Debug(("Max"))
+	return superlativeByComparison(d, context, compareTypePref{Greater: true})
+}

pkg/yqlib/operator_datetime.go

@@ -141,7 +141,7 @@ func fromUnixOp(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Conte
 	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
 		candidate := el.Value.(*CandidateNode)
 
-		actualTag := candidate.GuessTagFromCustomType()
+		actualTag := candidate.guessTagFromCustomType()
 
 		if actualTag != "!!int" && actualTag != "!!float" {
 			return Context{}, fmt.Errorf("from_unix only works on numbers, found %v instead", candidate.Tag)

pkg/yqlib/operator_divide.go

@@ -32,11 +32,11 @@ func divide(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateN
 
 func divideScalars(target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error {
 	lhsTag := lhs.Tag
-	rhsTag := rhs.GuessTagFromCustomType()
+	rhsTag := rhs.guessTagFromCustomType()
 	lhsIsCustom := false
 	if !strings.HasPrefix(lhsTag, "!!") {
 		// custom tag - we have to have a guess
-		lhsTag = lhs.GuessTagFromCustomType()
+		lhsTag = lhs.guessTagFromCustomType()
 		lhsIsCustom = true
 	}
 

pkg/yqlib/operator_encoder_decoder.go

@@ -9,11 +9,33 @@ import (
 	"strings"
 )
 
+func configureEncoder(format *Format, indent int) Encoder {
+
+	switch format {
+	case JSONFormat:
+		prefs := ConfiguredJSONPreferences.Copy()
+		prefs.Indent = indent
+		prefs.ColorsEnabled = false
+		prefs.UnwrapScalar = false
+		return NewJSONEncoder(prefs)
+	case YamlFormat:
+		var prefs = ConfiguredYamlPreferences.Copy()
+		prefs.Indent = indent
+		prefs.ColorsEnabled = false
+		return NewYamlEncoder(prefs)
+	case XMLFormat:
+		var xmlPrefs = ConfiguredXMLPreferences.Copy()
+		xmlPrefs.Indent = indent
+		return NewXMLEncoder(xmlPrefs)
+	}
+	return format.EncoderFactory()
+}
+
 func encodeToString(candidate *CandidateNode, prefs encoderPreferences) (string, error) {
 	var output bytes.Buffer
 	log.Debug("printing with indent: %v", prefs.indent)
 
-	encoder := prefs.format.GetInDocumentEncoder(prefs.indent)
+	encoder := configureEncoder(prefs.format, prefs.indent)
 	if encoder == nil {
 		return "", errors.New("no support for output format")
 	}

pkg/yqlib/operator_entries.go

@@ -64,7 +64,7 @@ func toEntriesOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode)
 func parseEntry(candidateNode *CandidateNode, position int) (*CandidateNode, *CandidateNode, error) {
 	prefs := traversePreferences{DontAutoCreate: true}
 
-	keyResults, err := traverseMap(Context{}, candidateNode, CreateStringScalarNode("key"), prefs, false)
+	keyResults, err := traverseMap(Context{}, candidateNode, createStringScalarNode("key"), prefs, false)
 
 	if err != nil {
 		return nil, nil, err
@@ -72,7 +72,7 @@ func parseEntry(candidateNode *CandidateNode, position int) (*CandidateNode, *Ca
 		return nil, nil, fmt.Errorf("expected to find one 'key' entry but found %v in position %v", keyResults.Len(), position)
 	}
 
-	valueResults, err := traverseMap(Context{}, candidateNode, CreateStringScalarNode("value"), prefs, false)
+	valueResults, err := traverseMap(Context{}, candidateNode, createStringScalarNode("value"), prefs, false)
 
 	if err != nil {
 		return nil, nil, err

pkg/yqlib/operator_env_test.go

@@ -5,6 +5,16 @@ import (
 )
 
 var envOperatorScenarios = []expressionScenario{
+	{
+		description:          "Read string environment variable",
+		skipDoc:              true,
+		environmentVariables: map[string]string{"myenv": "[cat,dog]"},
+		expression:           `env(myenv)[]`,
+		expected: []string{
+			"D0, P[0], (!!str)::cat\n",
+			"D0, P[1], (!!str)::dog\n",
+		},
+	},
 	{
 		description:          "Read string environment variable",
 		environmentVariables: map[string]string{"myenv": "cat meow"},

pkg/yqlib/operator_eval_test.go

@@ -14,6 +14,15 @@ var evalOperatorScenarios = []expressionScenario{
 			"D0, P[a b 1], (!!map)::{name: cat}\n",
 		},
 	},
+	{
+		description: "eval splat",
+		skipDoc:     true,
+		document:    `{pathExp: '.a.b[] | select(.name == "cat")', a: {b: [{name: dog}, {name: cat}]}}`,
+		expression:  `eval(.pathExp)[]`,
+		expected: []string{
+			"D0, P[a b 1 name], (!!str)::cat\n",
+		},
+	},
 	{
 		description:          "Dynamically update a path from an environment variable",
 		subdescription:       "The env variable can be any valid yq expression.",

pkg/yqlib/operator_filter_test.go

@@ -13,6 +13,16 @@ var filterOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::[1, 2]\n",
 		},
 	},
+	{
+		description: "Filter array splat",
+		skipDoc:     true,
+		document:    `[1,2,3]`,
+		expression:  `filter(. < 3)[]`,
+		expected: []string{
+			"D0, P[0], (!!int)::1\n",
+			"D0, P[1], (!!int)::2\n",
+		},
+	},
 	{
 		description: "Filter map values",
 		document:    `{c: {things: cool, frog: yes}, d: {things: hot, frog: false}}`,

pkg/yqlib/operator_flatten_test.go

@@ -14,6 +14,17 @@ var flattenOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::[1, 2, 3]\n",
 		},
 	},
+	{
+		description: "Flatten splat",
+		skipDoc:     true,
+		document:    `[1, [2], [[3]]]`,
+		expression:  `flatten[]`,
+		expected: []string{
+			"D0, P[0], (!!int)::1\n",
+			"D0, P[0], (!!int)::2\n",
+			"D0, P[0], (!!int)::3\n",
+		},
+	},
 	{
 		description: "Flatten with depth of one",
 		document:    `[1, [2], [[3]]]`,
@@ -22,6 +33,17 @@ var flattenOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::[1, 2, [3]]\n",
 		},
 	},
+	{
+		description: "Flatten with depth and splat",
+		skipDoc:     true,
+		document:    `[1, [2], [[3]]]`,
+		expression:  `flatten(1)[]`,
+		expected: []string{
+			"D0, P[0], (!!int)::1\n",
+			"D0, P[0], (!!int)::2\n",
+			"D0, P[0], (!!seq)::[3]\n",
+		},
+	},
 	{
 		description: "Flatten empty array",
 		document:    `[[]]`,

pkg/yqlib/operator_group_by_test.go

@@ -13,6 +13,16 @@ var groupByOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::- - {foo: 1, bar: 10}\n  - {foo: 1, bar: 1}\n- - {foo: 3, bar: 100}\n",
 		},
 	},
+	{
+		description: "Group splat",
+		skipDoc:     true,
+		document:    `[{foo: 1, bar: 10}, {foo: 3, bar: 100}, {foo: 1, bar: 1}]`,
+		expression:  `group_by(.foo)[]`,
+		expected: []string{
+			"D0, P[0], (!!seq)::- {foo: 1, bar: 10}\n- {foo: 1, bar: 1}\n",
+			"D0, P[1], (!!seq)::- {foo: 3, bar: 100}\n",
+		},
+	},
 	{
 		description: "Group by field, with nulls",
 		document:    `[{cat: dog}, {foo: 1, bar: 10}, {foo: 3, bar: 100}, {no: foo for you}, {foo: 1, bar: 1}]`,

pkg/yqlib/operator_keys_test.go

@@ -39,6 +39,16 @@ var keysOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::- dog\n- cat\n",
 		},
 	},
+	{
+		description: "Map keys with splat",
+		skipDoc:     true,
+		document:    `{dog: woof, cat: meow}`,
+		expression:  `keys[]`,
+		expected: []string{
+			"D0, P[dog], (!!str)::dog\n",
+			"D0, P[cat], (!!str)::cat\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `{}`,

pkg/yqlib/operator_map_test.go

@@ -15,6 +15,15 @@ var mapOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::[6, 7, 8]\n",
 		},
 	},
+	{
+		skipDoc:    true,
+		document:   `[1,2]`,
+		expression: `map(. + 1)[]`,
+		expected: []string{
+			"D0, P[0], (!!int)::2\n",
+			"D0, P[1], (!!int)::3\n",
+		},
+	},
 	{
 		description: "Map array",
 		document:    `[1,2,3]`,
@@ -33,6 +42,16 @@ var mapOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!map)::{x: 11, y: 21, z: 31}\n",
 		},
 	},
+	{
+		description: "map values splat",
+		skipDoc:     true,
+		document:    `{a: 1, b: 2}`,
+		expression:  `map_values(. + 1)[]`,
+		expected: []string{
+			"D0, P[a], (!!int)::2\n",
+			"D0, P[b], (!!int)::3\n",
+		},
+	},
 	{
 		description: "Map object values",
 		document:    `{a: 1, b: 2, c: 3}`,

pkg/yqlib/operator_modulo.go

@@ -33,11 +33,11 @@ func modulo(_ *dataTreeNavigator, _ Context, lhs *CandidateNode, rhs *CandidateN
 
 func moduloScalars(target *CandidateNode, lhs *CandidateNode, rhs *CandidateNode) error {
 	lhsTag := lhs.Tag
-	rhsTag := rhs.GuessTagFromCustomType()
+	rhsTag := rhs.guessTagFromCustomType()
 	lhsIsCustom := false
 	if !strings.HasPrefix(lhsTag, "!!") {
 		// custom tag - we have to have a guess
-		lhsTag = lhs.GuessTagFromCustomType()
+		lhsTag = lhs.guessTagFromCustomType()
 		lhsIsCustom = true
 	}
 

pkg/yqlib/operator_multiply.go

@@ -79,11 +79,11 @@ func multiply(preferences multiplyPreferences) func(d *dataTreeNavigator, contex
 
 func multiplyScalars(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
 	lhsTag := lhs.Tag
-	rhsTag := rhs.GuessTagFromCustomType()
+	rhsTag := rhs.guessTagFromCustomType()
 	lhsIsCustom := false
 	if !strings.HasPrefix(lhsTag, "!!") {
 		// custom tag - we have to have a guess
-		lhsTag = lhs.GuessTagFromCustomType()
+		lhsTag = lhs.guessTagFromCustomType()
 		lhsIsCustom = true
 	}
 
@@ -91,6 +91,8 @@ func multiplyScalars(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, er
 		return multiplyIntegers(lhs, rhs)
 	} else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") {
 		return multiplyFloats(lhs, rhs, lhsIsCustom)
+	} else if (lhsTag == "!!str" && rhsTag == "!!int") || (lhsTag == "!!int" && rhsTag == "!!str") {
+		return repeatString(lhs, rhs)
 	}
 	return nil, fmt.Errorf("cannot multiply %v with %v", lhs.Tag, rhs.Tag)
 }
@@ -135,6 +137,28 @@ func multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, e
 	return target, nil
 }
 
+func repeatString(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	var stringNode *CandidateNode
+	var intNode *CandidateNode
+	if lhs.Tag == "!!str" {
+		stringNode = lhs
+		intNode = rhs
+	} else {
+		stringNode = rhs
+		intNode = lhs
+	}
+	target := lhs.CopyWithoutContent()
+	target.UpdateAttributesFrom(stringNode, assignPreferences{})
+
+	count, err := parseInt(intNode.Value)
+	if err != nil {
+		return nil, err
+	}
+	target.Value = strings.Repeat(stringNode.Value, count)
+
+	return target, nil
+}
+
 func mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode, preferences multiplyPreferences) (*CandidateNode, error) {
 	var results = list.New()
 

pkg/yqlib/operator_multiply_test.go

@@ -1,6 +1,8 @@
 package yqlib
 
 import (
+	"fmt"
+	"strings"
 	"testing"
 )
 
@@ -174,6 +176,40 @@ var multiplyOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!map)::a: 12\nb: 4\n",
 		},
 	},
+	{
+		description: "Multiply string node X int",
+		document:    docNoComments,
+		expression:  ".b * 4",
+		expected: []string{
+			fmt.Sprintf("D0, P[b], (!!str)::%s\n", strings.Repeat("banana", 4)),
+		},
+	},
+	{
+		description: "Multiply int X string node",
+		document:    docNoComments,
+		expression:  "4 * .b",
+		expected: []string{
+			fmt.Sprintf("D0, P[], (!!str)::%s\n", strings.Repeat("banana", 4)),
+		},
+	},
+	{
+		description: "Multiply string X int node",
+		document: `n: 4
+`,
+		expression: `"banana" * .n`,
+		expected: []string{
+			fmt.Sprintf("D0, P[], (!!str)::%s\n", strings.Repeat("banana", 4)),
+		},
+	},
+	{
+		description: "Multiply int node X string",
+		document: `n: 4
+`,
+		expression: `.n * "banana"`,
+		expected: []string{
+			fmt.Sprintf("D0, P[n], (!!str)::%s\n", strings.Repeat("banana", 4)),
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   doc1,

pkg/yqlib/operator_omit.go

@@ -0,0 +1,73 @@
+package yqlib
+
+import (
+	"container/list"
+	"strconv"
+)
+
+func omitMap(original *CandidateNode, indices *CandidateNode) *CandidateNode {
+	filteredContent := make([]*CandidateNode, 0, max(0, len(original.Content)-len(indices.Content)*2))
+
+	for index := 0; index < len(original.Content); index += 2 {
+		pos := findInArray(indices, original.Content[index])
+		if pos < 0 {
+			clonedKey := original.Content[index].Copy()
+			clonedValue := original.Content[index+1].Copy()
+			filteredContent = append(filteredContent, clonedKey, clonedValue)
+		}
+	}
+	result := original.CopyWithoutContent()
+	result.AddChildren(filteredContent)
+	return result
+}
+
+func omitSequence(original *CandidateNode, indices *CandidateNode) *CandidateNode {
+	filteredContent := make([]*CandidateNode, 0, max(0, len(original.Content)-len(indices.Content)))
+
+	for index := 0; index < len(original.Content); index++ {
+		pos := findInArray(indices, createScalarNode(index, strconv.Itoa(index)))
+		if pos < 0 {
+			filteredContent = append(filteredContent, original.Content[index].Copy())
+		}
+	}
+	result := original.CopyWithoutContent()
+	result.AddChildren(filteredContent)
+	return result
+}
+
+func omitOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	log.Debugf("Omit")
+
+	contextIndicesToOmit, err := d.GetMatchingNodes(context, expressionNode.RHS)
+
+	if err != nil {
+		return Context{}, err
+	}
+	indicesToOmit := &CandidateNode{}
+	if contextIndicesToOmit.MatchingNodes.Len() > 0 {
+		indicesToOmit = contextIndicesToOmit.MatchingNodes.Front().Value.(*CandidateNode)
+	}
+	if len(indicesToOmit.Content) == 0 {
+		log.Debugf("No omit indices specified")
+		return context, nil
+	}
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		node := el.Value.(*CandidateNode)
+
+		var replacement *CandidateNode
+
+		if node.Kind == MappingNode {
+			replacement = omitMap(node, indicesToOmit)
+		} else if node.Kind == SequenceNode {
+			replacement = omitSequence(node, indicesToOmit)
+		} else {
+			log.Debugf("Omit from type %v (%v) is noop", node.Tag, node.GetNicePath())
+			return context, nil
+		}
+		replacement.LeadingContent = node.LeadingContent
+		results.PushBack(replacement)
+	}
+	return context.ChildContext(results), nil
+}

pkg/yqlib/operator_omit_test.go

@@ -0,0 +1,70 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var omitOperatorScenarios = []expressionScenario{
+	{
+		description:    "Omit keys from map",
+		subdescription: "Note that non existent keys are skipped.",
+		document:       "myMap: {cat: meow, dog: bark, thing: hamster, hamster: squeak}\n",
+		expression:     `.myMap |= omit(["hamster", "cat", "goat"])`,
+		expected: []string{
+			"D0, P[], (!!map)::myMap: {dog: bark, thing: hamster}\n",
+		},
+	},
+	{
+		description: "Omit splat",
+		skipDoc:     true,
+		document:    "{cat: meow, dog: bark, hamster: squeak}\n",
+		expression:  `omit(["dog"])[]`,
+		expected: []string{
+			"D0, P[cat], (!!str)::meow\n",
+			"D0, P[hamster], (!!str)::squeak\n",
+		},
+	},
+	{
+		description: "Omit keys from map",
+		skipDoc:     true,
+		document:    "!things myMap: {cat: meow, dog: bark, thing: hamster, hamster: squeak}\n",
+		expression:  `.myMap |= omit(["hamster", "cat", "goat"])`,
+		expected: []string{
+			"D0, P[], (!!map)::!things myMap: {dog: bark, thing: hamster}\n",
+		},
+	},
+	{
+		description: "Omit keys from map with comments",
+		skipDoc:     true,
+		document:    "# abc\nmyMap: {cat: meow, dog: bark, thing: hamster, hamster: squeak}\n# xyz\n",
+		expression:  `.myMap |= omit(["hamster", "cat", "goat"])`,
+		expected: []string{
+			"D0, P[], (!!map)::# abc\nmyMap: {dog: bark, thing: hamster}\n# xyz\n",
+		},
+	},
+	{
+		description:    "Omit indices from array",
+		subdescription: "Note that non existent indices are skipped.",
+		document:       `[cat, leopard, lion]`,
+		expression:     `omit([2, 0, 734, -5])`,
+		expected: []string{
+			"D0, P[], (!!seq)::[leopard]\n",
+		},
+	},
+	{
+		description: "Omit indices from array with comments",
+		skipDoc:     true,
+		document:    "# abc\n[cat, leopard, lion]\n# xyz",
+		expression:  `omit([2, 0, 734, -5])`,
+		expected: []string{
+			"D0, P[], (!!seq)::# abc\n[leopard]\n# xyz\n",
+		},
+	},
+}
+
+func TestOmitOperatorScenarios(t *testing.T) {
+	for _, tt := range omitOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentOperatorScenarios(t, "omit", omitOperatorScenarios)
+}

pkg/yqlib/operator_path_test.go

@@ -23,6 +23,16 @@ var pathOperatorScenarios = []expressionScenario{
 			"D0, P[a b], (!!seq)::- a\n- b\n",
 		},
 	},
+	{
+		description: "Map path",
+		document:    `{a: {b: cat}}`,
+		skipDoc:     true,
+		expression:  `.a.b | path[]`,
+		expected: []string{
+			"D0, P[a b 0], (!!str)::a\n",
+			"D0, P[a b 1], (!!str)::b\n",
+		},
+	},
 	{
 		skipDoc: true,
 		document: `a:
@@ -129,6 +139,15 @@ var pathOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!map)::a: [frog]\n",
 		},
 	},
+	{
+		description: "Delete splat",
+		skipDoc:     true,
+		document:    `a: [cat, frog]`,
+		expression:  `delpaths([["a", 0]])[]`,
+		expected: []string{
+			"D0, P[a], (!!seq)::[frog]\n",
+		},
+	},
 	{
 		description:    "Delete - wrong parameter",
 		subdescription: "delpaths does not work with a single path array",

pkg/yqlib/operator_pick_test.go

@@ -14,6 +14,16 @@ var pickOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!map)::myMap: {hamster: squeak, cat: meow}\n",
 		},
 	},
+	{
+		description: "Pick splat",
+		skipDoc:     true,
+		document:    "{cat: meow, dog: bark, thing: hamster, hamster: squeak}\n",
+		expression:  `pick(["hamster", "cat"])[]`,
+		expected: []string{
+			"D0, P[hamster], (!!str)::squeak\n",
+			"D0, P[cat], (!!str)::meow\n",
+		},
+	},
 	{
 		description: "Pick keys from map",
 		skipDoc:     true,

pkg/yqlib/operator_pivot.go

@@ -0,0 +1,121 @@
+package yqlib
+
+import (
+	"container/list"
+	"fmt"
+)
+
+func getUniqueElementTag(seq *CandidateNode) (string, error) {
+	switch l := len(seq.Content); l {
+	case 0:
+		return "", nil
+	default:
+		result := seq.Content[0].Tag
+		for i := 1; i < l; i++ {
+			t := seq.Content[i].Tag
+			if t != result {
+				return "", fmt.Errorf("sequence contains elements of %v and %v types", result, t)
+			}
+		}
+		return result, nil
+	}
+}
+
+var nullNodeFactory = func() *CandidateNode { return createScalarNode(nil, "") }
+
+func pad[E any](array []E, length int, factory func() E) []E {
+	sz := len(array)
+	if sz >= length {
+		return array
+	}
+	pad := make([]E, length-sz)
+	for i := 0; i < len(pad); i++ {
+		pad[i] = factory()
+	}
+	return append(array, pad...)
+}
+
+func pivotSequences(seq *CandidateNode) *CandidateNode {
+	sz := len(seq.Content)
+	if sz == 0 {
+		return seq
+	}
+	m := make(map[int][]*CandidateNode)
+
+	for i := 0; i < sz; i++ {
+		row := seq.Content[i]
+		for j := 0; j < len(row.Content); j++ {
+			e := m[j]
+			if e == nil {
+				e = make([]*CandidateNode, 0, sz)
+			}
+			m[j] = append(pad(e, i, nullNodeFactory), row.Content[j])
+		}
+	}
+	result := CandidateNode{Kind: SequenceNode}
+
+	for i := 0; i < len(m); i++ {
+		e := CandidateNode{Kind: SequenceNode}
+		e.AddChildren(pad(m[i], sz, nullNodeFactory))
+		result.AddChild(&e)
+	}
+	return &result
+}
+
+func pivotMaps(seq *CandidateNode) *CandidateNode {
+	sz := len(seq.Content)
+	if sz == 0 {
+		return &CandidateNode{Kind: MappingNode}
+	}
+	m := make(map[string][]*CandidateNode)
+	keys := make([]string, 0)
+
+	for i := 0; i < sz; i++ {
+		row := seq.Content[i]
+		for j := 0; j < len(row.Content); j += 2 {
+			k := row.Content[j].Value
+			v := row.Content[j+1]
+			e := m[k]
+			if e == nil {
+				keys = append(keys, k)
+				e = make([]*CandidateNode, 0, sz)
+			}
+			m[k] = append(pad(e, i, nullNodeFactory), v)
+		}
+	}
+	result := CandidateNode{Kind: MappingNode}
+	for _, k := range keys {
+		pivotRow := CandidateNode{Kind: SequenceNode}
+		pivotRow.AddChildren(
+			pad(m[k], sz, nullNodeFactory))
+		result.AddKeyValueChild(createScalarNode(k, k), &pivotRow)
+	}
+	return &result
+}
+
+func pivotOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {
+	log.Debug("Pivot")
+	results := list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		if candidate.Tag != "!!seq" {
+			return Context{}, fmt.Errorf("cannot pivot node of type %v", candidate.Tag)
+		}
+		tag, err := getUniqueElementTag(candidate)
+		if err != nil {
+			return Context{}, err
+		}
+		var pivot *CandidateNode
+		switch tag {
+		case "!!seq":
+			pivot = pivotSequences(candidate)
+		case "!!map":
+			pivot = pivotMaps(candidate)
+		default:
+			return Context{}, fmt.Errorf("can only pivot elements of !!seq or !!map types, received %v", tag)
+		}
+		results.PushBack(pivot)
+	}
+	return context.ChildContext(results), nil
+}

pkg/yqlib/operator_pivot_test.go

@@ -0,0 +1,57 @@
+package yqlib
+
+import "testing"
+
+var pivotOperatorScenarios = []expressionScenario{
+	{
+		description: "Pivot a sequence of sequences",
+		document:    "[[foo, bar, baz], [sis, boom, bah]]\n",
+		expression:  `pivot`,
+		expected: []string{
+			"D0, P[], ()::- - foo\n  - sis\n- - bar\n  - boom\n- - baz\n  - bah\n",
+		},
+	},
+	{
+		description: "Pivot splat",
+		skipDoc:     true,
+		document:    "[[foo, bar], [sis, boom]]\n",
+		expression:  `pivot[]`,
+		expected: []string{
+			"D0, P[0], ()::- foo\n- sis\n",
+			"D0, P[1], ()::- bar\n- boom\n",
+		},
+	},
+	{
+		description:    "Pivot sequence of heterogeneous sequences",
+		subdescription: `Missing values are "padded" to null.`,
+		document:       "[[foo, bar, baz], [sis, boom, bah, blah]]\n",
+		expression:     `pivot`,
+		expected: []string{
+			"D0, P[], ()::- - foo\n  - sis\n- - bar\n  - boom\n- - baz\n  - bah\n- -\n  - blah\n",
+		},
+	},
+	{
+		description: "Pivot sequence of maps",
+		document:    "[{foo: a, bar: b, baz: c}, {foo: x, bar: y, baz: z}]\n",
+		expression:  `pivot`,
+		expected: []string{
+			"D0, P[], ()::foo:\n    - a\n    - x\nbar:\n    - b\n    - y\nbaz:\n    - c\n    - z\n",
+		},
+	},
+	{
+		description:    "Pivot sequence of heterogeneous maps",
+		subdescription: `Missing values are "padded" to null.`,
+		document:       "[{foo: a, bar: b, baz: c}, {foo: x, bar: y, baz: z, what: ever}]\n",
+		expression:     `pivot`,
+		expected: []string{
+			"D0, P[], ()::foo:\n    - a\n    - x\nbar:\n    - b\n    - y\nbaz:\n    - c\n    - z\nwhat:\n    -\n    - ever\n",
+		},
+	},
+}
+
+func TestPivotOperatorScenarios(t *testing.T) {
+	for _, tt := range pivotOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentOperatorScenarios(t, "pivot", pivotOperatorScenarios)
+}

pkg/yqlib/operator_reverse_test.go

@@ -11,6 +11,16 @@ var reverseOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!seq)::[3, 2, 1]\n",
 		},
 	},
+	{
+		description: "Reverse",
+		skipDoc:     true,
+		document:    "[1, 2]",
+		expression:  `reverse[]`,
+		expected: []string{
+			"D0, P[1], (!!int)::2\n",
+			"D0, P[0], (!!int)::1\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   "[]",

pkg/yqlib/operator_select_test.go

@@ -75,6 +75,17 @@ var selectOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!map)::b: world\n",
 		},
 	},
+	{
+		description: "select splat",
+		skipDoc:     true,
+		document:    "a: hello",
+		document2:   "b: world",
+		expression:  `select(.a == "hello" or .b == "world")[]`,
+		expected: []string{
+			"D0, P[a], (!!str)::hello\n",
+			"D0, P[b], (!!str)::world\n",
+		},
+	},
 	{
 		description: "select does not update the map",
 		skipDoc:     true,