Bumping version

The package has been renamed from `yq` to `yq-go` to match other distros.

CONTRIBUTING.md

@@ -1,4 +1,4 @@
-# Development
+.# Development
 
 1. Install (Golang)[https://golang.org/]
 1. Run  `scripts/devtools.sh` to install the required devtools
@@ -25,6 +25,8 @@ The pipeline will run the tests and automatically concatenate the files together
 
 The first step is to find if what you want is automatically generated or not - start by looking in the master branch. 
 
+Note that PRs with small changes (e.g. minor typos) may not be merged (see https://joel.net/how-one-guy-ruined-hacktoberfest2020-drama).
+
 ### Updating dynamic documentation from master
 - Search for the documentation you want to update. If you find matches in a `*_test.go` file - update that, as that will automatically update the matching `*.md` file 
 - Assuming you are updating a `*_test.go` file, once updated, run the test to regenerated the docs. E.g. for the 'Add' test generated docs, from the pkg/yqlib folder run:

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.22.1 as builder
+FROM golang:1.22.3 as builder
 
 WORKDIR /go/src/mikefarah/yq
 

Dockerfile.dev

@@ -1,12 +1,15 @@
-FROM golang:1.22.1
+FROM golang:1.22.3
+
+RUN apt-get update && \
+    apt-get install -y npm && \
+    npm install -g npx cspell@latest
 
 COPY scripts/devtools.sh /opt/devtools.sh
 
-RUN set -e -x \
-    && /opt/devtools.sh
-ENV PATH=/go/bin:$PATH
+RUN set -e -x && \
+    /opt/devtools.sh
 
-RUN apt-get update && apt-get install -y npm && npm install -g npx cspell@latest
+ENV PATH=/go/bin:$PATH
 
 ENV CGO_ENABLED 0
 ENV GOPATH /go:/yq

Makefile.variables

@@ -26,6 +26,7 @@ ifeq ($(CYG_CHECK),1)
 else
 	# all non-windows environments
 	ROOT := $(shell pwd)
+	SELINUX := $(shell which getenforce 2>&1 >/dev/null && echo :z)
 endif
 
 DEV_IMAGE := ${PROJECT}_dev
@@ -33,7 +34,7 @@ DEV_IMAGE := ${PROJECT}_dev
 ENGINERUN := ${ENGINE} run --rm \
 	-e LDFLAGS="${LDFLAGS}" \
 	-e GITHUB_TOKEN="${GITHUB_TOKEN}" \
-	-v ${ROOT}/vendor:/go/src \
-	-v ${ROOT}:/${PROJECT}/src/${IMPORT_PATH} \
+	-v ${ROOT}/vendor:/go/src${SELINUX} \
+	-v ${ROOT}:/${PROJECT}/src/${IMPORT_PATH}${SELINUX} \
 	-w /${PROJECT}/src/${IMPORT_PATH} \
 	${DEV_IMAGE}

README.md

@@ -295,12 +295,18 @@ sudo port install yq
 Supported by @herbygillot (https://ports.macports.org/maintainer/github/herbygillot)
 
 ### Alpine Linux
-- Enable edge/community repo by adding ```$MIRROR/alpine/edge/community``` to ```/etc/apk/repositories```
-- Update database index with ```apk update```
-- Install yq with ```apk add yq```
 
-Supported by Tuan Hoang
-https://pkgs.alpinelinux.org/package/edge/community/x86/yq
+Alpine Linux v3.20+ (and Edge):
+```
+apk add yq-go
+```
+
+Alpine Linux up to v3.19:
+```
+apk add yq
+```
+
+Supported by Tuan Hoang (https://pkgs.alpinelinux.org/packages?name=yq-go)
 
 
 ## Features
@@ -349,7 +355,6 @@ Available Commands:
   eval             (default) Apply the expression to each document in each yaml file in sequence
   eval-all         Loads _all_ yaml documents of _all_ yaml files and runs expression once
   help             Help about any command
-  shell-completion Generate completion script
 
 Flags:
   -C, --colors                        force print with colors

cmd/completion.go

@@ -7,19 +7,20 @@ import (
 )
 
 var completionCmd = &cobra.Command{
-	Use:   "shell-completion [bash|zsh|fish|powershell]",
-	Short: "Generate completion script",
+	Use:     "completion [bash|zsh|fish|powershell]",
+	Aliases: []string{"shell-completion"},
+	Short:   "Generate the autocompletion script for the specified shell",
 	Long: `To load completions:
 
 Bash:
 
-$ source <(yq shell-completion bash)
+$ source <(yq completion bash)
 
 # To load completions for each session, execute once:
 Linux:
-  $ yq shell-completion bash > /etc/bash_completion.d/yq
+  $ yq completion bash > /etc/bash_completion.d/yq
 MacOS:
-  $ yq shell-completion bash > /usr/local/etc/bash_completion.d/yq
+  $ yq completion bash > /usr/local/etc/bash_completion.d/yq
 
 Zsh:
 
@@ -29,16 +30,16 @@ Zsh:
 $ echo "autoload -U compinit; compinit" >> ~/.zshrc
 
 # To load completions for each session, execute once:
-$ yq shell-completion zsh > "${fpath[1]}/_yq"
+$ yq completion zsh > "${fpath[1]}/_yq"
 
 # You will need to start a new shell for this setup to take effect.
 
 Fish:
 
-$ yq shell-completion fish | source
+$ yq completion fish | source
 
 # To load completions for each session, execute once:
-$ yq shell-completion fish > ~/.config/fish/completions/yq.fish
+$ yq completion fish > ~/.config/fish/completions/yq.fish
 `,
 	DisableFlagsInUseLine: true,
 	ValidArgs:             []string{"bash", "zsh", "fish", "powershell"},

cmd/utils.go

@@ -75,10 +75,6 @@ func initCommand(cmd *cobra.Command, args []string) (string, []string, error) {
 				outputFormat = "yaml"
 			}
 		} else if isAutomaticOutputFormat() {
-			// automatic input worked, we can do it for output too unless specified
-			if inputFormat == "json" {
-				yqlib.GetLogger().Warning("JSON file output is now JSON by default (instead of yaml). Use '-oy' or '--output-format=yaml' for yaml output")
-			}
 			outputFormat = inputFormat
 		}
 	} else if isAutomaticOutputFormat() {

cmd/version.go

@@ -11,7 +11,7 @@ var (
 	GitDescribe string
 
 	// Version is main version number that is being run at the moment.
-	Version = "v4.42.1"
+	Version = "v4.44.1"
 
 	// VersionPrerelease is a pre-release marker for the version. If this is "" (empty string)
 	// then it means that it is a final release. Otherwise, this is a pre-release

debian/changelog

@@ -1,210 +0,0 @@
-yq (4.16.2) focal; urgency=medium
-
-  * Fixed with semicolon space issue
-  * Updating with documentation
-  * Added STDIN example to the top
-  * minor readme cleanup
-  * Help text tweak
-  * Fixed docker timeout - simplify docker builds
-  * New release with docker build fixes
-  * Updating to go 1.17 to fix CVE #944
-  * Fix a typo in root.go
-  * Skip the tests if the nocheck Debian build option is specified
-  * Fixed select bug (#958)
-  * Sped up explode operator
-  * Slight performance improvement to context.ChildContext
-  * Speed up multiply
-  * Update README with recently added / changed options
-  * Make deepMatch report in linear time
-  * Removed leadingContentPreProcessing flag - header preprocessing is stable
-  * Revert "Removed leadingContentPreProcessing flag - header preprocessing is stable"
-  * Keep flag, it is needed in corner cases
-  * Updated Readme
-  * Man page
-  * Fixed expression parsing bug #970
-  * Bumping go-lang, docker versions
-  * Added test release flow
-  * Updated github action release to generate man page
-  * Bumping version
-  * Removing no longer needed github action
-  * Added decoder op
-  * Fixed newline handling when decoding/encoding
-  * Fixed newline handling in encoder/decoder
-  * Can specify indent in encode ops
-  * Added group_by operator
-  * Added flatten operator
-  * Fixed flatten error message
-  * Improving docs
-  * Split printer
-  * Refactored command logic
-  * Fix JSON encoding removing null #985
-  * Fixed acceptance tests
-  * gitbook
-  * Update document generation script
-  * Updating README
-  * Updating release instructions
-  * github action no longer uses data1.yml
-  * Create dependabot.yml
-  * Bump actions/create-release from 1.0.0 to 1.1.4
-  * Bump actions/setup-go from 1 to 2.1.4
-  * Bump github.com/goccy/go-yaml from 1.8.9 to 1.9.4
-  * Bump github.com/jinzhu/copier from 0.2.8 to 0.3.2
-  * Bump github.com/fatih/color from 1.10.0 to 1.13.0
-  * Bump github.com/spf13/cobra from 1.1.3 to 1.2.1
-  * Update dependabot.yml
-  * Update go.yml
-  * add build check to PRs
-  * Include secure as part of build process
-  * Fixing bad label in github action
-  * fixed printer test
-  * remove leading content indicator
-  * Fixed header preprocessing!
-  * lint : define golangci configuration file
-  * Update check.sh
-  * Load file acceptance test
-  * Minor improvement on handling front matter
-  * Improved load doc
-  * feature: detect MANPATh and install there
-  * Update install-man-page.sh
-  * simplify prod stage, move version label to action
-  * add labels, quote some values
-  * enable errorlint linter
-  * Added errorlint to devtools
-  * Added key operator
-  * Added more tests
-  * Fixing comments
-  * Attempt to fix golint problem
-  * Include version query for tools
-  * Clean up errored file?
-  * enable misspell linter
-  * updated readme
-  * update Golangci version to v1.43.0
-  * gci linter
-  * Better merge array by key example
-  * Added credit for merge by array example
-  * Better formatting of merge arrays example
-  * Better merge example
-  * Add accessor for the yq logger instance (#1013)
-  * Fixed collect op when working with multiple nodes
-  * Added map, map_values
-  * Add support for Podman as well as Docker (#1026)
-  * Bump github.com/jinzhu/copier from 0.3.2 to 0.3.4 (#1027)
-  * Added csv, tsv output formats
-  * Added encoder tests
-  * Cleanup test
-  * Fixed docker permission issue #1014
-  * Recording release notes for next release
-  * Assignment op no longer clobbers anchor (#1029)
-  * Added sort_by operator
-  * Improved error message
-  * Improved tips and tricks
-  * Report while filename failed to parse #1030
-  * Added script for extracting checksums
-  * Improved extract-checksum.sh
-  * Bump github.com/spf13/cobra from 1.2.1 to 1.3.0 (#1039)
-  * enable more linters (#1043)
-  * Bump Golang compiler #1037
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Tue, 21 Dec 2021 09:41:44 +0000
-
-yq (4.13.0) focal; urgency=medium
-
-  * New `with` operator for making multiple changes to a given path
-  * New `contains` operator, works like the `jq` equivalent
-  * Subtract operator now supports subtracting elements from arrays!
-  * Fixed Swapping values using variables #934
-  * Github Action now properly supports multiline output #936, thanks @pjxiao
-  * Fixed missing closing bracket validation #932
-  * Fix processing of hex numbers #929
-  * Fixed alternative and union operator issues #930
-  * Can now convert yaml to properties properties format (`-o=props`), See [docs](https://mikefarah.gitbook.io/yq/v/v4.x/usage/properties) for more info.
-  * Fixed document header/footer comment handling when merging (https://github.com/mikefarah/yq/issues/919)
-  * pretty print yaml 1.1 compatibility  (https://github.com/mikefarah/yq/issues/914)
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Thu, 16 Sep 2021 20:58:30 +0200
-
-yq (4.9.6) focal; urgency=medium
-
-  * Added darwin/arm64 build, thanks @alecthomas
-  * Incremented docker alpine base version, thanks @da6d6i7-bronga
-  * Bug fix: multiline expression
-  * Bug fix: special character
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Tue, 29 Jun 2021 21:32:14 +0200
-
-yq (3.3.2) focal; urgency=medium
-
-  * Bug fix: existStatus bug (#459)
-  * Automatically makes an os temp directory if it does not exist (#461)
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Fri, 07 Aug 2020 18:53:01 +0200
-
-yq (3.3-0) focal; urgency=medium
-
-  * You can control string styles (quotes) using the new --style flag
-  * String values now always have quotes when outputting to json
-  * Negative array indices now traverse the array backwards
-  * Added a --stripComments flag to print yaml without any comments
-  * Bumped go to version 1.14
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Thu, 30 Apr 2020 20:45:44 +0200
-
-yq (3.1-2) eoan; urgency=medium
-
-  * Bug fix: yq 3 was removing empty inline-style objects and arrays (#355)
-  * Bug fix: Merge option returned different output when switching order of 
-    merging files(#347)
-  * Bug fix: Add new object to existing array object was failing in 3.1.1 (#361)
-  * Bug fix: yq 3 empty keys did not allow merging of values (#356)
-  * Bug fix: keys quoted during merge (#363)
-  * Bug fix: Correct length with wc -l (#362)
-  * Bug fix: Write to empty document removed path (#359)
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Mon, 24 Feb 2020 20:31:58 +0100
-
-yq (3.1-1) eoan; urgency=medium
-
-  * Keeps yaml comments and formatting, can specify yaml tags when updating.
-  * Handles anchors
-  * Can print out matching paths and values when splatting
-  * JSON output works for all commands
-  * Yaml files with multiple documents are printed out as one JSON 
-    document per line.
-  * Deep splat (**) to match arbitrary paths
-  * Update scripts file format has changed to be more powerful
-  * Reading and splatting, matching results are printed once per line
-  * Bugfixing
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Tue, 11 Feb 2020 22:18:24 +0100
-
-yq (2.2-1) bionic; urgency=medium
-
-  * Added Windows support for the "--inplace" command flag
-  * Prefix now supports arrays
-  * Add prefix command
-  * Bump Alpine version to 3.8
-  * Improved docker build process
-  * Lint fixes
-  * Build support for all linux architectures supported by gox
-
- -- Roberto Mier Escandon <rmescandon@gmail.com>  Sat, 19 Jan 2019 15:50:47 +0100
-
-yq (2.1-0) bionic; urgency=medium
-
-  * Ability to read multiple documents in a single file
-  * Ability to append list items instead of overwriting
-
- -- Roberto Mier Escandón <rmescandon@gmail.com>  Tue, 10 Jul 2018 14:02:42 +0200
-
-yq (2.0-0) bionic; urgency=medium
-
-  * Release 2.0.0
-
- -- Roberto Mier Escandón <rmescandon@gmail.com>  Wed, 20 Jun 2018 10:29:53 +0200
-
-yq (1.15-0) bionic; urgency=medium
-
-  * Release 1.15
-
- -- Roberto Mier Escandón <rmescandon@gmail.com>  Wed, 06 Jun 2018 11:32:03 +0200
-

debian/compat

@@ -1 +0,0 @@
-10

debian/control

@@ -1,22 +0,0 @@
-Source: yq
-Section: devel
-Priority: optional
-Maintainer: Roberto Mier Escandón <rmescandon@gmail.com>
-Build-Depends: debhelper (>=10),
-               golang-1.17-go,
-               pandoc,
-               rsync
-Standards-Version: 4.1.4
-Homepage: https://github.com/mikefarah/yq.git
-Vcs-Browser: https://github.com/mikefarah/yq.git
-Vcs-Git: https://github.com/mikefarah/yq.git
-XS-Go-Import-Path: github.com/mikefarah/yq
-XSBC-Original-Maintainer: Roberto Mier Escandón <rmescandon@gmail.com>
-
-Package: yq
-Architecture: any
-Depends: ${shlibs:Depends}, ${misc:Depends}
-Description: lightweight and portable command-line data file processor
- .
- The aim of the project is to be the 
- [jq](https://github.com/stedolan/jq) or sed of yaml files.

debian/copyright

@@ -1,24 +0,0 @@
-Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-Upstream-Name: yq
-Source: https://github.com/mikefarah/yq.git
-
-Files: *
-Copyright: 2017 Mike Farah
-License: Expat
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"), to deal
-    in the Software without restriction, including without limitation the rights
-    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-    copies of the Software, and to permit persons to whom the Software is
-    furnished to do so, subject to the following conditions:
-    .
-    The above copyright notice and this permission notice shall be included in all
-    copies or substantial portions of the Software.
-    .
-    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-    SOFTWARE.

debian/gbp.conf

@@ -1,2 +0,0 @@
-[DEFAULT]
-pristine-tar = True

debian/rules

@@ -1,74 +0,0 @@
-#!/usr/bin/make -f
-#
-# Copyright (C) 2018-2021 Roberto Mier Escandón <rmescandon@gmail.com>
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License version 3 as
-# published by the Free Software Foundation.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-PROJECT := yq
-OWNER := mikefarah
-REPO := github.com
-
-export DH_OPTIONS
-export DH_GOPKG := ${REPO}/${OWNER}/${PROJECT}
-export GOROOT := /usr/local/go
-export GOPATH := ${CURDIR}/_build
-export GOBIN := ${GOPATH}/bin
-export PATH := ${GOROOT}/bin:${GOBIN}:${PATH}
-export GOCACHE := /tmp/gocache
-export GOFLAGS := -mod=vendor
-export GO111MODULE := on
-
-SRCDIR := ${GOPATH}/src/${DH_GOPKG}
-DESTDIR := ${CURDIR}/debian/${PROJECT}
-BINDIR := /usr/bin
-MANDIR := /usr/share/man/man1/
-ASSETSDIR := /usr/share/${PROJECT}
-
-%:
-	dh $@
-
-override_dh_auto_build:
-	mkdir -p ${SRCDIR}
-	mkdir -p ${GOBIN}
-	# copy project to local srcdir to build from there
-	rsync -avz --progress --exclude=_build --exclude=debian --exclude=tmp. --exclude=go.mod --exclude=docs . $(SRCDIR)
-	# build go code
-	( \
-		cd ${SRCDIR} && \
-		go install -buildmode=pie ./... \
-	)
-
-	# build man page
-	( \
-		cd ${SRCDIR} && \
-		./scripts/generate-man-page-md.sh && \
-		./scripts/generate-man-page.sh \
-	)
-	
-override_dh_auto_test:
-ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
-	(cd ${SRCDIR} && go test -v ./...)
-endif
-
-override_dh_auto_install:
-	cp ${GOBIN}/yq ${DESTDIR}/${BINDIR}
-	cp -f ${SRCDIR}/LICENSE ${DESTDIR}/${ASSETSDIR}
-	chmod a+x ${DESTDIR}/${BINDIR}/yq
-
-	# man
-	mkdir -p "${DESTDIR}"/"${MANDIR}"
-	cp "${SRCDIR}"/yq.1 "${DESTDIR}"/"${MANDIR}" \
-
-override_dh_auto_clean:
-	dh_clean
-	rm -rf ${CURDIR}/_build

debian/source/format

@@ -1 +0,0 @@
-3.0 (native)

debian/yq.dirs

@@ -1,3 +0,0 @@
-usr/bin
-usr/share/yq
-usr/share/man/man1

go.mod

@@ -11,13 +11,13 @@ require (
 	github.com/goccy/go-yaml v1.11.3
 	github.com/jinzhu/copier v0.4.0
 	github.com/magiconair/properties v1.8.7
-	github.com/pelletier/go-toml/v2 v2.1.1
+	github.com/pelletier/go-toml/v2 v2.2.2
 	github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
 	github.com/yuin/gopher-lua v1.1.1
-	golang.org/x/net v0.22.0
-	golang.org/x/text v0.14.0
+	golang.org/x/net v0.24.0
+	golang.org/x/text v0.15.0
 	gopkg.in/op/go-logging.v1 v1.0.0-20160211212156-b2cb9fa56473
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -26,7 +26,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 )
 

go.sum

@@ -43,8 +43,8 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
-github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
-github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
+github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
+github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e h1:aoZm08cpOy4WuID//EZDgcC4zIxODThtZNPirFr42+A=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -57,23 +57,25 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
 github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
-golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
-golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
+golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
+golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
+golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
+golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
+golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk=
 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=

pkg/yqlib/chown_not_linux_os.go

@@ -7,6 +7,6 @@ import (
 	"os"
 )
 
-func changeOwner(info fs.FileInfo, file *os.File) error {
+func changeOwner(_ fs.FileInfo, _ *os.File) error {
 	return nil
 }

pkg/yqlib/doc/operators/headers/max.md

@@ -0,0 +1,3 @@
+# Max
+
+Computes the maximum among an incoming sequence of scalar values.

pkg/yqlib/doc/operators/headers/min.md

@@ -0,0 +1,3 @@
+# Min
+
+Computes the minimum among an incoming sequence of scalar values.

pkg/yqlib/doc/operators/headers/omit.md

@@ -0,0 +1,3 @@
+# Omit
+
+Works like `pick`, but instead you specify the keys/indices that you _don't_ want included.

pkg/yqlib/doc/operators/headers/pivot.md

@@ -0,0 +1,3 @@
+# Pivot
+
+Emulates the `PIVOT` function supported by several popular RDBMS systems.

pkg/yqlib/doc/operators/headers/slice-array.md

@@ -2,4 +2,4 @@
 
 The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.
 
-You may leave out the first or second number, which will will refer to the start or end of the array respectively.
+You may leave out the first or second number, which will refer to the start or end of the array respectively.

pkg/yqlib/doc/operators/max.md

@@ -0,0 +1,51 @@
+# Max
+
+Computes the maximum among an incoming sequence of scalar values.
+
+## Maximum int
+Given a sample.yml file of:
+```yaml
+- 99
+- 16
+- 12
+- 6
+- 66
+```
+then
+```bash
+yq 'max' sample.yml
+```
+will output
+```yaml
+99
+```
+
+## Maximum string
+Given a sample.yml file of:
+```yaml
+- foo
+- bar
+- baz
+```
+then
+```bash
+yq 'max' sample.yml
+```
+will output
+```yaml
+foo
+```
+
+## Maximum of empty
+Given a sample.yml file of:
+```yaml
+[]
+```
+then
+```bash
+yq 'max' sample.yml
+```
+will output
+```yaml
+```
+

pkg/yqlib/doc/operators/min.md

@@ -0,0 +1,51 @@
+# Min
+
+Computes the minimum among an incoming sequence of scalar values.
+
+## Minimum int
+Given a sample.yml file of:
+```yaml
+- 99
+- 16
+- 12
+- 6
+- 66
+```
+then
+```bash
+yq 'min' sample.yml
+```
+will output
+```yaml
+6
+```
+
+## Minimum string
+Given a sample.yml file of:
+```yaml
+- foo
+- bar
+- baz
+```
+then
+```bash
+yq 'min' sample.yml
+```
+will output
+```yaml
+bar
+```
+
+## Minimum of empty
+Given a sample.yml file of:
+```yaml
+[]
+```
+then
+```bash
+yq 'min' sample.yml
+```
+will output
+```yaml
+```
+

pkg/yqlib/doc/operators/multiply-merge.md

@@ -55,6 +55,62 @@ a: 12
 b: 4
 ```
 
+## Multiply string node X int
+Given a sample.yml file of:
+```yaml
+b: banana
+```
+then
+```bash
+yq '.b * 4' sample.yml
+```
+will output
+```yaml
+bananabananabananabanana
+```
+
+## Multiply int X string node
+Given a sample.yml file of:
+```yaml
+b: banana
+```
+then
+```bash
+yq '4 * .b' sample.yml
+```
+will output
+```yaml
+bananabananabananabanana
+```
+
+## Multiply string X int node
+Given a sample.yml file of:
+```yaml
+n: 4
+```
+then
+```bash
+yq '"banana" * .n' sample.yml
+```
+will output
+```yaml
+bananabananabananabanana
+```
+
+## Multiply int node X string
+Given a sample.yml file of:
+```yaml
+n: 4
+```
+then
+```bash
+yq '.n * "banana"' sample.yml
+```
+will output
+```yaml
+bananabananabananabanana
+```
+
 ## Merge objects together, returning merged result only
 Given a sample.yml file of:
 ```yaml

pkg/yqlib/doc/operators/omit.md

@@ -0,0 +1,44 @@
+# Omit
+
+Works like `pick`, but instead you specify the keys/indices that you _don't_ want included.
+
+## Omit keys from map
+Note that non existent keys are skipped.
+
+Given a sample.yml file of:
+```yaml
+myMap:
+  cat: meow
+  dog: bark
+  thing: hamster
+  hamster: squeak
+```
+then
+```bash
+yq '.myMap |= omit(["hamster", "cat", "goat"])' sample.yml
+```
+will output
+```yaml
+myMap:
+  dog: bark
+  thing: hamster
+```
+
+## Omit indices from array
+Note that non existent indices are skipped.
+
+Given a sample.yml file of:
+```yaml
+- cat
+- leopard
+- lion
+```
+then
+```bash
+yq 'omit([2, 0, 734, -5])' sample.yml
+```
+will output
+```yaml
+- leopard
+```
+

pkg/yqlib/doc/operators/pivot.md

@@ -0,0 +1,117 @@
+# Pivot
+
+Emulates the `PIVOT` function supported by several popular RDBMS systems.
+
+## Pivot a sequence of sequences
+Given a sample.yml file of:
+```yaml
+- - foo
+  - bar
+  - baz
+- - sis
+  - boom
+  - bah
+```
+then
+```bash
+yq 'pivot' sample.yml
+```
+will output
+```yaml
+- - foo
+  - sis
+- - bar
+  - boom
+- - baz
+  - bah
+```
+
+## Pivot sequence of heterogeneous sequences
+Missing values are "padded" to null.
+
+Given a sample.yml file of:
+```yaml
+- - foo
+  - bar
+  - baz
+- - sis
+  - boom
+  - bah
+  - blah
+```
+then
+```bash
+yq 'pivot' sample.yml
+```
+will output
+```yaml
+- - foo
+  - sis
+- - bar
+  - boom
+- - baz
+  - bah
+- -
+  - blah
+```
+
+## Pivot sequence of maps
+Given a sample.yml file of:
+```yaml
+- foo: a
+  bar: b
+  baz: c
+- foo: x
+  bar: y
+  baz: z
+```
+then
+```bash
+yq 'pivot' sample.yml
+```
+will output
+```yaml
+foo:
+  - a
+  - x
+bar:
+  - b
+  - y
+baz:
+  - c
+  - z
+```
+
+## Pivot sequence of heterogeneous maps
+Missing values are "padded" to null.
+
+Given a sample.yml file of:
+```yaml
+- foo: a
+  bar: b
+  baz: c
+- foo: x
+  bar: y
+  baz: z
+  what: ever
+```
+then
+```bash
+yq 'pivot' sample.yml
+```
+will output
+```yaml
+foo:
+  - a
+  - x
+bar:
+  - b
+  - y
+baz:
+  - c
+  - z
+what:
+  -
+  - ever
+```
+

pkg/yqlib/doc/operators/slice-array.md

@@ -2,7 +2,7 @@
 
 The slice array operator takes an array as input and returns a subarray. Like the `jq` equivalent, `.[10:15]` will return an array of length 5, starting from index 10 inclusive, up to index 15 exclusive. Negative numbers count backwards from the end of the array.
 
-You may leave out the first or second number, which will will refer to the start or end of the array respectively.
+You may leave out the first or second number, which will refer to the start or end of the array respectively.
 
 ## Slicing arrays
 Given a sample.yml file of:

pkg/yqlib/doc/usage/properties.md

@@ -230,6 +230,23 @@ person:
     - pizza
 ```
 
+## Decode properties: numbers
+All values are assumed to be strings when parsing properties, but you can use the `from_yaml` operator on all the strings values to autoparse into the correct type.
+
+Given a sample.properties file of:
+```properties
+a.b = 10
+```
+then
+```bash
+yq -p=props ' (.. | select(tag == "!!str")) |= from_yaml' sample.properties
+```
+will output
+```yaml
+a:
+  b: 10
+```
+
 ## Decode properties - array should be a map
 If you have a numeric map key in your property files, use array_to_map to convert them to maps.
 

pkg/yqlib/format.go

@@ -123,9 +123,11 @@ func FormatStringFromFilename(filename string) string {
 }
 
 func FormatFromString(format string) (*Format, error) {
-	for _, printerFormat := range Formats {
-		if printerFormat.MatchesName(format) {
-			return printerFormat, nil
+	if format != "" {
+		for _, printerFormat := range Formats {
+			if printerFormat.MatchesName(format) {
+				return printerFormat, nil
+			}
 		}
 	}
 	return nil, fmt.Errorf("unknown format '%v' please use [%v]", format, GetAvailableOutputFormatString())

pkg/yqlib/format_test.go

@@ -0,0 +1,52 @@
+package yqlib
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/mikefarah/yq/v4/test"
+)
+
+type formatStringScenario struct {
+	description    string
+	input          string
+	expectedFormat *Format
+	expectedError  string
+}
+
+var formatStringScenarios = []formatStringScenario{
+	{
+		description:    "yaml",
+		input:          "yaml",
+		expectedFormat: YamlFormat,
+	},
+	{
+		description:   "Unknown format type",
+		input:         "doc",
+		expectedError: "unknown format 'doc' please use",
+	},
+	{
+		description:   "blank should error",
+		input:         "",
+		expectedError: "unknown format '' please use",
+	},
+}
+
+func TestFormatFromString(t *testing.T) {
+	for _, tt := range formatStringScenarios {
+		actualFormat, actualError := FormatFromString(tt.input)
+
+		if tt.expectedError != "" {
+			if actualError == nil {
+				t.Errorf("Expected [%v] error but found none", tt.expectedError)
+			} else {
+				test.AssertResultWithContext(t, true, strings.Contains(actualError.Error(), tt.expectedError),
+					fmt.Sprintf("Expected [%v] to contain [%v]", actualError.Error(), tt.expectedError),
+				)
+			}
+		} else {
+			test.AssertResult(t, tt.expectedFormat, actualFormat)
+		}
+	}
+}

pkg/yqlib/lexer_participle.go

@@ -40,6 +40,7 @@ var participleYqRules = []*participleYqRule{
 	simpleOp("map", mapOpType),
 	simpleOp("filter", filterOpType),
 	simpleOp("pick", pickOpType),
+	simpleOp("omit", omitOpType),
 
 	{"FlattenWithDepth", `flatten\([0-9]+\)`, flattenWithDepth(), 0},
 	{"Flatten", `flatten`, opTokenWithPrefs(flattenOpType, nil, flattenPreferences{depth: -1}), 0},
@@ -198,6 +199,9 @@ var participleYqRules = []*participleYqRule{
 	{"GreaterThan", `\s*>\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: false, Greater: true}), 0},
 	{"LessThan", `\s*<\s*`, opTokenWithPrefs(compareOpType, nil, compareTypePref{OrEqual: false, Greater: false}), 0},
 
+	simpleOp("min", minOpType),
+	simpleOp("max", maxOpType),
+
 	{"AssignRelative", `\|=[c]*`, assignOpToken(true), 0},
 	{"Assign", `=[c]*`, assignOpToken(false), 0},
 
@@ -223,6 +227,8 @@ var participleYqRules = []*participleYqRule{
 	{"SubtractAssign", `\-=`, opToken(subtractAssignOpType), 0},
 	{"Subtract", `\-`, opToken(subtractOpType), 0},
 	{"Comment", `#.*`, nil, 0},
+
+	simpleOp("pivot", pivotOpType),
 }
 
 type yqAction func(lexer.Token) (*token, error)

pkg/yqlib/operation.go

@@ -73,6 +73,8 @@ var equalsOpType = &operationType{Type: "EQUALS", NumArgs: 2, Precedence: 40, Ha
 var notEqualsOpType = &operationType{Type: "NOT_EQUALS", NumArgs: 2, Precedence: 40, Handler: notEqualsOperator}
 
 var compareOpType = &operationType{Type: "COMPARE", NumArgs: 2, Precedence: 40, Handler: compareOperator}
+var minOpType = &operationType{Type: "MIN", NumArgs: 0, Precedence: 40, Handler: minOperator}
+var maxOpType = &operationType{Type: "MAX", NumArgs: 0, Precedence: 40, Handler: maxOperator}
 
 // createmap needs to be above union, as we use union to build the components of the objects
 var createMapOpType = &operationType{Type: "CREATE_MAP", NumArgs: 2, Precedence: 15, Handler: createMapOperator}
@@ -90,6 +92,7 @@ var mapOpType = &operationType{Type: "MAP", NumArgs: 1, Precedence: 50, Handler:
 var filterOpType = &operationType{Type: "FILTER", NumArgs: 1, Precedence: 50, Handler: filterOperator}
 var errorOpType = &operationType{Type: "ERROR", NumArgs: 1, Precedence: 50, Handler: errorOperator}
 var pickOpType = &operationType{Type: "PICK", NumArgs: 1, Precedence: 50, Handler: pickOperator}
+var omitOpType = &operationType{Type: "OMIT", NumArgs: 1, Precedence: 50, Handler: omitOperator}
 var evalOpType = &operationType{Type: "EVAL", NumArgs: 1, Precedence: 50, Handler: evalOperator}
 var mapValuesOpType = &operationType{Type: "MAP_VALUES", NumArgs: 1, Precedence: 50, Handler: mapValuesOperator}
 
@@ -189,6 +192,8 @@ var groupByOpType = &operationType{Type: "GROUP_BY", NumArgs: 1, Precedence: 50,
 var flattenOpType = &operationType{Type: "FLATTEN_BY", NumArgs: 0, Precedence: 50, Handler: flattenOp}
 var deleteChildOpType = &operationType{Type: "DELETE", NumArgs: 1, Precedence: 40, Handler: deleteChildOperator}
 
+var pivotOpType = &operationType{Type: "PIVOT", NumArgs: 0, Precedence: 50, Handler: pivotOperator}
+
 // debugging purposes only
 func (p *Operation) toString() string {
 	if p == nil {

pkg/yqlib/operator_compare.go

@@ -1,6 +1,7 @@
 package yqlib
 
 import (
+	"container/list"
 	"fmt"
 	"strconv"
 )
@@ -129,3 +130,40 @@ func compareScalars(context Context, prefs compareTypePref, lhs *CandidateNode,
 
 	return false, fmt.Errorf("%v not yet supported for comparison", lhs.Tag)
 }
+
+func superlativeByComparison(d *dataTreeNavigator, context Context, prefs compareTypePref) (Context, error) {
+	fn := compare(prefs)
+
+	var results = list.New()
+
+	for seq := context.MatchingNodes.Front(); seq != nil; seq = seq.Next() {
+		splatted, err := splat(context.SingleChildContext(seq.Value.(*CandidateNode)), traversePreferences{})
+		if err != nil {
+			return Context{}, err
+		}
+		result := splatted.MatchingNodes.Front()
+		if result != nil {
+			for el := result.Next(); el != nil; el = el.Next() {
+				cmp, err := fn(d, context, el.Value.(*CandidateNode), result.Value.(*CandidateNode))
+				if err != nil {
+					return Context{}, err
+				}
+				if isTruthyNode(cmp) {
+					result = el
+				}
+			}
+			results.PushBack(result.Value)
+		}
+	}
+	return context.ChildContext(results), nil
+}
+
+func minOperator(d *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {
+	log.Debug(("Min"))
+	return superlativeByComparison(d, context, compareTypePref{Greater: false})
+}
+
+func maxOperator(d *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {
+	log.Debug(("Max"))
+	return superlativeByComparison(d, context, compareTypePref{Greater: true})
+}

pkg/yqlib/operator_multiply.go

@@ -91,6 +91,8 @@ func multiplyScalars(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, er
 		return multiplyIntegers(lhs, rhs)
 	} else if (lhsTag == "!!int" || lhsTag == "!!float") && (rhsTag == "!!int" || rhsTag == "!!float") {
 		return multiplyFloats(lhs, rhs, lhsIsCustom)
+	} else if (lhsTag == "!!str" && rhsTag == "!!int") || (lhsTag == "!!int" && rhsTag == "!!str") {
+		return repeatString(lhs, rhs)
 	}
 	return nil, fmt.Errorf("cannot multiply %v with %v", lhs.Tag, rhs.Tag)
 }
@@ -135,6 +137,28 @@ func multiplyIntegers(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, e
 	return target, nil
 }
 
+func repeatString(lhs *CandidateNode, rhs *CandidateNode) (*CandidateNode, error) {
+	var stringNode *CandidateNode
+	var intNode *CandidateNode
+	if lhs.Tag == "!!str" {
+		stringNode = lhs
+		intNode = rhs
+	} else {
+		stringNode = rhs
+		intNode = lhs
+	}
+	target := lhs.CopyWithoutContent()
+	target.UpdateAttributesFrom(stringNode, assignPreferences{})
+
+	count, err := parseInt(intNode.Value)
+	if err != nil {
+		return nil, err
+	}
+	target.Value = strings.Repeat(stringNode.Value, count)
+
+	return target, nil
+}
+
 func mergeObjects(d *dataTreeNavigator, context Context, lhs *CandidateNode, rhs *CandidateNode, preferences multiplyPreferences) (*CandidateNode, error) {
 	var results = list.New()
 

pkg/yqlib/operator_multiply_test.go

@@ -1,6 +1,8 @@
 package yqlib
 
 import (
+	"fmt"
+	"strings"
 	"testing"
 )
 
@@ -174,6 +176,40 @@ var multiplyOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!map)::a: 12\nb: 4\n",
 		},
 	},
+	{
+		description: "Multiply string node X int",
+		document:    docNoComments,
+		expression:  ".b * 4",
+		expected: []string{
+			fmt.Sprintf("D0, P[b], (!!str)::%s\n", strings.Repeat("banana", 4)),
+		},
+	},
+	{
+		description: "Multiply int X string node",
+		document:    docNoComments,
+		expression:  "4 * .b",
+		expected: []string{
+			fmt.Sprintf("D0, P[], (!!str)::%s\n", strings.Repeat("banana", 4)),
+		},
+	},
+	{
+		description: "Multiply string X int node",
+		document: `n: 4
+`,
+		expression: `"banana" * .n`,
+		expected: []string{
+			fmt.Sprintf("D0, P[], (!!str)::%s\n", strings.Repeat("banana", 4)),
+		},
+	},
+	{
+		description: "Multiply int node X string",
+		document: `n: 4
+`,
+		expression: `.n * "banana"`,
+		expected: []string{
+			fmt.Sprintf("D0, P[n], (!!str)::%s\n", strings.Repeat("banana", 4)),
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   doc1,

pkg/yqlib/operator_omit.go

@@ -0,0 +1,73 @@
+package yqlib
+
+import (
+	"container/list"
+	"strconv"
+)
+
+func omitMap(original *CandidateNode, indices *CandidateNode) *CandidateNode {
+	filteredContent := make([]*CandidateNode, 0, max(0, len(original.Content)-len(indices.Content)*2))
+
+	for index := 0; index < len(original.Content); index += 2 {
+		pos := findInArray(indices, original.Content[index])
+		if pos < 0 {
+			clonedKey := original.Content[index].Copy()
+			clonedValue := original.Content[index+1].Copy()
+			filteredContent = append(filteredContent, clonedKey, clonedValue)
+		}
+	}
+	result := original.CopyWithoutContent()
+	result.AddChildren(filteredContent)
+	return result
+}
+
+func omitSequence(original *CandidateNode, indices *CandidateNode) *CandidateNode {
+	filteredContent := make([]*CandidateNode, 0, max(0, len(original.Content)-len(indices.Content)))
+
+	for index := 0; index < len(original.Content); index++ {
+		pos := findInArray(indices, createScalarNode(index, strconv.Itoa(index)))
+		if pos < 0 {
+			filteredContent = append(filteredContent, original.Content[index].Copy())
+		}
+	}
+	result := original.CopyWithoutContent()
+	result.AddChildren(filteredContent)
+	return result
+}
+
+func omitOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) {
+	log.Debugf("Omit")
+
+	contextIndicesToOmit, err := d.GetMatchingNodes(context, expressionNode.RHS)
+
+	if err != nil {
+		return Context{}, err
+	}
+	indicesToOmit := &CandidateNode{}
+	if contextIndicesToOmit.MatchingNodes.Len() > 0 {
+		indicesToOmit = contextIndicesToOmit.MatchingNodes.Front().Value.(*CandidateNode)
+	}
+	if len(indicesToOmit.Content) == 0 {
+		log.Debugf("No omit indices specified")
+		return context, nil
+	}
+	var results = list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		node := el.Value.(*CandidateNode)
+
+		var replacement *CandidateNode
+
+		if node.Kind == MappingNode {
+			replacement = omitMap(node, indicesToOmit)
+		} else if node.Kind == SequenceNode {
+			replacement = omitSequence(node, indicesToOmit)
+		} else {
+			log.Debugf("Omit from type %v (%v) is noop", node.Tag, node.GetNicePath())
+			return context, nil
+		}
+		replacement.LeadingContent = node.LeadingContent
+		results.PushBack(replacement)
+	}
+	return context.ChildContext(results), nil
+}

pkg/yqlib/operator_omit_test.go

@@ -0,0 +1,60 @@
+package yqlib
+
+import (
+	"testing"
+)
+
+var omitOperatorScenarios = []expressionScenario{
+	{
+		description:    "Omit keys from map",
+		subdescription: "Note that non existent keys are skipped.",
+		document:       "myMap: {cat: meow, dog: bark, thing: hamster, hamster: squeak}\n",
+		expression:     `.myMap |= omit(["hamster", "cat", "goat"])`,
+		expected: []string{
+			"D0, P[], (!!map)::myMap: {dog: bark, thing: hamster}\n",
+		},
+	},
+	{
+		description: "Omit keys from map",
+		skipDoc:     true,
+		document:    "!things myMap: {cat: meow, dog: bark, thing: hamster, hamster: squeak}\n",
+		expression:  `.myMap |= omit(["hamster", "cat", "goat"])`,
+		expected: []string{
+			"D0, P[], (!!map)::!things myMap: {dog: bark, thing: hamster}\n",
+		},
+	},
+	{
+		description: "Omit keys from map with comments",
+		skipDoc:     true,
+		document:    "# abc\nmyMap: {cat: meow, dog: bark, thing: hamster, hamster: squeak}\n# xyz\n",
+		expression:  `.myMap |= omit(["hamster", "cat", "goat"])`,
+		expected: []string{
+			"D0, P[], (!!map)::# abc\nmyMap: {dog: bark, thing: hamster}\n# xyz\n",
+		},
+	},
+	{
+		description:    "Omit indices from array",
+		subdescription: "Note that non existent indices are skipped.",
+		document:       `[cat, leopard, lion]`,
+		expression:     `omit([2, 0, 734, -5])`,
+		expected: []string{
+			"D0, P[], (!!seq)::[leopard]\n",
+		},
+	},
+	{
+		description: "Omit indices from array with comments",
+		skipDoc:     true,
+		document:    "# abc\n[cat, leopard, lion]\n# xyz",
+		expression:  `omit([2, 0, 734, -5])`,
+		expected: []string{
+			"D0, P[], (!!seq)::# abc\n[leopard]\n# xyz\n",
+		},
+	},
+}
+
+func TestOmitOperatorScenarios(t *testing.T) {
+	for _, tt := range omitOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentOperatorScenarios(t, "omit", omitOperatorScenarios)
+}

pkg/yqlib/operator_pivot.go

@@ -0,0 +1,121 @@
+package yqlib
+
+import (
+	"container/list"
+	"fmt"
+)
+
+func getUniqueElementTag(seq *CandidateNode) (string, error) {
+	switch l := len(seq.Content); l {
+	case 0:
+		return "", nil
+	default:
+		result := seq.Content[0].Tag
+		for i := 1; i < l; i++ {
+			t := seq.Content[i].Tag
+			if t != result {
+				return "", fmt.Errorf("sequence contains elements of %v and %v types", result, t)
+			}
+		}
+		return result, nil
+	}
+}
+
+var nullNodeFactory = func() *CandidateNode { return createScalarNode(nil, "") }
+
+func pad[E any](array []E, length int, factory func() E) []E {
+	sz := len(array)
+	if sz >= length {
+		return array
+	}
+	pad := make([]E, length-sz)
+	for i := 0; i < len(pad); i++ {
+		pad[i] = factory()
+	}
+	return append(array, pad...)
+}
+
+func pivotSequences(seq *CandidateNode) *CandidateNode {
+	sz := len(seq.Content)
+	if sz == 0 {
+		return seq
+	}
+	m := make(map[int][]*CandidateNode)
+
+	for i := 0; i < sz; i++ {
+		row := seq.Content[i]
+		for j := 0; j < len(row.Content); j++ {
+			e := m[j]
+			if e == nil {
+				e = make([]*CandidateNode, 0, sz)
+			}
+			m[j] = append(pad(e, i, nullNodeFactory), row.Content[j])
+		}
+	}
+	result := CandidateNode{Kind: SequenceNode}
+
+	for i := 0; i < len(m); i++ {
+		e := CandidateNode{Kind: SequenceNode}
+		e.AddChildren(pad(m[i], sz, nullNodeFactory))
+		result.AddChild(&e)
+	}
+	return &result
+}
+
+func pivotMaps(seq *CandidateNode) *CandidateNode {
+	sz := len(seq.Content)
+	if sz == 0 {
+		return &CandidateNode{Kind: MappingNode}
+	}
+	m := make(map[string][]*CandidateNode)
+	keys := make([]string, 0)
+
+	for i := 0; i < sz; i++ {
+		row := seq.Content[i]
+		for j := 0; j < len(row.Content); j += 2 {
+			k := row.Content[j].Value
+			v := row.Content[j+1]
+			e := m[k]
+			if e == nil {
+				keys = append(keys, k)
+				e = make([]*CandidateNode, 0, sz)
+			}
+			m[k] = append(pad(e, i, nullNodeFactory), v)
+		}
+	}
+	result := CandidateNode{Kind: MappingNode}
+	for _, k := range keys {
+		pivotRow := CandidateNode{Kind: SequenceNode}
+		pivotRow.AddChildren(
+			pad(m[k], sz, nullNodeFactory))
+		result.AddKeyValueChild(createScalarNode(k, k), &pivotRow)
+	}
+	return &result
+}
+
+func pivotOperator(_ *dataTreeNavigator, context Context, _ *ExpressionNode) (Context, error) {
+	log.Debug("Pivot")
+	results := list.New()
+
+	for el := context.MatchingNodes.Front(); el != nil; el = el.Next() {
+		candidate := el.Value.(*CandidateNode)
+		if candidate.Tag != "!!seq" {
+			return Context{}, fmt.Errorf("cannot pivot node of type %v", candidate.Tag)
+		}
+		tag, err := getUniqueElementTag(candidate)
+		if err != nil {
+			return Context{}, err
+		}
+		var pivot *CandidateNode
+		switch tag {
+		case "!!seq":
+			pivot = pivotSequences(candidate)
+		case "!!map":
+			pivot = pivotMaps(candidate)
+		default:
+			return Context{}, fmt.Errorf("can only pivot elements of !!seq or !!map types, received %v", tag)
+		}
+		results.PushBack(pivot)
+	}
+	return context.ChildContext(results), nil
+}

pkg/yqlib/operator_pivot_test.go

@@ -0,0 +1,47 @@
+package yqlib
+
+import "testing"
+
+var pivotOperatorScenarios = []expressionScenario{
+	{
+		description: "Pivot a sequence of sequences",
+		document:    "[[foo, bar, baz], [sis, boom, bah]]\n",
+		expression:  `pivot`,
+		expected: []string{
+			"D0, P[], ()::- - foo\n  - sis\n- - bar\n  - boom\n- - baz\n  - bah\n",
+		},
+	},
+	{
+		description:    "Pivot sequence of heterogeneous sequences",
+		subdescription: `Missing values are "padded" to null.`,
+		document:       "[[foo, bar, baz], [sis, boom, bah, blah]]\n",
+		expression:     `pivot`,
+		expected: []string{
+			"D0, P[], ()::- - foo\n  - sis\n- - bar\n  - boom\n- - baz\n  - bah\n- -\n  - blah\n",
+		},
+	},
+	{
+		description: "Pivot sequence of maps",
+		document:    "[{foo: a, bar: b, baz: c}, {foo: x, bar: y, baz: z}]\n",
+		expression:  `pivot`,
+		expected: []string{
+			"D0, P[], ()::foo:\n    - a\n    - x\nbar:\n    - b\n    - y\nbaz:\n    - c\n    - z\n",
+		},
+	},
+	{
+		description:    "Pivot sequence of heterogeneous maps",
+		subdescription: `Missing values are "padded" to null.`,
+		document:       "[{foo: a, bar: b, baz: c}, {foo: x, bar: y, baz: z, what: ever}]\n",
+		expression:     `pivot`,
+		expected: []string{
+			"D0, P[], ()::foo:\n    - a\n    - x\nbar:\n    - b\n    - y\nbaz:\n    - c\n    - z\nwhat:\n    -\n    - ever\n",
+		},
+	},
+}
+
+func TestPivotOperatorScenarios(t *testing.T) {
+	for _, tt := range pivotOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentOperatorScenarios(t, "pivot", pivotOperatorScenarios)
+}

pkg/yqlib/operator_strings.go

@@ -54,40 +54,52 @@ func interpolate(d *dataTreeNavigator, context Context, str string) (string, err
 	for i := 0; i < len(runes); i++ {
 		char := runes[i]
 		if !inExpression {
-			if char == '\\' && i != len(runes)-1 && runes[i+1] == '(' {
-				inExpression = true
-				i = i + 1 // skip over the next open bracket
-				continue
+			if char == '\\' && i < len(runes)-1 {
+				switch runes[i+1] {
+				case '(':
+					inExpression = true
+					// skip the lparen
+					i++
+					continue
+				case '\\':
+					// skip the escaped backslash
+					i++
+				default:
+					log.Debugf("Ignoring non-escaping backslash @ %v[%d]", str, i)
+				}
 			}
 			sb.WriteRune(char)
 		} else { // we are in an expression
-			if char == ')' && nestedBracketsCounter == 0 {
-				// finished the expression!
-				log.Debugf("Expression is :%v", expSb.String())
-				value, err := evaluate(d, context, expSb.String())
-				if err != nil {
-					return "", err
+			if char == ')' {
+				if nestedBracketsCounter == 0 {
+					// finished the expression!
+					log.Debugf("Expression is :%v", expSb.String())
+					value, err := evaluate(d, context, expSb.String())
+					if err != nil {
+						return "", err
+					}
+					inExpression = false
+					expSb = strings.Builder{} // reset this
+
+					sb.WriteString(value)
+					continue
 				}
-
-				inExpression = false
-				expSb = strings.Builder{} // reset this
-
-				sb.WriteString(value)
-				continue
+				nestedBracketsCounter--
 			} else if char == '(' {
 				nestedBracketsCounter++
-			} else if char == '\\' && i != len(runes)-1 && runes[i+1] == ')' {
-				// close brackets is escaped, skip over it
-				expSb.WriteRune(char)
-				expSb.WriteRune(runes[i+1])
-				i = i + 1
-				continue
-			} else if char == ')' {
-				nestedBracketsCounter--
+			} else if char == '\\' && i < len(runes)-1 {
+				switch esc := runes[i+1]; esc {
+				case ')', '\\':
+					// write escaped character
+					expSb.WriteRune(esc)
+					i++
+					continue
+				default:
+					log.Debugf("Ignoring non-escaping backslash @ %v[%d]", str, i)
+				}
 			}
 			expSb.WriteRune(char)
 		}
-
 	}
 	if inExpression {
 		return "", fmt.Errorf("unclosed interpolation string \\(")

pkg/yqlib/operator_strings_test.go

@@ -26,7 +26,7 @@ var stringsOperatorScenarios = []expressionScenario{
 		description: "Interpolation - just escape",
 		expression:  `"\\"`,
 		expected: []string{
-			"D0, P[], (!!str)::\\\\\n",
+			"D0, P[], (!!str)::\\\n",
 		},
 	},
 	{
@@ -47,6 +47,15 @@ var stringsOperatorScenarios = []expressionScenario{
 			"D0, P[], (!!str)::Hi (.value)\n",
 		},
 	},
+	{
+		skipDoc:     true,
+		description: "Interpolation - don't!",
+		document:    `value: things`,
+		expression:  `"Hi \\(.value)"`,
+		expected: []string{
+			"D0, P[], (!!str)::Hi \\(.value)\n",
+		},
+	},
 	{
 		skipDoc:     true,
 		description: "Interpolation - random close bracket",
@@ -63,6 +72,13 @@ var stringsOperatorScenarios = []expressionScenario{
 		expression:    `"Hi \("`,
 		expectedError: "unclosed interpolation string \\(",
 	},
+	{
+		skipDoc:       true,
+		description:   "Interpolation - unclosed interpolation string due to escape",
+		document:      `value: things`,
+		expression:    `"Hi \(\)"`,
+		expectedError: "unclosed interpolation string \\(",
+	},
 	{
 		description:    "To up (upper) case",
 		subdescription: "Works with unicode characters",

pkg/yqlib/operators_compare_test.go

@@ -383,3 +383,67 @@ func TestCompareOperatorScenarios(t *testing.T) {
 	}
 	documentOperatorScenarios(t, "compare", compareOperatorScenarios)
 }
+
+var minOperatorScenarios = []expressionScenario{
+	{
+		description: "Minimum int",
+		document:    "[99, 16, 12, 6, 66]\n",
+		expression:  `min`,
+		expected: []string{
+			"D0, P[3], (!!int)::6\n",
+		},
+	},
+	{
+		description: "Minimum string",
+		document:    "[foo, bar, baz]\n",
+		expression:  `min`,
+		expected: []string{
+			"D0, P[1], (!!str)::bar\n",
+		},
+	},
+	{
+		description: "Minimum of empty",
+		document:    "[]\n",
+		expression:  `min`,
+		expected:    []string{},
+	},
+}
+
+func TestMinOperatorScenarios(t *testing.T) {
+	for _, tt := range minOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentOperatorScenarios(t, "min", minOperatorScenarios)
+}
+
+var maxOperatorScenarios = []expressionScenario{
+	{
+		description: "Maximum int",
+		document:    "[99, 16, 12, 6, 66]\n",
+		expression:  `max`,
+		expected: []string{
+			"D0, P[0], (!!int)::99\n",
+		},
+	},
+	{
+		description: "Maximum string",
+		document:    "[foo, bar, baz]\n",
+		expression:  `max`,
+		expected: []string{
+			"D0, P[0], (!!str)::foo\n",
+		},
+	},
+	{
+		description: "Maximum of empty",
+		document:    "[]\n",
+		expression:  `max`,
+		expected:    []string{},
+	},
+}
+
+func TestMaxOperatorScenarios(t *testing.T) {
+	for _, tt := range maxOperatorScenarios {
+		testScenario(t, &tt)
+	}
+	documentOperatorScenarios(t, "max", maxOperatorScenarios)
+}

pkg/yqlib/properties_test.go

@@ -227,7 +227,14 @@ var propertyScenarios = []formatScenario{
 		expected:     "- person\n- name\n",
 		scenarioType: "decode",
 	},
-
+	{
+		description:    "Decode properties: numbers",
+		subdescription: "All values are assumed to be strings when parsing properties, but you can use the `from_yaml` operator on all the strings values to autoparse into the correct type.",
+		input:          "a.b = 10",
+		expression:     " (.. | select(tag == \"!!str\")) |= from_yaml",
+		expected:       "a:\n  b: 10\n",
+		scenarioType:   "decode",
+	},
 	{
 		description:    "Decode properties - array should be a map",
 		subdescription: "If you have a numeric map key in your property files, use array_to_map to convert them to maps.",

release_notes.txt

@@ -1,10 +1,22 @@
-4.43.1: (not released yet)
+4.44.1:
+  - Added min/max operators (#1992) Thanks @mbenson
+  - Added pivot oeprator (#1993) Thanks @mbenson
+  - Fix: shell-completion (#2006) Thanks @codekow
+  - Handle escaped backslashes (#1997) Thanks @mbenson
+  - Fix npe when given filename ending with "." (#1994)
+  - Fix: linux (w/ selinux) build (#2004) Thanks @codekow
+  - Bumped dependencies 
+
+4.43.1:
+  - Added omit operator #1989 thanks @mbenson!
   - Added tostring #72
   - Added string interpolation #1149
   - Can specify parent(n) levels #1970
+  - Can now multiply strings by numbers #1988  thanks @mbenson!
   - Fixed CSV line break issue #1974
   - Adding a EvaluateAll function to StringEvaluator #1966
   - yqlib, default to colors off when using yaml library #1964
+  - Removed JSON output warning
   - Bumped dependencies
 
 

snap/snapcraft.yaml

@@ -1,5 +1,5 @@
 name: yq
-version: 'v4.42.1'
+version: 'v4.44.1'
 summary: A lightweight and portable command-line data file processor
 description: |
   `yq` uses [jq](https://github.com/stedolan/jq) like syntax but works with yaml, json, xml, csv, properties and TOML files.
@@ -21,6 +21,6 @@ parts:
   yq:
     plugin: go
     source: https://github.com/mikefarah/yq.git
-    source-tag: v4.42.1
+    source-tag: v4.44.1
     build-snaps:
       - go/latest/stable