From 0b49bebad911bb3a563fd819545dc5ee26a445be Mon Sep 17 00:00:00 2001 From: robertomier Date: Thu, 23 Dec 2021 01:43:21 +0100 Subject: [PATCH 1/7] chore: script to build the deb sources (#1045) * chore: script to build the deb sources now it includes the man files. This script makes only sense executing on the machine having the gpg key needed to sign and push to ppa * fix: dput dependency and proper configuration --- .gitignore | 6 +- debian/control | 3 +- debian/rules | 23 +++- release_instructions.txt | 10 +- scripts/release-deb.sh | 264 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 299 insertions(+), 7 deletions(-) create mode 100755 scripts/release-deb.sh diff --git a/.gitignore b/.gitignore index 00b1ab74..0c2a0b48 100644 --- a/.gitignore +++ b/.gitignore @@ -48,4 +48,8 @@ test*.yml # man page man.md -yq.1 \ No newline at end of file +yq.1 + +# debian pkg +_build +debian/files diff --git a/debian/control b/debian/control index 9794eaab..625bfb3d 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,8 @@ Section: devel Priority: optional Maintainer: Roberto Mier Escandón Build-Depends: debhelper (>=10), - golang-1.15, + golang-1.17-go, + pandoc, rsync Standards-Version: 4.1.4 Homepage: https://github.com/mikefarah/yq.git diff --git a/debian/rules b/debian/rules index 912f8d33..4c40d6db 100755 --- a/debian/rules +++ b/debian/rules @@ -1,6 +1,6 @@ #!/usr/bin/make -f # -# Copyright (C) 2018 Roberto Mier Escandón +# Copyright (C) 2018-2021 Roberto Mier Escandón # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3 as @@ -17,20 +17,21 @@ PROJECT := yq OWNER := mikefarah REPO := github.com -GOVERSION := 1.15 export DH_OPTIONS export DH_GOPKG := ${REPO}/${OWNER}/${PROJECT} -export GOROOT := /usr/lib/go-${GOVERSION} +export GOROOT := /usr/local/go export GOPATH := ${CURDIR}/_build export GOBIN := ${GOPATH}/bin export PATH := ${GOROOT}/bin:${GOBIN}:${PATH} export GOCACHE := /tmp/gocache export GOFLAGS := -mod=vendor +export GO111MODULE := on SRCDIR := ${GOPATH}/src/${DH_GOPKG} DESTDIR := ${CURDIR}/debian/${PROJECT} BINDIR := /usr/bin +MANDIR := /usr/share/man/man1/ ASSETSDIR := /usr/share/${PROJECT} %: @@ -42,7 +43,17 @@ override_dh_auto_build: # copy project to local srcdir to build from there rsync -avz --progress --exclude=_build --exclude=debian --exclude=tmp. --exclude=go.mod --exclude=docs . $(SRCDIR) # build go code - (cd ${SRCDIR} && go install -buildmode=pie ./...) + ( \ + cd ${SRCDIR} && \ + go install -buildmode=pie ./... \ + ) + + # build man page + ( \ + cd ${SRCDIR} && \ + ./scripts/generate-man-page-md.sh && \ + ./scripts/generate-man-page.sh \ + ) override_dh_auto_test: ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS))) @@ -54,6 +65,10 @@ override_dh_auto_install: cp -f ${SRCDIR}/LICENSE ${DESTDIR}/${ASSETSDIR} chmod a+x ${DESTDIR}/${BINDIR}/yq + # man + mkdir -p "${DESTDIR}"/"${MANDIR}" + cp "${SRCDIR}"/yq.1 "${DESTDIR}"/"${MANDIR}" \ + override_dh_auto_clean: dh_clean rm -rf ${CURDIR}/_build diff --git a/release_instructions.txt b/release_instructions.txt index f0eedc74..0cbbda3f 100644 --- a/release_instructions.txt +++ b/release_instructions.txt @@ -33,7 +33,15 @@ - build and push latest and new version tag - docker build . -t mikefarah/yq:latest -t mikefarah/yq:3 -t mikefarah/yq:3.X -- debian package +- debian package (with release script) + - execute the script `./scripts/release-deb.sh` *on the machine having the private + gpg key to sign the generated sources* providing the version to release, the + ppa where deploying the files and the passphrase of the private key if needed: + ``` + ./scripts/release-deb.sh -o output -p -s --passphrase PASSPHRASE VERSION + ``` + +- debian package (manually) - ensure you get all vendor dependencies before packaging ```go mod vendor``` - execute diff --git a/scripts/release-deb.sh b/scripts/release-deb.sh new file mode 100755 index 00000000..e6c49fd1 --- /dev/null +++ b/scripts/release-deb.sh @@ -0,0 +1,264 @@ +#!/bin/bash -eux +# +# Copyright (C) 2021 Roberto Mier Escandón +# +# This script creates a .deb package file with yq valid for ubuntu 20.04 by default +# You can pass + +DOCKER_IMAGE_NAME=yq-deb-builder +DOCKER_IMAGE_TAG=$(git describe --always --tags) +OUTPUT= +GOVERSION="1.17.4" +KEYID= +MAINTAINER= +DO_PUBLISH= +PPA="rmescandon/yq" +VERSION= +DISTRIBUTION= +DO_SIGN= +PASSPHRASE= + +show_help() { + echo " usage: $(basename "$0") VERSION [options...]" + echo "" + echo " positional arguments" + echo " VERSION" + echo "" + echo " optional arguments:" + echo " -h, --help Shows this help" + echo " -d, --distribution DISTRO The distribution to use for the changelog generation. If not provided, last changelog entry" + echo " distribution is considered" + echo " --goversion VERSION The version of golang to use. Default to $GOVERSION" + echo " -k, --sign-key KEYID Sign the package sources with the provided gpg key id (long format). When not provided this" + echo " paramater, the generated sources are not signed" + echo " -s, --sign Sign the package sources with a gpg key of the maintainer" + echo " -m, --maintainer WHO The maintainer used as author of the changelog. git.name and git.email (see git config) is" + echo " the considered format" + echo " -o DIR, --output DIR The path where leaving the generated debian package. Default to a temporary folder if not set" + echo " -p The resultant file is being published to ppa" + echo " --ppa PPA Push resultant files to indicated ppa. This option should be given along with a signing key." + echo " Otherwise, the server could reject the package building. Default is set to 'rmescandon/yq'" + echo " --passphrase PASSPHRASE Passphrase to decrypt the signage key" + exit 1 +} +# read input args +while [ $# -ne 0 ]; do + case $1 in + -h|--help) + show_help + ;; + -d|--distribution) + shift + DISTRIBUTION="$1" + ;; + --goversion) + shift + GOVERSION="$1" + ;; + -k|--sign-key) + shift + DO_SIGN='y' + KEYID="$1" + ;; + -s|--sign) + DO_SIGN='y' + ;; + -m|--maintainer) + shift + MAINTAINER="$1" + ;; + -o|--output) + shift + OUTPUT="$1" + ;; + -p) + DO_PUBLISH="y" + ;; + --ppa) + shift + DO_PUBLISH="y" + PPA="$1" + ;; + --passphrase) + shift + PASSPHRASE="$1" + ;; + *) + if [ -z "$VERSION" ]; then + VERSION="$1" + else + show_help + fi + esac + shift +done + +[ -n "$VERSION" ] || (echo "error - you have to provide a version" && show_help) + +if [ -n "$OUTPUT" ]; then + OUTPUT="$(realpath "$OUTPUT")" + mkdir -p "$OUTPUT" +else + # Temporary folder where leaving the built deb package in case that output folder is not provided + OUTPUT="$(mktemp -d)" +fi + +# Define the folders with the source project and the build artifacts and files +srcdir="$(realpath "$(dirname "$0")"/..)" +blddir="$(cd "${srcdir}" && mkdir -p build && cd build && echo "$(pwd)")" +# clean on exit +cleanup() { + rm -f "${blddir}/build.sh" || true + rm -f "${blddir}/Dockerfile" || true + rm -f "${blddir}/dput.cf" || true + docker rmi "${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" -f > /dev/null 2>&1 || true +} +trap cleanup EXIT INT + +# configure the dput config in case publishing is requested +lp_id="$(echo "$PPA" | cut -d'/' -f1)" +ppa_id="$(echo "$PPA" | cut -d'/' -f2)" +cat << EOF > ${blddir}/dput.cf +[ppa] +fqdn = ppa.launchpad.net +method = ftp +incoming = ~${lp_id}/ubuntu/${ppa_id} +login = anonymous +EOF + +# create the main script +cat << EOF > ${blddir}/build.sh +#!/bin/bash +set -e -o pipefail + +PATH=$PATH:/usr/local/go/bin +export GPG_TTY=$(tty) + +go mod vendor + +### bump debian/changelog + +# maintainer +export DEBEMAIL="$MAINTAINER" +if [ -z "$MAINTAINER" ]; then + export DEBEMAIL="\$(dpkg-parsechangelog -S Maintainer)" +fi + +# prepend a 'v' char to complete the tag name from where calculating the changelog +SINCE="v\$(dpkg-parsechangelog -S Version)" + +# distribution +DISTRIBUTION="$DISTRIBUTION" +if [ -z "$DISTRIBUTION" ]; then + DISTRIBUTION="\$(dpkg-parsechangelog -S Distribution)" +fi + +# generate changelog +gbp dch --ignore-branch --no-multimaint -N "$VERSION" -s "\$SINCE" -D "\$DISTRIBUTION" + +# using -d to prevent failing when searching for golang dep on control file +params=("-d" "-S") + +# add the -sa option for signing along with the key to use when provided key id +if [ -n "$DO_SIGN" ]; then + params+=("-sa") + + # read from gpg the key id associated with the maintainer if not provided explicitly + if [ -z "$KEYID" ]; then + KEYID="\$(gpg --list-keys "\$(dpkg-parsechangelog -S Maintainer)" | head -2 | tail -1 | xargs)" + else + KEYID="$KEYID" + fi + + params+=("--sign-key="\$KEYID"") + + if [ -n "$PASSPHRASE" ]; then + gpg-agent --verbose --daemon --options /home/yq/.gnupg/gpg-agent.conf --log-file /tmp/gpg-agent.log --allow-preset-passphrase --default-cache-ttl=31536000 + KEYGRIP="\$(gpg --with-keygrip -k "\$KEYID" | grep 'Keygrip = ' | cut -d'=' -f2 | head -1 | xargs)" + /usr/lib/gnupg/gpg-preset-passphrase --preset --passphrase "$PASSPHRASE" "\$KEYGRIP" + fi + +else + params+=("-us" "-uc") +fi + +debuild \${params[@]} +mv ../yq_* /home/yq/output + +echo "" +echo -e "\tfind resulting package at: "$OUTPUT"" + +# publish to ppa whether given +if [ -n "$DO_PUBLISH" ]; then + dput -c /etc/dput.cf ppa /home/yq/output/yq_*.changes +fi +EOF +chmod +x "${blddir}"/build.sh + +# build the docker image with all dependencies +cat << EOF > ${blddir}/Dockerfile +FROM bitnami/minideb:bullseye as base +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 +ENV DEBIAN_FRONTEND noninteractive +ENV GO111MODULE on +ENV GOMODCACHE /home/yq/go + +RUN set -e \ + && sed -i -- 's/# deb-src/deb-src/g' /etc/apt/sources.list \ + && apt-get -qq update + +# install golang on its $GOVERSION +FROM base as golang +RUN apt-get -qq -y --no-install-recommends install \ + ca-certificates \ + wget +RUN wget "https://golang.org/dl/go${GOVERSION}.linux-amd64.tar.gz" -4 +RUN tar -C /usr/local -xvf "go${GOVERSION}.linux-amd64.tar.gz" + +FROM base +RUN apt-get -qq -y --no-install-recommends install \ + build-essential \ + debhelper \ + devscripts \ + dput \ + fakeroot \ + git-buildpackage \ + gpg-agent \ + libdistro-info-perl \ + pandoc \ + rsync \ + sensible-utils && \ + apt-get clean && \ + rm -rf /tmp/* /var/tmp/* + +COPY --from=golang /usr/local/go /usr/local/go + +# build debian package as yq user +RUN useradd -ms /bin/bash yq && \ + mkdir /home/yq/src && chown -R yq: /home/yq/src && \ + mkdir /home/yq/output && chown -R yq: /home/yq/output + +ADD ./build/dput.cf /etc/dput.cf +ADD ./build/build.sh /usr/bin/build.sh +RUN chmod +x /usr/bin/build.sh && chown -R yq: /usr/bin/build.sh + +USER yq + +WORKDIR /home/yq/src +VOLUME ["/home/yq/src"] + +# dir where output packages are finally left +VOLUME ["/home/yq/output"] + +CMD ["/usr/bin/build.sh"] +EOF + +DOCKER_BUILDKIT=1 docker build --pull -f "${blddir}"/Dockerfile -t "${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" . + +docker run --rm -i \ + -v "${srcdir}":/home/yq/src:delegated \ + -v "${OUTPUT}":/home/yq/output \ + -v "${HOME}"/.gnupg:/home/yq/.gnupg:delegated \ + -u "$(id -u)" \ + "${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" From 5e30f3ba90a1d5a3307193a7e44274062742b5a7 Mon Sep 17 00:00:00 2001 From: robertomier Date: Sun, 2 Jan 2022 23:20:13 +0100 Subject: [PATCH 2/7] Update changelog for version v4.16.2 (#1046) * Update changelog for version v4.16.2 * fix: remove useless changelog comments from debian changelog --- debian/changelog | 109 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/debian/changelog b/debian/changelog index 56dbfbca..25c4b943 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,112 @@ +yq (4.16.2) focal; urgency=medium + + * Fixed with semicolon space issue + * Updating with documentation + * Added STDIN example to the top + * minor readme cleanup + * Help text tweak + * Fixed docker timeout - simplify docker builds + * New release with docker build fixes + * Updating to go 1.17 to fix CVE #944 + * Fix a typo in root.go + * Skip the tests if the nocheck Debian build option is specified + * Fixed select bug (#958) + * Sped up explode operator + * Slight performance improvement to context.ChildContext + * Speed up multiply + * Update README with recently added / changed options + * Make deepMatch report in linear time + * Removed leadingContentPreProcessing flag - header preprocessing is stable + * Revert "Removed leadingContentPreProcessing flag - header preprocessing is stable" + * Keep flag, it is needed in corner cases + * Updated Readme + * Man page + * Fixed expression parsing bug #970 + * Bumping go-lang, docker versions + * Added test release flow + * Updated github action release to generate man page + * Bumping version + * Removing no longer needed github action + * Added decoder op + * Fixed newline handling when decoding/encoding + * Fixed newline handling in encoder/decoder + * Can specify indent in encode ops + * Added group_by operator + * Added flatten operator + * Fixed flatten error message + * Improving docs + * Split printer + * Refactored command logic + * Fix JSON encoding removing null #985 + * Fixed acceptance tests + * gitbook + * Update document generation script + * Updating README + * Updating release instructions + * github action no longer uses data1.yml + * Create dependabot.yml + * Bump actions/create-release from 1.0.0 to 1.1.4 + * Bump actions/setup-go from 1 to 2.1.4 + * Bump github.com/goccy/go-yaml from 1.8.9 to 1.9.4 + * Bump github.com/jinzhu/copier from 0.2.8 to 0.3.2 + * Bump github.com/fatih/color from 1.10.0 to 1.13.0 + * Bump github.com/spf13/cobra from 1.1.3 to 1.2.1 + * Update dependabot.yml + * Update go.yml + * add build check to PRs + * Include secure as part of build process + * Fixing bad label in github action + * fixed printer test + * remove leading content indicator + * Fixed header preprocessing! + * lint : define golangci configuration file + * Update check.sh + * Load file acceptance test + * Minor improvement on handling front matter + * Improved load doc + * feature: detect MANPATh and install there + * Update install-man-page.sh + * simplify prod stage, move version label to action + * add labels, quote some values + * enable errorlint linter + * Added errorlint to devtools + * Added key operator + * Added more tests + * Fixing comments + * Attempt to fix golint problem + * Include version query for tools + * Clean up errored file? + * enable misspell linter + * updated readme + * update Golangci version to v1.43.0 + * gci linter + * Better merge array by key example + * Added credit for merge by array example + * Better formatting of merge arrays example + * Better merge example + * Add accessor for the yq logger instance (#1013) + * Fixed collect op when working with multiple nodes + * Added map, map_values + * Add support for Podman as well as Docker (#1026) + * Bump github.com/jinzhu/copier from 0.3.2 to 0.3.4 (#1027) + * Added csv, tsv output formats + * Added encoder tests + * Cleanup test + * Fixed docker permission issue #1014 + * Recording release notes for next release + * Assignment op no longer clobbers anchor (#1029) + * Added sort_by operator + * Improved error message + * Improved tips and tricks + * Report while filename failed to parse #1030 + * Added script for extracting checksums + * Improved extract-checksum.sh + * Bump github.com/spf13/cobra from 1.2.1 to 1.3.0 (#1039) + * enable more linters (#1043) + * Bump golang compiler #1037 + + -- Roberto Mier Escandon Tue, 21 Dec 2021 09:41:44 +0000 + yq (4.13.0) focal; urgency=medium * New `with` operator for making multiple changes to a given path From eac75f7dad43079d892efedfe0e19c990157b9ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A9lio?= Date: Sun, 2 Jan 2022 19:20:38 -0300 Subject: [PATCH 3/7] Arch Linux install instructions (#1059) --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 9dd86b0e..1506d5f4 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,12 @@ webi yq See [webi](https://webinstall.dev/) Supported by @adithyasunil26 (https://github.com/webinstall/webi-installers/tree/master/yq) +### Arch Linux + +``` +pacman -S go-yq +``` + ### Windows: [![Chocolatey](https://img.shields.io/chocolatey/v/yq.svg)](https://chocolatey.org/packages/yq) [![Chocolatey](https://img.shields.io/chocolatey/dt/yq.svg)](https://chocolatey.org/packages/yq) From 2d7c63e4e46e3d739edfa2820bbc1f4f3a215ba9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 14 Jan 2022 13:33:30 +1100 Subject: [PATCH 4/7] Bump github.com/goccy/go-yaml from 1.9.4 to 1.9.5 (#1065) Bumps [github.com/goccy/go-yaml](https://github.com/goccy/go-yaml) from 1.9.4 to 1.9.5. - [Release notes](https://github.com/goccy/go-yaml/releases) - [Changelog](https://github.com/goccy/go-yaml/blob/master/CHANGELOG.md) - [Commits](https://github.com/goccy/go-yaml/compare/v1.9.4...v1.9.5) --- updated-dependencies: - dependency-name: github.com/goccy/go-yaml dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index be0a0302..0549f8e3 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/mikefarah/yq/v4 require ( github.com/elliotchance/orderedmap v1.4.0 github.com/fatih/color v1.13.0 - github.com/goccy/go-yaml v1.9.4 + github.com/goccy/go-yaml v1.9.5 github.com/jinzhu/copier v0.3.4 github.com/magiconair/properties v1.8.5 github.com/spf13/cobra v1.3.0 diff --git a/go.sum b/go.sum index aad032da..6bbb4d7c 100644 --- a/go.sum +++ b/go.sum @@ -123,8 +123,8 @@ github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTM github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/goccy/go-yaml v1.9.4 h1:S0GCYjwHKVI6IHqio7QWNKNThUl6NLzFd/g8Z65Axw8= -github.com/goccy/go-yaml v1.9.4/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= +github.com/goccy/go-yaml v1.9.5 h1:Eh/+3uk9kLxG4koCX6lRMAPS1OaMSAi+FJcya0INdB0= +github.com/goccy/go-yaml v1.9.5/go.mod h1:U/jl18uSupI5rdI2jmuCswEA2htH9eXfferR3KfscvA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= From 84ddf1862f7b913773f33be5f90ec0adde83ea64 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 14 Jan 2022 14:55:25 +1100 Subject: [PATCH 5/7] Fixed #1048 --- pkg/yqlib/operator_union.go | 16 ++++++++++++---- pkg/yqlib/operator_union_test.go | 8 ++++++++ 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/pkg/yqlib/operator_union.go b/pkg/yqlib/operator_union.go index e1d54e3a..f2334b2d 100644 --- a/pkg/yqlib/operator_union.go +++ b/pkg/yqlib/operator_union.go @@ -1,5 +1,7 @@ package yqlib +import "container/list" + func unionOperator(d *dataTreeNavigator, context Context, expressionNode *ExpressionNode) (Context, error) { log.Debug("unionOperator") log.Debug("context: %v", NodesToString(context.MatchingNodes)) @@ -18,19 +20,25 @@ func unionOperator(d *dataTreeNavigator, context Context, expressionNode *Expres log.Debug("lhs: %v", lhs.ToString()) log.Debug("rhs: %v", rhs.ToString()) + results := lhs.ChildContext(list.New()) + for el := lhs.MatchingNodes.Front(); el != nil; el = el.Next() { + node := el.Value.(*CandidateNode) + results.MatchingNodes.PushBack(node) + } + // this can happen when both expressions modify the context // instead of creating their own. /// (.foo = "bar"), (.thing = "cat") - if rhs.MatchingNodes != lhs.MatchingNodes { for el := rhs.MatchingNodes.Front(); el != nil; el = el.Next() { node := el.Value.(*CandidateNode) log.Debug("processing %v", NodeToString(node)) - lhs.MatchingNodes.PushBack(node) + results.MatchingNodes.PushBack(node) } } - log.Debug("all together: %v", lhs.ToString()) - return lhs, nil + log.Debug("and lets print it out") + log.Debug("all together: %v", results.ToString()) + return results, nil } diff --git a/pkg/yqlib/operator_union_test.go b/pkg/yqlib/operator_union_test.go index 1409fd7f..ce12f19a 100644 --- a/pkg/yqlib/operator_union_test.go +++ b/pkg/yqlib/operator_union_test.go @@ -13,6 +13,14 @@ var unionOperatorScenarios = []expressionScenario{ "D0, P[], (doc)::{}\n", }, }, + { + skipDoc: true, + description: "clone test", + expression: `"abc" as $a | [$a, "cat"]`, + expected: []string{ + "D0, P[], (!!seq)::- abc\n- cat\n", + }, + }, { skipDoc: true, expression: `(.foo = "bar"), (.toe = "jam")`, From 78b45a3eb06ab44e2a5c5e195f266babd86ace9e Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 14 Jan 2022 15:22:55 +1100 Subject: [PATCH 6/7] Fixed #1062 --- pkg/yqlib/file_utils.go | 14 ++++++-------- pkg/yqlib/front_matter.go | 2 +- pkg/yqlib/front_matter_test.go | 6 +++--- pkg/yqlib/write_in_place_handler.go | 4 ++-- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/pkg/yqlib/file_utils.go b/pkg/yqlib/file_utils.go index 7a423720..9e252c01 100644 --- a/pkg/yqlib/file_utils.go +++ b/pkg/yqlib/file_utils.go @@ -1,30 +1,28 @@ package yqlib import ( + "fmt" "io" "io/ioutil" "os" ) -func safelyRenameFile(from string, to string) { +func tryRenameFile(from string, to string) { if renameError := os.Rename(from, to); renameError != nil { log.Debugf("Error renaming from %v to %v, attempting to copy contents", from, to) log.Debug(renameError.Error()) + log.Debug("going to try copying instead") // can't do this rename when running in docker to a file targeted in a mounted volume, // so gracefully degrade to copying the entire contents. if copyError := copyFileContents(from, to); copyError != nil { - log.Errorf("Failed copying from %v to %v", from, to) - log.Error(copyError.Error()) + panic(fmt.Errorf("failed copying from %v to %v: %w", from, to, copyError)) } else { - removeErr := os.Remove(from) - if removeErr != nil { - log.Errorf("failed removing original file: %s", from) - } + tryRemoveTempFile(from) } } } -func tryRemoveFile(filename string) { +func tryRemoveTempFile(filename string) { log.Debug("Removing temp file: %v", filename) removeErr := os.Remove(filename) if removeErr != nil { diff --git a/pkg/yqlib/front_matter.go b/pkg/yqlib/front_matter.go index 2e198882..e192f443 100644 --- a/pkg/yqlib/front_matter.go +++ b/pkg/yqlib/front_matter.go @@ -33,7 +33,7 @@ func (f *frontMatterHandlerImpl) GetContentReader() io.Reader { } func (f *frontMatterHandlerImpl) CleanUp() { - tryRemoveFile(f.yamlFrontMatterFilename) + tryRemoveTempFile(f.yamlFrontMatterFilename) } // Splits the given file by yaml front matter diff --git a/pkg/yqlib/front_matter_test.go b/pkg/yqlib/front_matter_test.go index 351a386d..7ab2c26f 100644 --- a/pkg/yqlib/front_matter_test.go +++ b/pkg/yqlib/front_matter_test.go @@ -66,7 +66,7 @@ yaml: doc } test.AssertResult(t, expectedContent, string(contentBytes)) - tryRemoveFile(file) + tryRemoveTempFile(file) fmHandler.CleanUp() } @@ -103,7 +103,7 @@ yaml: doc } test.AssertResult(t, expectedContent, string(contentBytes)) - tryRemoveFile(file) + tryRemoveTempFile(file) fmHandler.CleanUp() } @@ -137,6 +137,6 @@ yaml: doc } test.AssertResult(t, expectedContent, string(contentBytes)) - tryRemoveFile(file) + tryRemoveTempFile(file) fmHandler.CleanUp() } diff --git a/pkg/yqlib/write_in_place_handler.go b/pkg/yqlib/write_in_place_handler.go index 533854cc..5460e338 100644 --- a/pkg/yqlib/write_in_place_handler.go +++ b/pkg/yqlib/write_in_place_handler.go @@ -44,8 +44,8 @@ func (w *writeInPlaceHandlerImpl) FinishWriteInPlace(evaluatedSuccessfully bool) safelyCloseFile(w.tempFile) if evaluatedSuccessfully { log.Debug("Moving temp file to target") - safelyRenameFile(w.tempFile.Name(), w.inputFilename) + tryRenameFile(w.tempFile.Name(), w.inputFilename) } else { - tryRemoveFile(w.tempFile.Name()) + tryRemoveTempFile(w.tempFile.Name()) } } From 606ef91cc697375855cc0105a38fed462fc985a4 Mon Sep 17 00:00:00 2001 From: Mike Farah Date: Fri, 14 Jan 2022 15:40:20 +1100 Subject: [PATCH 7/7] Better fix #1062 (return error instead of panic) --- cmd/evaluate_all_command.go | 8 ++++++-- cmd/evalute_sequence_command.go | 8 ++++++-- pkg/yqlib/file_utils.go | 8 ++++---- pkg/yqlib/write_in_place_handler.go | 11 ++++++----- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/cmd/evaluate_all_command.go b/cmd/evaluate_all_command.go index c52e2526..df8b15bc 100644 --- a/cmd/evaluate_all_command.go +++ b/cmd/evaluate_all_command.go @@ -39,7 +39,7 @@ Note that it consumes more memory than eval. } return cmdEvalAll } -func evaluateAll(cmd *cobra.Command, args []string) error { +func evaluateAll(cmd *cobra.Command, args []string) (cmdError error) { // 0 args, read std in // 1 arg, null input, process expression // 1 arg, read file in sequence @@ -67,7 +67,11 @@ func evaluateAll(cmd *cobra.Command, args []string) error { } // need to indirectly call the function so that completedSuccessfully is // passed when we finish execution as opposed to now - defer func() { writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully) }() + defer func() { + if cmdError == nil { + cmdError = writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully) + } + }() } format, err := yqlib.OutputFormatFromString(outputFormat) diff --git a/cmd/evalute_sequence_command.go b/cmd/evalute_sequence_command.go index 2a371ed4..e0e14d27 100644 --- a/cmd/evalute_sequence_command.go +++ b/cmd/evalute_sequence_command.go @@ -53,7 +53,7 @@ func processExpression(expression string) string { return expression } -func evaluateSequence(cmd *cobra.Command, args []string) error { +func evaluateSequence(cmd *cobra.Command, args []string) (cmdError error) { // 0 args, read std in // 1 arg, null input, process expression // 1 arg, read file in sequence @@ -80,7 +80,11 @@ func evaluateSequence(cmd *cobra.Command, args []string) error { } // need to indirectly call the function so that completedSuccessfully is // passed when we finish execution as opposed to now - defer func() { writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully) }() + defer func() { + if cmdError == nil { + cmdError = writeInPlaceHandler.FinishWriteInPlace(completedSuccessfully) + } + }() } format, err := yqlib.OutputFormatFromString(outputFormat) diff --git a/pkg/yqlib/file_utils.go b/pkg/yqlib/file_utils.go index 9e252c01..c9a7014e 100644 --- a/pkg/yqlib/file_utils.go +++ b/pkg/yqlib/file_utils.go @@ -7,7 +7,7 @@ import ( "os" ) -func tryRenameFile(from string, to string) { +func tryRenameFile(from string, to string) error { if renameError := os.Rename(from, to); renameError != nil { log.Debugf("Error renaming from %v to %v, attempting to copy contents", from, to) log.Debug(renameError.Error()) @@ -15,11 +15,11 @@ func tryRenameFile(from string, to string) { // can't do this rename when running in docker to a file targeted in a mounted volume, // so gracefully degrade to copying the entire contents. if copyError := copyFileContents(from, to); copyError != nil { - panic(fmt.Errorf("failed copying from %v to %v: %w", from, to, copyError)) - } else { - tryRemoveTempFile(from) + return fmt.Errorf("failed copying from %v to %v: %w", from, to, copyError) } + tryRemoveTempFile(from) } + return nil } func tryRemoveTempFile(filename string) { diff --git a/pkg/yqlib/write_in_place_handler.go b/pkg/yqlib/write_in_place_handler.go index 5460e338..d3bbb487 100644 --- a/pkg/yqlib/write_in_place_handler.go +++ b/pkg/yqlib/write_in_place_handler.go @@ -6,7 +6,7 @@ import ( type writeInPlaceHandler interface { CreateTempFile() (*os.File, error) - FinishWriteInPlace(evaluatedSuccessfully bool) + FinishWriteInPlace(evaluatedSuccessfully bool) error } type writeInPlaceHandlerImpl struct { @@ -39,13 +39,14 @@ func (w *writeInPlaceHandlerImpl) CreateTempFile() (*os.File, error) { return file, err } -func (w *writeInPlaceHandlerImpl) FinishWriteInPlace(evaluatedSuccessfully bool) { +func (w *writeInPlaceHandlerImpl) FinishWriteInPlace(evaluatedSuccessfully bool) error { log.Debug("Going to write-inplace, evaluatedSuccessfully=%v, target=%v", evaluatedSuccessfully, w.inputFilename) safelyCloseFile(w.tempFile) if evaluatedSuccessfully { log.Debug("Moving temp file to target") - tryRenameFile(w.tempFile.Name(), w.inputFilename) - } else { - tryRemoveTempFile(w.tempFile.Name()) + return tryRenameFile(w.tempFile.Name(), w.inputFilename) } + tryRemoveTempFile(w.tempFile.Name()) + + return nil }