From 0b49bebad911bb3a563fd819545dc5ee26a445be Mon Sep 17 00:00:00 2001 From: robertomier Date: Thu, 23 Dec 2021 01:43:21 +0100 Subject: [PATCH] chore: script to build the deb sources (#1045) * chore: script to build the deb sources now it includes the man files. This script makes only sense executing on the machine having the gpg key needed to sign and push to ppa * fix: dput dependency and proper configuration --- .gitignore | 6 +- debian/control | 3 +- debian/rules | 23 +++- release_instructions.txt | 10 +- scripts/release-deb.sh | 264 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 299 insertions(+), 7 deletions(-) create mode 100755 scripts/release-deb.sh diff --git a/.gitignore b/.gitignore index 00b1ab74..0c2a0b48 100644 --- a/.gitignore +++ b/.gitignore @@ -48,4 +48,8 @@ test*.yml # man page man.md -yq.1 \ No newline at end of file +yq.1 + +# debian pkg +_build +debian/files diff --git a/debian/control b/debian/control index 9794eaab..625bfb3d 100644 --- a/debian/control +++ b/debian/control @@ -3,7 +3,8 @@ Section: devel Priority: optional Maintainer: Roberto Mier Escandón Build-Depends: debhelper (>=10), - golang-1.15, + golang-1.17-go, + pandoc, rsync Standards-Version: 4.1.4 Homepage: https://github.com/mikefarah/yq.git diff --git a/debian/rules b/debian/rules index 912f8d33..4c40d6db 100755 --- a/debian/rules +++ b/debian/rules @@ -1,6 +1,6 @@ #!/usr/bin/make -f # -# Copyright (C) 2018 Roberto Mier Escandón +# Copyright (C) 2018-2021 Roberto Mier Escandón # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License version 3 as @@ -17,20 +17,21 @@ PROJECT := yq OWNER := mikefarah REPO := github.com -GOVERSION := 1.15 export DH_OPTIONS export DH_GOPKG := ${REPO}/${OWNER}/${PROJECT} -export GOROOT := /usr/lib/go-${GOVERSION} +export GOROOT := /usr/local/go export GOPATH := ${CURDIR}/_build export GOBIN := ${GOPATH}/bin export PATH := ${GOROOT}/bin:${GOBIN}:${PATH} export GOCACHE := /tmp/gocache export GOFLAGS := -mod=vendor +export GO111MODULE := on SRCDIR := ${GOPATH}/src/${DH_GOPKG} DESTDIR := ${CURDIR}/debian/${PROJECT} BINDIR := /usr/bin +MANDIR := /usr/share/man/man1/ ASSETSDIR := /usr/share/${PROJECT} %: @@ -42,7 +43,17 @@ override_dh_auto_build: # copy project to local srcdir to build from there rsync -avz --progress --exclude=_build --exclude=debian --exclude=tmp. --exclude=go.mod --exclude=docs . $(SRCDIR) # build go code - (cd ${SRCDIR} && go install -buildmode=pie ./...) + ( \ + cd ${SRCDIR} && \ + go install -buildmode=pie ./... \ + ) + + # build man page + ( \ + cd ${SRCDIR} && \ + ./scripts/generate-man-page-md.sh && \ + ./scripts/generate-man-page.sh \ + ) override_dh_auto_test: ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS))) @@ -54,6 +65,10 @@ override_dh_auto_install: cp -f ${SRCDIR}/LICENSE ${DESTDIR}/${ASSETSDIR} chmod a+x ${DESTDIR}/${BINDIR}/yq + # man + mkdir -p "${DESTDIR}"/"${MANDIR}" + cp "${SRCDIR}"/yq.1 "${DESTDIR}"/"${MANDIR}" \ + override_dh_auto_clean: dh_clean rm -rf ${CURDIR}/_build diff --git a/release_instructions.txt b/release_instructions.txt index f0eedc74..0cbbda3f 100644 --- a/release_instructions.txt +++ b/release_instructions.txt @@ -33,7 +33,15 @@ - build and push latest and new version tag - docker build . -t mikefarah/yq:latest -t mikefarah/yq:3 -t mikefarah/yq:3.X -- debian package +- debian package (with release script) + - execute the script `./scripts/release-deb.sh` *on the machine having the private + gpg key to sign the generated sources* providing the version to release, the + ppa where deploying the files and the passphrase of the private key if needed: + ``` + ./scripts/release-deb.sh -o output -p -s --passphrase PASSPHRASE VERSION + ``` + +- debian package (manually) - ensure you get all vendor dependencies before packaging ```go mod vendor``` - execute diff --git a/scripts/release-deb.sh b/scripts/release-deb.sh new file mode 100755 index 00000000..e6c49fd1 --- /dev/null +++ b/scripts/release-deb.sh @@ -0,0 +1,264 @@ +#!/bin/bash -eux +# +# Copyright (C) 2021 Roberto Mier Escandón +# +# This script creates a .deb package file with yq valid for ubuntu 20.04 by default +# You can pass + +DOCKER_IMAGE_NAME=yq-deb-builder +DOCKER_IMAGE_TAG=$(git describe --always --tags) +OUTPUT= +GOVERSION="1.17.4" +KEYID= +MAINTAINER= +DO_PUBLISH= +PPA="rmescandon/yq" +VERSION= +DISTRIBUTION= +DO_SIGN= +PASSPHRASE= + +show_help() { + echo " usage: $(basename "$0") VERSION [options...]" + echo "" + echo " positional arguments" + echo " VERSION" + echo "" + echo " optional arguments:" + echo " -h, --help Shows this help" + echo " -d, --distribution DISTRO The distribution to use for the changelog generation. If not provided, last changelog entry" + echo " distribution is considered" + echo " --goversion VERSION The version of golang to use. Default to $GOVERSION" + echo " -k, --sign-key KEYID Sign the package sources with the provided gpg key id (long format). When not provided this" + echo " paramater, the generated sources are not signed" + echo " -s, --sign Sign the package sources with a gpg key of the maintainer" + echo " -m, --maintainer WHO The maintainer used as author of the changelog. git.name and git.email (see git config) is" + echo " the considered format" + echo " -o DIR, --output DIR The path where leaving the generated debian package. Default to a temporary folder if not set" + echo " -p The resultant file is being published to ppa" + echo " --ppa PPA Push resultant files to indicated ppa. This option should be given along with a signing key." + echo " Otherwise, the server could reject the package building. Default is set to 'rmescandon/yq'" + echo " --passphrase PASSPHRASE Passphrase to decrypt the signage key" + exit 1 +} +# read input args +while [ $# -ne 0 ]; do + case $1 in + -h|--help) + show_help + ;; + -d|--distribution) + shift + DISTRIBUTION="$1" + ;; + --goversion) + shift + GOVERSION="$1" + ;; + -k|--sign-key) + shift + DO_SIGN='y' + KEYID="$1" + ;; + -s|--sign) + DO_SIGN='y' + ;; + -m|--maintainer) + shift + MAINTAINER="$1" + ;; + -o|--output) + shift + OUTPUT="$1" + ;; + -p) + DO_PUBLISH="y" + ;; + --ppa) + shift + DO_PUBLISH="y" + PPA="$1" + ;; + --passphrase) + shift + PASSPHRASE="$1" + ;; + *) + if [ -z "$VERSION" ]; then + VERSION="$1" + else + show_help + fi + esac + shift +done + +[ -n "$VERSION" ] || (echo "error - you have to provide a version" && show_help) + +if [ -n "$OUTPUT" ]; then + OUTPUT="$(realpath "$OUTPUT")" + mkdir -p "$OUTPUT" +else + # Temporary folder where leaving the built deb package in case that output folder is not provided + OUTPUT="$(mktemp -d)" +fi + +# Define the folders with the source project and the build artifacts and files +srcdir="$(realpath "$(dirname "$0")"/..)" +blddir="$(cd "${srcdir}" && mkdir -p build && cd build && echo "$(pwd)")" +# clean on exit +cleanup() { + rm -f "${blddir}/build.sh" || true + rm -f "${blddir}/Dockerfile" || true + rm -f "${blddir}/dput.cf" || true + docker rmi "${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" -f > /dev/null 2>&1 || true +} +trap cleanup EXIT INT + +# configure the dput config in case publishing is requested +lp_id="$(echo "$PPA" | cut -d'/' -f1)" +ppa_id="$(echo "$PPA" | cut -d'/' -f2)" +cat << EOF > ${blddir}/dput.cf +[ppa] +fqdn = ppa.launchpad.net +method = ftp +incoming = ~${lp_id}/ubuntu/${ppa_id} +login = anonymous +EOF + +# create the main script +cat << EOF > ${blddir}/build.sh +#!/bin/bash +set -e -o pipefail + +PATH=$PATH:/usr/local/go/bin +export GPG_TTY=$(tty) + +go mod vendor + +### bump debian/changelog + +# maintainer +export DEBEMAIL="$MAINTAINER" +if [ -z "$MAINTAINER" ]; then + export DEBEMAIL="\$(dpkg-parsechangelog -S Maintainer)" +fi + +# prepend a 'v' char to complete the tag name from where calculating the changelog +SINCE="v\$(dpkg-parsechangelog -S Version)" + +# distribution +DISTRIBUTION="$DISTRIBUTION" +if [ -z "$DISTRIBUTION" ]; then + DISTRIBUTION="\$(dpkg-parsechangelog -S Distribution)" +fi + +# generate changelog +gbp dch --ignore-branch --no-multimaint -N "$VERSION" -s "\$SINCE" -D "\$DISTRIBUTION" + +# using -d to prevent failing when searching for golang dep on control file +params=("-d" "-S") + +# add the -sa option for signing along with the key to use when provided key id +if [ -n "$DO_SIGN" ]; then + params+=("-sa") + + # read from gpg the key id associated with the maintainer if not provided explicitly + if [ -z "$KEYID" ]; then + KEYID="\$(gpg --list-keys "\$(dpkg-parsechangelog -S Maintainer)" | head -2 | tail -1 | xargs)" + else + KEYID="$KEYID" + fi + + params+=("--sign-key="\$KEYID"") + + if [ -n "$PASSPHRASE" ]; then + gpg-agent --verbose --daemon --options /home/yq/.gnupg/gpg-agent.conf --log-file /tmp/gpg-agent.log --allow-preset-passphrase --default-cache-ttl=31536000 + KEYGRIP="\$(gpg --with-keygrip -k "\$KEYID" | grep 'Keygrip = ' | cut -d'=' -f2 | head -1 | xargs)" + /usr/lib/gnupg/gpg-preset-passphrase --preset --passphrase "$PASSPHRASE" "\$KEYGRIP" + fi + +else + params+=("-us" "-uc") +fi + +debuild \${params[@]} +mv ../yq_* /home/yq/output + +echo "" +echo -e "\tfind resulting package at: "$OUTPUT"" + +# publish to ppa whether given +if [ -n "$DO_PUBLISH" ]; then + dput -c /etc/dput.cf ppa /home/yq/output/yq_*.changes +fi +EOF +chmod +x "${blddir}"/build.sh + +# build the docker image with all dependencies +cat << EOF > ${blddir}/Dockerfile +FROM bitnami/minideb:bullseye as base +ENV LANG C.UTF-8 +ENV LC_ALL C.UTF-8 +ENV DEBIAN_FRONTEND noninteractive +ENV GO111MODULE on +ENV GOMODCACHE /home/yq/go + +RUN set -e \ + && sed -i -- 's/# deb-src/deb-src/g' /etc/apt/sources.list \ + && apt-get -qq update + +# install golang on its $GOVERSION +FROM base as golang +RUN apt-get -qq -y --no-install-recommends install \ + ca-certificates \ + wget +RUN wget "https://golang.org/dl/go${GOVERSION}.linux-amd64.tar.gz" -4 +RUN tar -C /usr/local -xvf "go${GOVERSION}.linux-amd64.tar.gz" + +FROM base +RUN apt-get -qq -y --no-install-recommends install \ + build-essential \ + debhelper \ + devscripts \ + dput \ + fakeroot \ + git-buildpackage \ + gpg-agent \ + libdistro-info-perl \ + pandoc \ + rsync \ + sensible-utils && \ + apt-get clean && \ + rm -rf /tmp/* /var/tmp/* + +COPY --from=golang /usr/local/go /usr/local/go + +# build debian package as yq user +RUN useradd -ms /bin/bash yq && \ + mkdir /home/yq/src && chown -R yq: /home/yq/src && \ + mkdir /home/yq/output && chown -R yq: /home/yq/output + +ADD ./build/dput.cf /etc/dput.cf +ADD ./build/build.sh /usr/bin/build.sh +RUN chmod +x /usr/bin/build.sh && chown -R yq: /usr/bin/build.sh + +USER yq + +WORKDIR /home/yq/src +VOLUME ["/home/yq/src"] + +# dir where output packages are finally left +VOLUME ["/home/yq/output"] + +CMD ["/usr/bin/build.sh"] +EOF + +DOCKER_BUILDKIT=1 docker build --pull -f "${blddir}"/Dockerfile -t "${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" . + +docker run --rm -i \ + -v "${srcdir}":/home/yq/src:delegated \ + -v "${OUTPUT}":/home/yq/output \ + -v "${HOME}"/.gnupg:/home/yq/.gnupg:delegated \ + -u "$(id -u)" \ + "${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"