chore: script to build the deb sources (#1045)

* chore: script to build the deb sources

now it includes the man files. This script makes only sense executing
on the machine having the gpg key needed to sign and push to ppa

* fix: dput dependency and proper configuration

.gitignore

@@ -48,4 +48,8 @@ test*.yml
 
 # man page
 man.md
-yq.1
+yq.1
+
+# debian pkg
+_build
+debian/files

debian/control

@@ -3,7 +3,8 @@ Section: devel
 Priority: optional
 Maintainer: Roberto Mier Escandón <rmescandon@gmail.com>
 Build-Depends: debhelper (>=10),
-               golang-1.15,
+               golang-1.17-go,
+               pandoc,
                rsync
 Standards-Version: 4.1.4
 Homepage: https://github.com/mikefarah/yq.git

debian/rules

@@ -1,6 +1,6 @@
 #!/usr/bin/make -f
 #
-# Copyright (C) 2018 Roberto Mier Escandón <rmescandon@gmail.com>
+# Copyright (C) 2018-2021 Roberto Mier Escandón <rmescandon@gmail.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License version 3 as
@@ -17,20 +17,21 @@
 PROJECT := yq
 OWNER := mikefarah
 REPO := github.com
-GOVERSION := 1.15
 
 export DH_OPTIONS
 export DH_GOPKG := ${REPO}/${OWNER}/${PROJECT}
-export GOROOT := /usr/lib/go-${GOVERSION}
+export GOROOT := /usr/local/go
 export GOPATH := ${CURDIR}/_build
 export GOBIN := ${GOPATH}/bin
 export PATH := ${GOROOT}/bin:${GOBIN}:${PATH}
 export GOCACHE := /tmp/gocache
 export GOFLAGS := -mod=vendor
+export GO111MODULE := on
 
 SRCDIR := ${GOPATH}/src/${DH_GOPKG}
 DESTDIR := ${CURDIR}/debian/${PROJECT}
 BINDIR := /usr/bin
+MANDIR := /usr/share/man/man1/
 ASSETSDIR := /usr/share/${PROJECT}
 
 %:
@@ -42,7 +43,17 @@ override_dh_auto_build:
 	# copy project to local srcdir to build from there
 	rsync -avz --progress --exclude=_build --exclude=debian --exclude=tmp. --exclude=go.mod --exclude=docs . $(SRCDIR)
 	# build go code
-	(cd ${SRCDIR} && go install -buildmode=pie ./...)
+	( \
+		cd ${SRCDIR} && \
+		go install -buildmode=pie ./... \
+	)
+
+	# build man page
+	( \
+		cd ${SRCDIR} && \
+		./scripts/generate-man-page-md.sh && \
+		./scripts/generate-man-page.sh \
+	)
 	
 override_dh_auto_test:
 ifeq (,$(filter nocheck,$(DEB_BUILD_OPTIONS)))
@@ -54,6 +65,10 @@ override_dh_auto_install:
 	cp -f ${SRCDIR}/LICENSE ${DESTDIR}/${ASSETSDIR}
 	chmod a+x ${DESTDIR}/${BINDIR}/yq
 
+	# man
+	mkdir -p "${DESTDIR}"/"${MANDIR}"
+	cp "${SRCDIR}"/yq.1 "${DESTDIR}"/"${MANDIR}" \
+
 override_dh_auto_clean:
 	dh_clean
 	rm -rf ${CURDIR}/_build

release_instructions.txt

@@ -33,7 +33,15 @@
   - build and push latest and new version tag
   - docker build .  -t mikefarah/yq:latest -t mikefarah/yq:3 -t mikefarah/yq:3.X
 
-- debian package
+- debian package (with release script)
+  - execute the script `./scripts/release-deb.sh` *on the machine having the private
+      gpg key to sign the generated sources* providing the version to release, the
+      ppa where deploying the files and the passphrase of the private key if needed:
+      ```
+      ./scripts/release-deb.sh -o output -p -s --passphrase PASSPHRASE VERSION
+      ```
+
+- debian package (manually)
   - ensure you get all vendor dependencies before packaging
 	```go mod vendor```
   - execute 

scripts/release-deb.sh

@@ -0,0 +1,264 @@
+#!/bin/bash -eux
+#
+# Copyright (C) 2021 Roberto Mier Escandón <rmescandon@gmail.com>
+#
+# This script creates a .deb package file with yq valid for ubuntu 20.04 by default
+# You can pass 
+
+DOCKER_IMAGE_NAME=yq-deb-builder
+DOCKER_IMAGE_TAG=$(git describe --always --tags)
+OUTPUT=
+GOVERSION="1.17.4"
+KEYID=
+MAINTAINER=
+DO_PUBLISH=
+PPA="rmescandon/yq"
+VERSION=
+DISTRIBUTION=
+DO_SIGN=
+PASSPHRASE=
+
+show_help() {
+  echo "  usage: $(basename "$0") VERSION [options...]"
+  echo ""
+  echo "  positional arguments"
+  echo "    VERSION"
+  echo ""
+  echo "  optional arguments:"
+  echo "    -h, --help		              Shows this help"
+  echo "    -d, --distribution DISTRO   The distribution to use for the changelog generation. If not provided, last changelog entry"
+  echo "                                  distribution is considered"
+  echo "    --goversion VERSION         The version of golang to use. Default to $GOVERSION"
+  echo "    -k, --sign-key KEYID        Sign the package sources with the provided gpg key id (long format). When not provided this"
+  echo "                                  paramater, the generated sources are not signed"
+  echo "    -s, --sign                  Sign the package sources with a gpg key of the maintainer"
+  echo "    -m, --maintainer WHO        The maintainer used as author of the changelog. git.name and git.email (see git config) is"
+  echo "                                  the considered format"
+  echo "    -o DIR, --output DIR        The path where leaving the generated debian package. Default to a temporary folder if not set"
+  echo "    -p                          The resultant file is being published to ppa"
+  echo "    --ppa PPA                   Push resultant files to indicated ppa. This option should be given along with a signing key."
+  echo "                                  Otherwise, the server could reject the package building. Default is set to 'rmescandon/yq'"
+  echo "    --passphrase PASSPHRASE     Passphrase to decrypt the signage key"
+  exit 1
+}
+# read input args
+while [ $# -ne 0 ]; do
+  case $1 in
+    -h|--help)
+      show_help
+      ;;
+    -d|--distribution)
+      shift
+      DISTRIBUTION="$1"
+      ;;
+    --goversion)
+      shift
+      GOVERSION="$1"
+      ;;
+    -k|--sign-key)
+      shift
+      DO_SIGN='y'
+      KEYID="$1"
+      ;;
+    -s|--sign)
+      DO_SIGN='y'
+      ;;
+    -m|--maintainer)
+      shift
+      MAINTAINER="$1"
+      ;;
+    -o|--output)
+      shift
+      OUTPUT="$1"
+      ;;
+    -p)
+      DO_PUBLISH="y"
+      ;;
+    --ppa)
+      shift
+      DO_PUBLISH="y"
+      PPA="$1"
+      ;;
+    --passphrase)
+      shift
+      PASSPHRASE="$1"
+      ;;
+    *)
+      if [ -z "$VERSION" ]; then
+        VERSION="$1"
+      else
+        show_help
+      fi
+  esac
+  shift
+done
+
+[ -n "$VERSION" ] || (echo "error - you have to provide a version" && show_help)
+
+if [ -n "$OUTPUT" ]; then
+  OUTPUT="$(realpath "$OUTPUT")"
+  mkdir -p "$OUTPUT"
+else
+  # Temporary folder where leaving the built deb package in case that output folder is not provided
+  OUTPUT="$(mktemp -d)"
+fi 
+
+# Define the folders with the source project and the build artifacts and files
+srcdir="$(realpath "$(dirname "$0")"/..)"
+blddir="$(cd "${srcdir}" && mkdir -p build && cd build && echo "$(pwd)")"
+# clean on exit
+cleanup() {
+  rm -f "${blddir}/build.sh" || true
+  rm -f "${blddir}/Dockerfile" || true
+  rm -f "${blddir}/dput.cf" || true
+  docker rmi "${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" -f > /dev/null 2>&1 || true
+}
+trap cleanup EXIT INT
+
+# configure the dput config in case publishing is requested
+lp_id="$(echo "$PPA" | cut -d'/' -f1)"
+ppa_id="$(echo "$PPA" | cut -d'/' -f2)"
+cat << EOF > ${blddir}/dput.cf
+[ppa]
+fqdn                    = ppa.launchpad.net
+method                  = ftp
+incoming                = ~${lp_id}/ubuntu/${ppa_id}
+login                   = anonymous
+EOF
+
+# create the main script
+cat << EOF > ${blddir}/build.sh
+#!/bin/bash 
+set -e -o pipefail
+
+PATH=$PATH:/usr/local/go/bin
+export GPG_TTY=$(tty)
+
+go mod vendor
+
+### bump debian/changelog
+
+# maintainer
+export DEBEMAIL="$MAINTAINER"
+if [ -z "$MAINTAINER" ]; then
+  export DEBEMAIL="\$(dpkg-parsechangelog -S Maintainer)"
+fi
+
+# prepend a 'v' char to complete the tag name from where calculating the changelog
+SINCE="v\$(dpkg-parsechangelog -S Version)"
+
+# distribution
+DISTRIBUTION="$DISTRIBUTION"
+if [ -z "$DISTRIBUTION" ]; then
+  DISTRIBUTION="\$(dpkg-parsechangelog -S Distribution)"
+fi
+
+# generate changelog
+gbp dch --ignore-branch --no-multimaint -N "$VERSION" -s "\$SINCE" -D "\$DISTRIBUTION"
+
+# using -d to prevent failing when searching for golang dep on control file
+params=("-d" "-S")
+
+# add the -sa option for signing along with the key to use when provided key id
+if [ -n "$DO_SIGN" ]; then
+  params+=("-sa")
+
+  # read from gpg the key id associated with the maintainer if not provided explicitly
+  if [ -z "$KEYID" ]; then
+    KEYID="\$(gpg --list-keys "\$(dpkg-parsechangelog -S Maintainer)" | head -2 | tail -1 | xargs)"
+  else
+    KEYID="$KEYID"
+  fi
+  
+  params+=("--sign-key="\$KEYID"")
+
+  if [ -n "$PASSPHRASE" ]; then
+    gpg-agent --verbose --daemon --options /home/yq/.gnupg/gpg-agent.conf --log-file /tmp/gpg-agent.log --allow-preset-passphrase --default-cache-ttl=31536000
+    KEYGRIP="\$(gpg --with-keygrip -k "\$KEYID" | grep 'Keygrip = ' | cut -d'=' -f2 | head -1 | xargs)"
+    /usr/lib/gnupg/gpg-preset-passphrase --preset --passphrase "$PASSPHRASE" "\$KEYGRIP"
+  fi
+
+else
+  params+=("-us" "-uc")
+fi
+
+debuild  \${params[@]}
+mv ../yq_* /home/yq/output
+
+echo ""
+echo -e "\tfind resulting package at: "$OUTPUT""
+
+# publish to ppa whether given
+if [ -n "$DO_PUBLISH" ]; then
+  dput -c /etc/dput.cf ppa /home/yq/output/yq_*.changes
+fi
+EOF
+chmod +x "${blddir}"/build.sh
+
+# build the docker image with all dependencies
+cat << EOF > ${blddir}/Dockerfile
+FROM bitnami/minideb:bullseye as base
+ENV LANG C.UTF-8
+ENV LC_ALL C.UTF-8
+ENV DEBIAN_FRONTEND noninteractive
+ENV GO111MODULE on
+ENV GOMODCACHE /home/yq/go
+
+RUN set -e \
+  && sed -i -- 's/# deb-src/deb-src/g' /etc/apt/sources.list \
+  && apt-get -qq update
+
+# install golang on its $GOVERSION
+FROM base as golang
+RUN apt-get -qq -y --no-install-recommends install \
+    ca-certificates \
+    wget
+RUN wget "https://golang.org/dl/go${GOVERSION}.linux-amd64.tar.gz" -4
+RUN tar -C /usr/local -xvf "go${GOVERSION}.linux-amd64.tar.gz"
+
+FROM base
+RUN apt-get -qq -y --no-install-recommends install \
+    build-essential \
+    debhelper \
+    devscripts \
+    dput \
+    fakeroot \
+    git-buildpackage \
+    gpg-agent \
+    libdistro-info-perl \
+    pandoc \
+    rsync \
+    sensible-utils && \
+  apt-get clean && \
+  rm -rf /tmp/* /var/tmp/*
+
+COPY --from=golang /usr/local/go /usr/local/go
+
+# build debian package as yq user
+RUN useradd -ms /bin/bash yq && \
+  mkdir /home/yq/src && chown -R yq: /home/yq/src && \
+  mkdir /home/yq/output && chown -R yq: /home/yq/output
+
+ADD ./build/dput.cf /etc/dput.cf
+ADD ./build/build.sh /usr/bin/build.sh
+RUN chmod +x /usr/bin/build.sh && chown -R yq: /usr/bin/build.sh
+
+USER yq
+
+WORKDIR /home/yq/src
+VOLUME ["/home/yq/src"]
+
+# dir where output packages are finally left
+VOLUME ["/home/yq/output"]
+
+CMD ["/usr/bin/build.sh"]
+EOF
+
+DOCKER_BUILDKIT=1 docker build --pull -f "${blddir}"/Dockerfile -t "${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}" .
+
+docker run --rm -i \
+  -v "${srcdir}":/home/yq/src:delegated \
+  -v "${OUTPUT}":/home/yq/output \
+  -v "${HOME}"/.gnupg:/home/yq/.gnupg:delegated \
+  -u "$(id -u)" \
+  "${DOCKER_IMAGE_NAME}:${DOCKER_IMAGE_TAG}"