Merge pull request #4879 from guggero/release-fix

release: actually create deterministic release archives

.github/workflows/main.yml

@@ -23,6 +23,7 @@ env:
   # /.travis.yml
   # /Dockerfile
   # /dev.Dockerfile
+  # /make/builder.Dockerfile
   # /.github/workflows/release.yml
   GO_VERSION: 1.15.6
 

.github/workflows/release.yaml

@@ -10,13 +10,11 @@ defaults:
     shell: bash
 
 env:
-  DOCKER_REPO: lightninglabs
-  DOCKER_IMAGE: lnd
-  
   # If you change this value, please change it in the following files as well:
   # /.travis.yml
   # /Dockerfile
   # /dev.Dockerfile
+  # /make/builder.Dockerfile
   # /.github/workflows/main.yml
   GO_VERSION: 1.15.6
 
@@ -27,6 +25,8 @@ jobs:
     steps:
       - name: git checkout
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
 
       - name: go cache
         uses: actions/cache@v1

.travis.yml

@@ -19,6 +19,7 @@ go:
   # If you change this value, please change it in the following files as well:
   # /Dockerfile
   # /dev.Dockerfile
+  # /make/builder.Dockerfile
   # /.github/workflows/main.yml
   # /.github/workflows/release.yml
   - "1.15.6"

Dockerfile

@@ -1,3 +1,9 @@
+# If you change this value, please change it in the following files as well:
+# /.travis.yml
+# /dev.Dockerfile
+# /make/builder.Dockerfile
+# /.github/workflows/main.yml
+# /.github/workflows/release.yml
 FROM golang:1.15.6-alpine as builder
 
 # Force Go to use the cgo based DNS resolver. This is required to ensure DNS

Makefile

@@ -26,7 +26,7 @@ IOS_BUILD := $(IOS_BUILD_DIR)/Lndmobile.framework
 ANDROID_BUILD_DIR := $(MOBILE_BUILD_DIR)/android
 ANDROID_BUILD := $(ANDROID_BUILD_DIR)/Lndmobile.aar
 
-COMMIT := $(shell git describe --abbrev=40 --dirty)
+COMMIT := $(shell git describe --tags --dirty)
 COMMIT_HASH := $(shell git rev-parse HEAD)
 
 BTCD_COMMIT := $(shell cat go.mod | \
@@ -156,6 +156,14 @@ release:
 	$(VERSION_CHECK)
 	./scripts/release.sh build-release "$(VERSION_TAG)" "$(BUILD_SYSTEM)" "$(RELEASE_TAGS)" "$(RELEASE_LDFLAGS)"
 
+docker-release:
+	@$(call print, "Building release helper docker image.")
+	if [ "$(tag)" = "" ]; then echo "Must specify tag=<commit_or_tag>!"; exit 1; fi
+
+	docker build -t lnd-release-helper -f make/builder.Dockerfile make/
+	$(DOCKER_RELEASE_HELPER) scripts/release.sh check-tag "$(VERSION_TAG)"
+	$(DOCKER_RELEASE_HELPER) scripts/release.sh build-release "$(VERSION_TAG)" "$(BUILD_SYSTEM)" "$(RELEASE_TAGS)" "$(RELEASE_LDFLAGS)"
+
 scratch: build
 
 

dev.Dockerfile

@@ -1,3 +1,9 @@
+# If you change this value, please change it in the following files as well:
+# /.travis.yml
+# /Dockerfile
+# /make/builder.Dockerfile
+# /.github/workflows/main.yml
+# /.github/workflows/release.yml
 FROM golang:1.15.6-alpine as builder
 
 LABEL maintainer="Olaoluwa Osuntokun <laolu@lightning.engineering>"

docs/release.md

@@ -10,7 +10,23 @@ utilize a work around needed until `go1.13.2`.
 
 ## Building a New Release
 
-### macOS/Linux/Windows (WSL)
+### MacOS
+
+The first requirement is to have [`docker`](https://www.docker.com/)
+installed locally and running. The second requirement is to have `make`
+installed. Everything else (including `golang`) is included in the release
+helper image.
+
+To build a release, run the following commands:
+
+1. `git clone https://github.com/lightningnetwork/lnd.git`
+2. `cd lnd`
+3. `git checkout <TAG> # <TAG> is the name of the next release/tag`
+4. `make docker-release tag=<TAG>`
+
+Where `<TAG>` is the name of the next release of `lnd`.
+
+### Linux/Windows (WSL)
 
 No prior set up is needed on Linux or macOS is required in order to build the
 release binaries. However, on Windows, the only way to build the release
@@ -19,7 +35,8 @@ the release binaries following these steps:
 
 1. `git clone https://github.com/lightningnetwork/lnd.git`
 2. `cd lnd`
-3. `make release tag=<TAG> # <TAG> is the name of the next release/tag`
+3. `git checkout <TAG> # <TAG> is the name of the next release/tag`
+4. `make release tag=<TAG>`
 
 This will then create a directory of the form `lnd-<TAG>` containing archives
 of the release binaries for each supported operating system and architecture,

make/builder.Dockerfile

@@ -0,0 +1,34 @@
+# If you change this value, please change it in the following files as well:
+# /.travis.yml
+# /Dockerfile
+# /dev.Dockerfile
+# /.github/workflows/main.yml
+# /.github/workflows/release.yml
+FROM golang:1.15.6-buster
+
+MAINTAINER Olaoluwa Osuntokun <laolu@lightning.engineering>
+
+# Golang build related environment variables that are static and used for all
+# architectures/OSes.
+ENV GODEBUG netdns=cgo
+ENV GO111MODULE=auto
+ENV CGO_ENABLED=0
+
+# Set up cache directories. Those will be mounted from the host system to speed
+# up builds. If go isn't installed on the host system, those will fall back to
+# temp directories during the build (see make/release_flags.mk).
+ENV GOCACHE=/tmp/build/.cache
+ENV GOMODCACHE=/tmp/build/.modcache
+
+RUN apt-get update && apt-get install -y \
+    git \
+    make \
+    tar \
+    zip \
+    bash \
+  && mkdir -p /tmp/build/lnd \
+  && mkdir -p /tmp/build/.cache \
+  && mkdir -p /tmp/build/.modcache \
+  && chmod -R 777 /tmp/build/
+
+WORKDIR /tmp/build/lnd

make/release_flags.mk

@@ -1,6 +1,16 @@
 VERSION_TAG = $(shell date +%Y%m%d)-01
 VERSION_CHECK = @$(call print, "Building master with date version tag")
 
+DOCKER_RELEASE_HELPER = docker run \
+  -it \
+  --rm \
+  --user $(shell id -u):$(shell id -g) \
+  -v $(shell pwd):/tmp/build/lnd \
+  -v $(shell bash -c "go env GOCACHE || (mkdir -p /tmp/go-cache; echo /tmp/go-cache)"):/tmp/build/.cache \
+  -v $(shell bash -c "go env GOMODCACHE || (mkdir -p /tmp/go-modcache; echo /tmp/go-modcache)"):/tmp/build/.modcache \
+  -e SKIP_VERSION_CHECK \
+  lnd-release-helper
+
 BUILD_SYSTEM = darwin-amd64 \
 dragonfly-amd64 \
 freebsd-386 \

scripts/release.sh

@@ -13,6 +13,61 @@ LND_VERSION_REGEX="lnd version (.+) commit"
 PKG="github.com/lightningnetwork/lnd"
 PACKAGE=lnd
 
+# Needed for setting file timestamps to get reproducible archives.
+BUILD_DATE="2020-01-01 00:00:00"
+BUILD_DATE_STAMP="202001010000.00"
+
+# reproducible_tar_gzip creates a reproducible tar.gz file of a directory. This
+# includes setting all file timestamps and ownership settings uniformly.
+function reproducible_tar_gzip() {
+  local dir=$1
+  local tar_cmd=tar
+
+  # MacOS has a version of BSD tar which doesn't support setting the --mtime
+  # flag. We need gnu-tar, or gtar for short to be installed for this script to
+  # work properly.
+  tar_version=$(tar --version)
+  if [[ ! "$tar_version" =~ "GNU tar" ]]; then
+    if ! command -v "gtar"; then
+      echo "GNU tar is required but cannot be found!"
+      echo "On MacOS please run 'brew install gnu-tar' to install gtar."
+      exit 1
+    fi
+
+    # We have gtar installed, use that instead.
+    tar_cmd=gtar
+  fi
+
+  # Pin down the timestamp time zone.
+  export TZ=UTC
+
+  find "${dir}" -print0 | LC_ALL=C sort -r -z | $tar_cmd \
+    "--mtime=${BUILD_DATE}" --no-recursion --null --mode=u+rw,go+r-w,a+X \
+    --owner=0 --group=0 --numeric-owner -c -T - | gzip -9n > "${dir}.tar.gz"
+
+  rm -r "${dir}"
+}
+
+# reproducible_zip creates a reproducible zip file of a directory. This
+# includes setting all file timestamps.
+function reproducible_zip() {
+  local dir=$1
+
+  # Pin down file name encoding and timestamp time zone.
+  export TZ=UTC
+
+  # Set the date of each file in the directory that's about to be packaged to
+  # the same timestamp and make sure the same permissions are used everywhere.
+  chmod -R 0755 "${dir}"
+  touch -t "${BUILD_DATE_STAMP}" "${dir}"
+  find "${dir}" -print0 | LC_ALL=C sort -r -z | xargs -0r touch \
+    -t "${BUILD_DATE_STAMP}"
+
+  find "${dir}" | LC_ALL=C sort -r | zip -o -X -r -@ "${dir}.zip"
+
+  rm -r "${dir}"
+}
+
 # green prints one line of green text (if the terminal supports it).
 function green() {
   echo -e "\e[0;32m${1}\e[0m"
@@ -37,7 +92,7 @@ function check_tag_correct() {
   fi
 
   # If a tag is specified, ensure that that tag is present and checked out.
-  if [[ $tag != $(git describe) ]]; then
+  if [[ $tag != $(git describe --tags) ]]; then
     red "tag $tag not checked out"
     exit 1
   fi
@@ -82,20 +137,27 @@ function build_release() {
 
   green " - Packaging vendor"
   go mod vendor
-  tar -czf vendor.tar.gz vendor
+  reproducible_tar_gzip vendor
 
   maindir=$PACKAGE-$tag
   mkdir -p $maindir
+  mv vendor.tar.gz "${maindir}/"
 
-  cp vendor.tar.gz $maindir/
+  # Don't use tag in source directory, otherwise our file names get too long and
-  rm vendor.tar.gz
+  # tar starts to package them non-deterministically.
-  rm -r vendor
+  package_source="${PACKAGE}-source"
 
-  package_source="${maindir}/${PACKAGE}-source-${tag}.tar"
+  # The git archive command doesn't support setting timestamps and file
-  git archive -o "${package_source}" HEAD
+  # permissions. That's why we unpack the tar again, then use our reproducible
-  gzip -f "${package_source}" >"${package_source}.gz"
+  # method to create the final archive.
+  git archive -o "${maindir}/${package_source}.tar" HEAD
 
   cd "${maindir}"
+  mkdir -p ${package_source}
+  tar -xf "${package_source}.tar" -C ${package_source}
+  rm "${package_source}.tar"
+  reproducible_tar_gzip ${package_source}
+  mv "${package_source}.tar.gz" "${package_source}-$tag.tar.gz" 
 
   for i in $sys; do
     os=$(echo $i | cut -f1 -d-)
@@ -120,15 +182,13 @@ function build_release() {
     popd
 
     if [[ $os == "windows" ]]; then
-      zip -r "${dir}.zip" "${dir}"
+      reproducible_zip "${dir}"
     else
-      tar -cvzf "${dir}.tar.gz" "${dir}"
+      reproducible_tar_gzip "${dir}"
     fi
-
-    rm -r "${dir}"
   done
 
-  shasum -a 256 * >manifest-$tag.txt
+  sha256sum * >manifest-$tag.txt
 }
 
 # usage prints the usage of the whole script.