scripts: package release artifacts deterministically

A number of tricks need to be applied to the tar and zip commands
respectively to make sure they always produce deterministic archives.
This includes setting the file timestamps to a fixed date and setting
specific ownership attributes (tar) or no attributes at all (zip).

scripts/release.sh

@@ -13,6 +13,61 @@ LND_VERSION_REGEX="lnd version (.+) commit"
 PKG="github.com/lightningnetwork/lnd"
 PACKAGE=lnd
 
+# Needed for setting file timestamps to get reproducible archives.
+BUILD_DATE="2020-01-01 00:00:00"
+BUILD_DATE_STAMP="202001010000.00"
+
+# reproducible_tar_gzip creates a reproducible tar.gz file of a directory. This
+# includes setting all file timestamps and ownership settings uniformly.
+function reproducible_tar_gzip() {
+  local dir=$1
+  local tar_cmd=tar
+
+  # MacOS has a version of BSD tar which doesn't support setting the --mtime
+  # flag. We need gnu-tar, or gtar for short to be installed for this script to
+  # work properly.
+  tar_version=$(tar --version)
+  if [[ ! "$tar_version" =~ "GNU tar" ]]; then
+    if ! command -v "gtar"; then
+      echo "GNU tar is required but cannot be found!"
+      echo "On MacOS please run 'brew install gnu-tar' to install gtar."
+      exit 1
+    fi
+
+    # We have gtar installed, use that instead.
+    tar_cmd=gtar
+  fi
+
+  # Pin down the timestamp time zone.
+  export TZ=UTC
+
+  find "${dir}" -print0 | LC_ALL=C sort -r -z | $tar_cmd \
+    "--mtime=${BUILD_DATE}" --no-recursion --null --mode=u+rw,go+r-w,a+X \
+    --owner=0 --group=0 --numeric-owner -c -T - | gzip -9n > "${dir}.tar.gz"
+
+  rm -r "${dir}"
+}
+
+# reproducible_zip creates a reproducible zip file of a directory. This
+# includes setting all file timestamps.
+function reproducible_zip() {
+  local dir=$1
+
+  # Pin down file name encoding and timestamp time zone.
+  export TZ=UTC
+
+  # Set the date of each file in the directory that's about to be packaged to
+  # the same timestamp and make sure the same permissions are used everywhere.
+  chmod -R 0755 "${dir}"
+  touch -t "${BUILD_DATE_STAMP}" "${dir}"
+  find "${dir}" -print0 | LC_ALL=C sort -r -z | xargs -0r touch \
+    -t "${BUILD_DATE_STAMP}"
+
+  find "${dir}" | LC_ALL=C sort -r | zip -o -X -r -@ "${dir}.zip"
+
+  rm -r "${dir}"
+}
+
 # green prints one line of green text (if the terminal supports it).
 function green() {
   echo -e "\e[0;32m${1}\e[0m"
@@ -82,20 +137,27 @@ function build_release() {
 
   green " - Packaging vendor"
   go mod vendor
-  tar -czf vendor.tar.gz vendor
+  reproducible_tar_gzip vendor
 
   maindir=$PACKAGE-$tag
   mkdir -p $maindir
+  mv vendor.tar.gz "${maindir}/"
 
-  cp vendor.tar.gz $maindir/
+  # Don't use tag in source directory, otherwise our file names get too long and
-  rm vendor.tar.gz
+  # tar starts to package them non-deterministically.
-  rm -r vendor
+  package_source="${PACKAGE}-source"
 
-  package_source="${maindir}/${PACKAGE}-source-${tag}.tar"
+  # The git archive command doesn't support setting timestamps and file
-  git archive -o "${package_source}" HEAD
+  # permissions. That's why we unpack the tar again, then use our reproducible
-  gzip -f "${package_source}" >"${package_source}.gz"
+  # method to create the final archive.
+  git archive -o "${maindir}/${package_source}.tar" HEAD
 
   cd "${maindir}"
+  mkdir -p ${package_source}
+  tar -xf "${package_source}.tar" -C ${package_source}
+  rm "${package_source}.tar"
+  reproducible_tar_gzip ${package_source}
+  mv "${package_source}.tar.gz" "${package_source}-$tag.tar.gz" 
 
   for i in $sys; do
     os=$(echo $i | cut -f1 -d-)
@@ -120,15 +182,13 @@ function build_release() {
     popd
 
     if [[ $os == "windows" ]]; then
-      zip -r "${dir}.zip" "${dir}"
+      reproducible_zip "${dir}"
     else
-      tar -cvzf "${dir}.tar.gz" "${dir}"
+      reproducible_tar_gzip "${dir}"
     fi
-
-    rm -r "${dir}"
   done
 
-  shasum -a 256 * >manifest-$tag.txt
+  sha256sum * >manifest-$tag.txt
 }
 
 # usage prints the usage of the whole script.