Merge pull request #4914 from guggero/docker-verification

docker: add verification script

.github/workflows/release.yaml

@@ -70,42 +70,32 @@ jobs:
 
             ```
             curl https://keybase.io/bitconner/pgp_keys.asc | gpg --import
+            curl https://keybase.io/roasbeef/pgp_keys.asc | gpg --import
             ```
 
-            Once you have the required PGP keys, you can verify the release (assuming `manifest-${{ env.RELEASE_VERSION }}.txt` and `manifest-${{ env.RELEASE_VERSION }}.txt.sig` are in the current directory) with:
+            Once you have the required PGP keys, you can verify the release (assuming `manifest-roasbeef-${{ env.RELEASE_VERSION }}.txt.asc` is in the current directory) with:
 
             ```
-            gpg --verify manifest-${{ env.RELEASE_VERSION }}.txt.sig
+            gpg --verify manifest-roasbeef-${{ env.RELEASE_VERSION }}.txt.asc
             ```
 
             You should see the following if the verification was successful:
 
             ```
-            gpg: assuming signed data in 'manifest-${{ env.RELEASE_VERSION }}.txt'
-            gpg: Signature made Thu Oct  1 16:38:32 2020 PDT
-            gpg:                using RSA key 9C8D61868A7C492003B2744EE7D737B67FA592C7
-            gpg: Good signature from "Conner Fromknecht <conner@lightning.engineering>" [ultimate]
-            ```
-
-            That will verify the signature of the manifest file, which ensures integrity and authenticity of the archive you've downloaded locally containing the binaries. Next, depending on your operating system, you should then re-compute the `sha256` hash of the archive with `shasum -a 256 <filename>`, compare it with the corresponding one in the manifest file, and ensure they match *exactly*.
-
-
-            For this release roasbeef's signature is the secondary signature which can be verified with the following command: 
-            ```
-            gpg --verify roasbeef-manifest-${{ env.RELEASE_VERSION }}.txt.sig manifest-${{ env.RELEASE_VERSION }}.txt
             gpg: Signature made Wed Sep 30 17:35:20 2020 PDT
             gpg:                using RSA key 4AB7F8DA6FAEBB3B70B1F903BC13F65E2DC84465
             gpg: Good signature from "Olaoluwa Osuntokun <laolu32@gmail.com>" [ultimate]
             ```
 
+            That will verify the signature of the manifest file, which ensures integrity and authenticity of the archive you've downloaded locally containing the binaries. Next, depending on your operating system, you should then re-compute the `sha256` hash of the archive with `shasum -a 256 <filename>`, compare it with the corresponding one in the manifest file, and ensure they match *exactly*.
+
             ## Verifying the Release Timestamp 
 
-            From this new version onwards, in addition time-stamping the _git tag_ with [OpenTimeStamps](https://opentimestamps.org/), we'll also now timestamp the manifest file along with its signature. Two new files are now included along with the rest of our release artifacts: ` manifest-${{ env.RELEASE_VERSION }}.txt.sig.ots` and `manifest-${{ env.RELEASE_VERSION }}.txt.ots`.
+            From this new version onwards, in addition time-stamping the _git tag_ with [OpenTimeStamps](https://opentimestamps.org/), we'll also now timestamp the manifest file along with its signature. Two new files are now included along with the rest of our release artifacts: ` manifest-roasbeef-${{ env.RELEASE_VERSION }}.txt.asc.ots`.
 
             Assuming you have the opentimestamps client installed locally, the timestamps can be verified with the following commands: 
             ```
-            ots verify manifest-${{ env.RELEASE_VERSION }}.txt.ots
+            ots verify manifest-roasbeef-${{ env.RELEASE_VERSION }}.txt.asc.ots -f manifest-roasbeef-${{ env.RELEASE_VERSION }}.txt.asc
-            ots verify manifest-${{ env.RELEASE_VERSION }}.txt.sig.ots -f roasbeef-manifest-${{ env.RELEASE_VERSION }}.txt.sig
             ```
 
             Alternatively, [the open timestamps website](https://opentimestamps.org/) can be used to verify timestamps if one doesn't have a `bitcoind` instance accessible locally. 
@@ -129,6 +119,18 @@ jobs:
             gpg: Good signature from "Olaoluwa Osuntokun <laolu32@gmail.com>" [ultimate]
             ```
 
+            ## Verifying the Docker Images
+
+            To verify the `lnd` and `lncli` binaries inside the docker images against the signed, reproducible release binaries, there is a verification script in the image that can be called (before starting the container for example):
+
+            ```shell
+            $ docker pull lightninglabs/lnd:${{ env.RELEASE_VERSION }}
+            $ docker run --rm --entrypoint="" lightninglabs/lnd:${{ env.RELEASE_VERSION }} /verify-install.sh
+            $ OK=$?
+            $ if [ "$OK" -ne "0" ]; then echo "Verification failed!"; exit 1; done
+            $ docker run lightninglabs/lnd [command-line options]
+            ```
+
             # Building the Contained Release 
 
             Users are able to rebuild the target release themselves without having to fetch any of the dependencies. In order to do so, assuming

Dockerfile

@@ -23,8 +23,7 @@ RUN apk add --no-cache --update alpine-sdk \
 &&  git clone https://github.com/lightningnetwork/lnd /go/src/github.com/lightningnetwork/lnd \
 &&  cd /go/src/github.com/lightningnetwork/lnd \
 &&  git checkout $checkout \
-&&  make \
+&&  make release-install
-&&  make install tags="signrpc walletrpc chainrpc invoicesrpc"
 
 # Start a new, final image.
 FROM alpine as final
@@ -32,15 +31,24 @@ FROM alpine as final
 # Define a root volume for data persistence.
 VOLUME /root/.lnd
 
-# Add bash, jq and ca-certs, for quality of life and SSL-related reasons.
+# Add utilities for quality of life and SSL-related reasons. We also require
+# curl and gpg for the signature verification script.
 RUN apk --no-cache add \
     bash \
     jq \
-    ca-certificates
+    ca-certificates \
+    gnupg \
+    curl
 
 # Copy the binaries from the builder image.
 COPY --from=builder /go/bin/lncli /bin/
 COPY --from=builder /go/bin/lnd /bin/
+COPY --from=builder /go/src/github.com/lightningnetwork/lnd/scripts/verify-install.sh /
+
+# Store the SHA256 hash of the binaries that were just produced for later
+# verification.
+RUN sha256sum /bin/lnd /bin/lncli > /shasums.txt \
+  && cat /shasums.txt
 
 # Expose lnd ports (p2p, rpc).
 EXPOSE 9735 10009

Makefile

@@ -151,6 +151,11 @@ install:
 	$(GOINSTALL) -tags="${tags}" $(LDFLAGS) $(PKG)/cmd/lnd
 	$(GOINSTALL) -tags="${tags}" $(LDFLAGS) $(PKG)/cmd/lncli
 
+release-install:
+	@$(call print, "Installing release lnd and lncli.")
+	env CGO_ENABLED=0 $(GOINSTALL) -v -trimpath -ldflags="$(RELEASE_LDFLAGS)" -tags="$(RELEASE_TAGS)" $(PKG)/cmd/lnd
+	env CGO_ENABLED=0 $(GOINSTALL) -v -trimpath -ldflags="$(RELEASE_LDFLAGS)" -tags="$(RELEASE_TAGS)" $(PKG)/cmd/lncli
+
 release:
 	@$(call print, "Releasing lnd and lncli binaries.")
 	$(VERSION_CHECK)

docs/DOCKER.md

@@ -21,7 +21,7 @@ spins up a `btcd` backend alongside `lnd`. Check out the documentation at
 [docker/README.md](../docker/README.md) to learn more about how to use that
 setup to create a small local Lightning Network.
 
-## Production
+## Production (manual build)
 
 To use Docker in a production environment, you can run `lnd` by creating a
 Docker container, adding the appropriate command-line options as parameters.
@@ -37,10 +37,36 @@ It is recommended that you checkout the latest released tag.
 You can continue by creating and running the container:
 
 ```
-$ docker run lnd [command-line options]
+$ docker run myrepository/lnd [command-line options]
 ```
 
-Note: there currently are no automated docker image builds available.
+## Production (official images)
+
+Starting with `lnd v0.12.0-beta`, there are official, automatically built docker
+images of `lnd` available in the
+[`lightninglabs/lnd` repository on Docker Hub](https://hub.docker.com/r/lightninglabs/lnd).
+
+You can just pull those images by specifying a release tag:
+
+```shell
+$ docker pull lightninglabs/lnd:v0.12.0-beta
+$ docker run lightninglabs/lnd [command-line options]
+```
+
+### Verifying docker images
+
+To verify the `lnd` and `lncli` binaries inside the docker images against the
+signed, [reproducible release binaries](release.md), there is a verification
+script in the image that can be called (before starting the container for
+example):
+
+```shell
+$ docker pull lightninglabs/lnd:v0.12.0-beta
+$ docker run --rm --entrypoint="" lightninglabs/lnd:v0.12.0-beta /verify-install.sh
+$ OK=$?
+$ if [ "$OK" -ne "0" ]; then echo "Verification failed!"; exit 1; done
+$ docker run lightninglabs/lnd [command-line options]
+```
 
 ## Volumes
 

docs/release.md

@@ -85,3 +85,19 @@ and `go` (matching the same version used in the release):
    release script and recompute the `SHA256` hash of the release binaries (lnd
    and lncli) with `shasum -a 256 <filename>`. These should match __exactly__
    as the ones noted above.
+
+## Verifying docker images
+
+To verify the `lnd` and `lncli` binaries inside the
+[official provided docker images](https://hub.docker.com/r/lightninglabs/lnd)
+against the signed, reproducible release binaries, there is a verification
+script in the image that can be called (before starting the container for
+example):
+
+```shell
+$ docker pull lightninglabs/lnd:v0.12.0-beta
+$ docker run --rm --entrypoint="" lightninglabs/lnd:v0.12.0-beta /verify-install.sh
+$ OK=$?
+$ if [ "$OK" -ne "0" ]; then echo "Verification failed!"; exit 1; done
+$ docker run lightninglabs/lnd [command-line options]
+```

scripts/release.sh

@@ -181,6 +181,10 @@ function build_release() {
     env CGO_ENABLED=0 GOOS=$os GOARCH=$arch GOARM=$arm go build -v -trimpath -ldflags="${ldflags}" -tags="${buildtags}" ${PKG}/cmd/lncli
     popd
 
+    # Add the hashes for the individual binaries as well for easy verification
+    # of a single installed binary.
+    sha256sum "${dir}/"* >> "manifest-$tag.txt" 
+
     if [[ $os == "windows" ]]; then
       reproducible_zip "${dir}"
     else
@@ -188,7 +192,10 @@ function build_release() {
     fi
   done
 
-  sha256sum * >manifest-$tag.txt
+  # Add the hash of the packages too, then sort by the second column (name).
+  sha256sum lnd-* vendor* >> "manifest-$tag.txt"
+  LC_ALL=C sort -k2 -o "manifest-$tag.txt" "manifest-$tag.txt"
+  cat "manifest-$tag.txt"
 }
 
 # usage prints the usage of the whole script.

scripts/verify-install.sh

@@ -0,0 +1,148 @@
+#!/bin/bash
+
+REPO=lightningnetwork
+PROJECT=lnd
+
+RELEASE_URL=https://github.com/$REPO/$PROJECT/releases
+API_URL=https://api.github.com/repos/$REPO/$PROJECT/releases
+SIGNATURE_SELECTOR=". | select(.name | test(\"manifest-.*(\\\\.txt\\\\.asc)$\")) | .name"
+HEADER_JSON="Accept: application/json"
+HEADER_GH_JSON="Accept: application/vnd.github.v3+json"
+
+# All keys that can sign lnd releases. The key must be downloadable/importable
+# from the URL given after the space.
+KEYS=()
+KEYS+=("F4FC70F07310028424EFC20A8E4256593F177720 https://keybase.io/guggero/pgp_keys.asc")
+KEYS+=("15E7ECF257098A4EF91655EB4CA7FE54A6213C91 https://keybase.io/carlakirkcohen/pgp_keys.asc")
+KEYS+=("9C8D61868A7C492003B2744EE7D737B67FA592C7 https://keybase.io/bitconner/pgp_keys.asc")
+KEYS+=("E4D85299674B2D31FAA1892E372CBD7633C61696 https://keybase.io/roasbeef/pgp_keys.asc")
+KEYS+=("729E9D9D92C75A5FBFEEE057B5DD717BEF7CA5B1 https://keybase.io/wpaulino/pgp_keys.asc")
+KEYS+=("7E81EF6B9989A9CC93884803118759E83439A9B1 https://keybase.io/eugene_/pgp_keys.asc")
+
+function check_command() {
+  echo -n "Checking if $1 is installed... "
+  if ! command -v "$1"; then
+    echo "ERROR: $1 is not installed or not in PATH!"
+    exit 1
+  fi
+}
+
+check_command curl
+check_command jq
+check_command gpg
+check_command lnd
+check_command lncli
+
+LND_BIN=$(which lnd)
+LNCLI_BIN=$(which lncli)
+
+LND_VERSION=$(lnd --version | cut -d'=' -f2)
+LNCLI_VERSION=$(lncli --version | cut -d'=' -f2)
+LND_SUM=$(sha256sum $LND_BIN | cut -d' ' -f1)
+LNCLI_SUM=$(sha256sum $LNCLI_BIN | cut -d' ' -f1)
+
+echo "Detected lnd version $LND_VERSION with SHA256 sum $LND_SUM"
+echo "Detected lncli version $LNCLI_VERSION with SHA256 sum $LNCLI_SUM"
+
+# Make sure lnd and lncli are installed with the same version.
+if [[ "$LNCLI_VERSION" != "$LND_VERSION" ]]; then
+  echo "ERROR: Version $LNCLI_VERSION of lncli does not match $LND_VERSION of lnd!"
+  exit 1
+fi
+
+# If we're inside the docker image, there should be a shasums.txt file in the
+# root directory. If that's the case, we first want to make sure we still have
+# the same hash as we did when building the image.
+if [[ -f /shasums.txt ]]; then
+  if ! grep -q "$LND_SUM" /shasums.txt; then
+    echo "ERROR: Hash $LND_SUM for lnd not found in /shasums.txt: "
+    cat /shasums.txt
+    exit 1
+  fi
+  if ! grep -q "$LNCLI_SUM" /shasums.txt; then
+    echo "ERROR: Hash $LNCLI_SUM for lnd not found in /shasums.txt: "
+    cat /shasums.txt
+    exit 1
+  fi
+fi
+
+# Import all the signing keys.
+for key in "${KEYS[@]}"; do
+  KEY_ID=$(echo $key | cut -d' ' -f1)
+  IMPORT_URL=$(echo $key | cut -d' ' -f2)
+  echo "Downloading and importing key $KEY_ID from $IMPORT_URL"
+  curl -L -s $IMPORT_URL | gpg --import
+
+  # Make sure we actually imported the correct key.
+  if ! gpg --list-key "$KEY_ID"; then
+    echo "ERROR: Imported key from $IMPORT_URL doesn't match ID $KEY_ID."
+  fi
+done
+
+echo ""
+
+# Download the JSON of the release itself. That'll contain the release ID we need for the next call.
+RELEASE_JSON=$(curl -L -s -H "$HEADER_JSON" "$RELEASE_URL/$LND_VERSION")
+
+TAG_NAME=$(echo $RELEASE_JSON | jq -r '.tag_name')
+RELEASE_ID=$(echo $RELEASE_JSON | jq -r '.id')
+echo "Release $TAG_NAME found with ID $RELEASE_ID"
+
+# Now download the asset list and filter by manifests and signatures.
+ASSETS=$(curl -L -s -H "$HEADER_GH_JSON" "$API_URL/$RELEASE_ID" | jq -c '.assets[]')
+SIGNATURES=$(echo $ASSETS | jq -r "$SIGNATURE_SELECTOR")
+
+# Download all "manifest-*.txt.asc" as those contain both the hashes that were
+# signed and the signature itself (=detached sig).
+TEMP_DIR=$(mktemp -d /tmp/lnd-sig-verification-XXXXXX)
+for signature in $SIGNATURES; do
+  echo "Downloading $signature"
+  curl -L -s -o "$TEMP_DIR/$signature" "$RELEASE_URL/download/$LND_VERSION/$signature"
+done
+
+echo ""
+cd $TEMP_DIR || exit 1
+
+NUM_CHECKS=0
+for signature in $SIGNATURES; do
+  # First make sure the downloaded signature file is valid.
+  echo "Verifying $signature"
+  if ! gpg --verify "$signature" 2>&1 | grep -q "Good signature"; then
+    echo "ERROR: Did not get valid signature for $signature!"
+    exit 1
+  fi
+
+  echo "Signature for $signature checks out: "
+  gpg --verify "$signature" 2>&1 | grep "using"
+
+  echo ""
+
+  # Then make sure that the hash of the installed binaries can be found in the
+  # signed list of hashes.
+  if ! grep -q "$LND_SUM" "$signature"; then
+    echo "ERROR: Hash $LND_SUM for lnd not found in $signature: "
+    cat "$signature"
+    exit 1
+  fi
+
+  if ! grep -q "$LNCLI_SUM" "$signature"; then
+    echo "ERROR: Hash $LNCLI_SUM for lncli not found in $signature: "
+    cat "$signature"
+    exit 1
+  fi
+
+  echo "Verified lnd and lncli hashes against $signature"
+  ((NUM_CHECKS=NUM_CHECKS+1))
+done
+
+# We want at least one signature that signs the hashes of the binaries we have
+# installed. If we arrive here without exiting, it means no signature manifests
+# were uploaded (yet) with the correct naming pattern.
+if [[ $NUM_CHECKS -lt 1 ]]; then
+  echo "ERROR: No valid signatures found!"
+  echo "Make sure the release $LND_VERSION contains any signed manifests."
+  exit 1
+fi
+
+echo ""
+echo "SUCCESS! Verified lnd and lncli against $NUM_CHECKS signature(s)."