Merge pull request #4911 from guggero/docker-rpc-compile

lnrpc+mobile: use docker to compile/format protos
2021-02-01 13:42:42 +01:00 · 2021-02-01 13:42:42 +01:00 · 315f97ffd5
commit 315f97ffd5
parent bd1c5378bf aff1a0334e
9 changed files with 123 additions and 233 deletions
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@ -16,7 +16,6 @@ env:
  # go needs absolute directories, using the $HOME variable doesn't work here.
  GOCACHE: /home/runner/work/go/pkg/build
  GOPATH: /home/runner/work/go
-  DOWNLOAD_CACHE: /home/runner/work/download_cache
  BITCOIN_VERSION: 0.20.1

  # If you change this value, please change it in the following files as well:
@ -29,10 +28,10 @@ env:

 jobs:
  ########################
-  # RPC compilation check
+  # RPC and mobile compilation check
  ########################
  rpc-check:
-    name: RPC compilation check
+    name: RPC and mobile compilation check
    runs-on: ubuntu-latest
    steps:
      - name: git checkout
@ -54,21 +53,15 @@ jobs:
        with:
          go-version: '~${{ env.GO_VERSION }}'

-      - name: download cache
-        uses: actions/cache@v1
-        with:
-          path: /home/runner/work/download_cache
-          key: lnd-${{ runner.os }}-download-${{ hashFiles('**/install_travis_proto.sh') }}
-          restore-keys: |
-            lnd-${{ runner.os }}-download-${{ hashFiles('**/install_travis_proto.sh') }}
-            lnd-${{ runner.os }}-download-
-
-      - name: install protoc and protobuf libraries
-        run: ./scripts/install_travis_proto.sh
-
      - name: run check
        run: make rpc-check
        
+      - name: build mobile RPC bindings
+        run: make mobile-rpc
+
+      - name: build mobile specific code
+        run: go build --tags="mobile" ./mobile
+
  ########################
  # check commits
  ########################
@ -169,50 +162,6 @@ jobs:
      - name: build release for all architectures
        run: make release

-  ########################
-  # mobile compilation
-  ########################
-  mobile-compile:
-    name: mobile compilation
-    runs-on: ubuntu-latest
-    steps:
-      - name: git checkout
-        uses: actions/checkout@v2
-
-      - name: go cache
-        uses: actions/cache@v1
-        with:
-          path: /home/runner/work/go
-          key: lnd-${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ github.job }}-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            lnd-${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ github.job }}-${{ hashFiles('**/go.sum') }}
-            lnd-${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ github.job }}-
-            lnd-${{ runner.os }}-go-${{ env.GO_VERSION }}-
-            lnd-${{ runner.os }}-go-
-
-      - name: setup go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v2
-        with:
-          go-version: '~${{ env.GO_VERSION }}'
-
-      - name: download cache
-        uses: actions/cache@v1
-        with:
-          path: /home/runner/work/download_cache
-          key: lnd-${{ runner.os }}-download-${{ hashFiles('**/install_travis_proto.sh') }}
-          restore-keys: |
-            lnd-${{ runner.os }}-download-${{ hashFiles('**/install_travis_proto.sh') }}
-            lnd-${{ runner.os }}-download-
-
-      - name: install protoc and protobuf libraries
-        run: ./scripts/install_travis_proto.sh
-
-      - name: build mobile RPC bindings
-        run: make mobile-rpc
-
-      - name: build mobile specific code
-        run: go build --tags="mobile" ./mobile
-
  ########################
  # sample configuration check
  ########################
@ -239,15 +188,6 @@ jobs:
        with:
          go-version: '~${{ env.GO_VERSION }}'

-      - name: download cache
-        uses: actions/cache@v1
-        with:
-          path: /home/runner/work/download_cache
-          key: lnd-${{ runner.os }}-download-${{ hashFiles('**/install_travis_proto.sh') }}
-          restore-keys: |
-            lnd-${{ runner.os }}-download-${{ hashFiles('**/install_travis_proto.sh') }}
-            lnd-${{ runner.os }}-download-
-
      - name: check all command line flags exist in sample-lnd.conf file
        run: make sample-conf-check

--- a/.travis.yml
+++ b/.travis.yml
@ -1,7 +1,6 @@
 language: go
 cache:
  directories:
-    - $DOWNLOAD_CACHE
    - $GOCACHE
    - $GOPATH/pkg/mod
    - $GOPATH/src/github.com/btcsuite
@ -27,7 +26,6 @@ go:
 env:
  global:
    - GOCACHE=$HOME/.go-build
-    - DOWNLOAD_CACHE=$HOME/download_cache
    - BITCOIN_VERSION=0.20.1

 sudo: required
@ -36,11 +34,6 @@ jobs:
  include:
    - stage: Sanity Check
      name: Lint and compile
-      before_script:
-        # Install the RPC tools as a before step so Travis collapses the output
-        # after it's done.
-        - ./scripts/install_travis_proto.sh
-
      script:
        # Step 1: Make sure no diff is produced when compiling with the correct
        # version.
--- a/13
+++ b/13
@ -6,7 +6,6 @@ BTCD_PKG := github.com/btcsuite/btcd
 GOVERALLS_PKG := github.com/mattn/goveralls
 LINT_PKG := github.com/golangci/golangci-lint/cmd/golangci-lint
 GOACC_PKG := github.com/ory/go-acc
-FALAFEL_PKG := github.com/lightninglabs/falafel
 GOIMPORTS_PKG := golang.org/x/tools/cmd/goimports
 GOFUZZ_BUILD_PKG := github.com/dvyukov/go-fuzz/go-fuzz-build
 GOFUZZ_PKG := github.com/dvyukov/go-fuzz/go-fuzz
@ -37,7 +36,6 @@ BTCD_COMMIT := $(shell cat go.mod | \

 LINT_COMMIT := v1.18.0
 GOACC_COMMIT := ddc355013f90fea78d83d3a6c71f1d37ac07ecd5
-FALAFEL_COMMIT := v0.7.1
 GOFUZZ_COMMIT := 21309f307f61

 DEPGET := cd /tmp && GO111MODULE=on go get -v
@ -113,10 +111,6 @@ btcd:
 	@$(call print, "Installing btcd.")
 	$(DEPGET) $(BTCD_PKG)@$(BTCD_COMMIT)

-falafel:
-	@$(call print, "Installing falafel.")
-	$(DEPGET) $(FALAFEL_PKG)@$(FALAFEL_COMMIT)
-
 goimports:
 	@$(call print, "Installing goimports.")
 	$(DEPGET) $(GOIMPORTS_PKG)
@ -264,7 +258,7 @@ list:

 rpc:
 	@$(call print, "Compiling protos.")
-	cd ./lnrpc; ./gen_protos.sh
+	cd ./lnrpc; ./gen_protos_docker.sh

 rpc-format:
 	@$(call print, "Formatting protos.")
@ -279,9 +273,9 @@ sample-conf-check:
 	@$(call print, "Making sure every flag has an example in the sample-lnd.conf file")
 	for flag in $$(GO_FLAGS_COMPLETION=1 go run -tags="$(RELEASE_TAGS)" $(PKG)/cmd/lnd -- | grep -v help | cut -c3-); do if ! grep -q $$flag sample-lnd.conf; then echo "Command line flag --$$flag not added to sample-lnd.conf"; exit 1; fi; done

-mobile-rpc: falafel goimports
+mobile-rpc:
 	@$(call print, "Creating mobile RPC from protos.")
-	cd ./mobile; ./gen_bindings.sh $(FALAFEL_COMMIT)
+	cd ./lnrpc; COMPILE_MOBILE=1 ./gen_protos_docker.sh

 vendor:
 	@$(call print, "Re-creating vendor directory.")
@ -318,7 +312,6 @@ clean:
 	unit \
 	unit-cover \
 	unit-race \
-	falafel \
 	goveralls \
 	travis-race \
 	travis-cover \
--- a/lnrpc/Dockerfile
+++ b/lnrpc/Dockerfile
@ -0,0 +1,31 @@
+FROM golang:1.15.6-buster
+
+RUN apt-get update && apt-get install -y \
+  git \
+  protobuf-compiler='3.6.1*' \
+  clang-format='1:7.0*'
+
+# We don't want any default values for these variables to make sure they're
+# explicitly provided by parsing the go.mod file. Otherwise we might forget to
+# update them here if we bump the versions.
+ARG PROTOC_GEN_VERSION
+ARG GRPC_GATEWAY_VERSION
+
+ENV FALAFEL_VERSION="v0.7.1"
+ENV GOCACHE=/tmp/build/.cache
+ENV GOMODCACHE=/tmp/build/.modcache
+
+RUN cd /tmp \
+  && mkdir -p /tmp/build/.cache \
+  && mkdir -p /tmp/build/.modcache \
+  && export GO111MODULE=on \
+  && go get github.com/golang/protobuf/protoc-gen-go@${PROTOC_GEN_VERSION} \
+  && go get github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway@${GRPC_GATEWAY_VERSION} \
+  && go get github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger@${GRPC_GATEWAY_VERSION} \
+  && go get github.com/lightninglabs/falafel@${FALAFEL_VERSION} \
+  && go get golang.org/x/tools/cmd/goimports \
+  && chmod -R 777 /tmp/build/
+
+WORKDIR /build
+
+CMD ["/bin/bash", "/build/lnrpc/gen_protos.sh"]
--- a/lnrpc/README.md
+++ b/lnrpc/README.md
@ -153,58 +153,21 @@ description):

 ## Generate protobuf definitions

-### Linux
+To compile the `lnrpc/**/*.proto` files and generate the protobuf definitions,
+you need to have [Docker](https://docs.docker.com/get-docker/) and `make`
+installed.

-For linux there is an easy install script that is also used for the Travis CI
-build. Just run the following command (requires `sudo` permissions and the tools
-`make`, `go`, `wget` and `unzip` to be installed) from the repository's root
-folder:
-
-```shell
-⛰  ./scripts/install_travis_proto.sh
-```
-
-### MacOS / Unix like systems
-
-1. Download [v.3.4.0](https://github.com/google/protobuf/releases/tag/v3.4.0) of
-`protoc` for your operating system and add it to your `PATH`.
-For example, if using macOS:
-```shell
-⛰  curl -LO https://github.com/google/protobuf/releases/download/v3.4.0/protoc-3.4.0-osx-x86_64.zip
-⛰  unzip protoc-3.4.0-osx-x86_64.zip -d protoc
-⛰  export PATH=$PWD/protoc/bin:$PATH
-```
-
-2. Install `golang/protobuf` at version `v1.3.2`.
-```shell
-⛰  git clone https://github.com/golang/protobuf $GOPATH/src/github.com/golang/protobuf
-⛰  cd $GOPATH/src/github.com/golang/protobuf
-⛰  git reset --hard v1.3.2
-⛰  make
-```
-
-3. Install 'genproto' at commit `20e1ac93f88cf06d2b1defb90b9e9e126c7dfff6`.
-```shell
-⛰  go get google.golang.org/genproto
-⛰  cd $GOPATH/src/google.golang.org/genproto
-⛰  git reset --hard 20e1ac93f88cf06d2b1defb90b9e9e126c7dfff6
-```
-
-4. Install `grpc-ecosystem/grpc-gateway` at version `v1.14.3`.
-```shell
-⛰  git clone https://github.com/grpc-ecosystem/grpc-gateway $GOPATH/src/github.com/grpc-ecosystem/grpc-gateway
-⛰  cd $GOPATH/src/github.com/grpc-ecosystem/grpc-gateway
-⛰  git reset --hard v1.14.3
-⛰  go install ./protoc-gen-grpc-gateway ./protoc-gen-swagger
-```
-
-5. Run [`gen_protos.sh`](https://github.com/lightningnetwork/lnd/blob/master/lnrpc/gen_protos.sh)
-or `make rpc` to generate new protobuf definitions.
+Simply run `make rpc` to start the compilation process.

 ## Format .proto files

 We use `clang-format` to make sure the `.proto` files are formatted correctly.
-You can install the formatter on Ubuntu by running `apt install clang-format` or on Mac by running `brew install clang-format`.
+
+When running the `make rpc` command, the `.proto` files are also formatted. To
+format the files without also compiling them, you can install the `clang-format`
+formatter on Ubuntu by running `apt install clang-format` or on Mac by running
+`brew install clang-format`.
+The `make format` command should then produce the correct result.

 Consult [this page](http://releases.llvm.org/download.html) to find binaries
 for other operating systems or distributions.
@ -213,7 +176,8 @@ for other operating systems or distributions.

 The following commands are available with `make`:

-* `rpc`: Compile `.proto` files (calls `lnrpc/gen_protos.sh`).
+* `rpc`: Compile and format all `.proto` files using Docker (calls
+  `lnrpc/gen_protos_docker.sh`).
 * `rpc-format`: Formats all `.proto` files according to our formatting rules.
  Requires `clang-format`, see previous chapter.
 * `rpc-check`: Runs both previous commands and makes sure the git work tree is
--- a/lnrpc/gen_protos.sh
+++ b/lnrpc/gen_protos.sh
@ -1,12 +1,16 @@
-#!/bin/sh
+#!/bin/bash

-echo "Generating root gRPC server protos"
+set -e

-PROTOS="rpc.proto walletunlocker.proto **/*.proto"
+# generate compiles the *.pb.go stubs from the *.proto files.
+function generate() {
+  echo "Generating root gRPC server protos"
  
-# For each of the sub-servers, we then generate their protos, but a restricted
-# set as they don't yet require REST proxies, or swagger docs.
-for file in $PROTOS; do
+  PROTOS="rpc.proto walletunlocker.proto **/*.proto"
+  
+  # For each of the sub-servers, we then generate their protos, but a restricted
+  # set as they don't yet require REST proxies, or swagger docs.
+  for file in $PROTOS; do
    DIRECTORY=$(dirname "${file}")
    echo "Generating protos from ${file}, into ${DIRECTORY}"
  
@ -25,4 +29,22 @@ for file in $PROTOS; do
    protoc -I/usr/local/include -I. \
      --swagger_out=logtostderr=true,grpc_api_configuration=rest-annotations.yaml:. \
      "${file}"
-done
+  done
+}
+
+# format formats the *.proto files with the clang-format utility.
+function format() {
+  find . -name "*.proto" -print0 | xargs -0 clang-format --style=file -i
+}
+
+# Compile and format the lnrpc package.
+pushd lnrpc
+format
+generate
+popd
+
+if [[ "$COMPILE_MOBILE" == "1" ]]; then
+  pushd mobile
+  ./gen_bindings.sh $FALAFEL_VERSION
+  popd
+fi
--- a/lnrpc/gen_protos_docker.sh
+++ b/lnrpc/gen_protos_docker.sh
@ -0,0 +1,24 @@
+#!/bin/bash
+
+set -e
+
+# Directory of the script file, independent of where it's called from.
+DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"
+
+PROTOC_GEN_VERSION=$(go list -f '{{.Version}}' -m github.com/golang/protobuf)
+GRPC_GATEWAY_VERSION=$(go list -f '{{.Version}}' -m github.com/grpc-ecosystem/grpc-gateway)
+
+echo "Building protobuf compiler docker image..."
+docker build -q -t lnd-protobuf-builder \
+  --build-arg PROTOC_GEN_VERSION="$PROTOC_GEN_VERSION" \
+  --build-arg GRPC_GATEWAY_VERSION="$GRPC_GATEWAY_VERSION" \
+  .
+
+echo "Compiling and formatting *.proto files..."
+docker run \
+  --rm \
+  --user "$UID:$(id -g)" \
+  -e UID=$UID \
+  -e COMPILE_MOBILE \
+  -v "$DIR/../:/build" \
+  lnd-protobuf-builder
--- a/mobile/gen_bindings.sh
+++ b/mobile/gen_bindings.sh
@ -47,7 +47,6 @@ for file in $PROTOS; do
  echo "Generating mobile protos from ${file}"

  protoc -I/usr/local/include -I. \
-         -I$GOPATH/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \
         --plugin=protoc-gen-custom=$falafel\
         --custom_out=./build \
         --custom_opt="$opts" \
@ -77,7 +76,6 @@ do
    echo "Generating mobile protos from ${file}, with build tag ${tag}"

    protoc -I/usr/local/include -I. \
-           -I$GOPATH/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \
           -I../lnrpc \
           --plugin=protoc-gen-custom=$falafel \
           --custom_out=./build \
--- a/scripts/install_travis_proto.sh
+++ b/scripts/install_travis_proto.sh
@ -1,75 +0,0 @@
-#!/usr/bin/env bash
-
-# Abort on error (-e) and print commands (-v).
-set -ev
-
-# See README.md in lnrpc why we need these specific versions/commits.
-PROTOC_VERSION=3.4.0
-PROTOBUF_VERSION="v1.3.2"
-GENPROTO_VERSION="20e1ac93f88cf06d2b1defb90b9e9e126c7dfff6"
-GRPC_GATEWAY_VERSION="v1.14.3"
-
-# This script is specific to Travis CI so we only need to support linux x64.
-PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip"
-PROTOC_DL_CACHE_DIR="${DOWNLOAD_CACHE:-/tmp/download_cache}/protoc"
-
-# install_protoc copies the cached protoc binary to the $PATH or downloads it
-# if no cached version is found.
-install_protoc() {
-  if [ -f "${PROTOC_DL_CACHE_DIR}/bin/protoc" ]; then
-    echo "Using cached version of protoc"
-  else
-    wget -O /tmp/protoc.zip $PROTOC_URL
-    mkdir -p "${PROTOC_DL_CACHE_DIR}"
-    unzip -o /tmp/protoc.zip -d "${PROTOC_DL_CACHE_DIR}"
-    chmod -R a+rx "${PROTOC_DL_CACHE_DIR}/"
-  fi
-  sudo cp "${PROTOC_DL_CACHE_DIR}/bin/protoc" /usr/local/bin
-  sudo cp -r "${PROTOC_DL_CACHE_DIR}/include" /usr/local
-}
-
-# install_protobuf downloads and compiles the Golang protobuf library that
-# encodes/decodes all protobuf messages from/to Go structs.
-install_protobuf() {
-  local install_path="$GOPATH/src/github.com/golang/protobuf"
-  if [ ! -d "$install_path" ]; then
-    git clone https://github.com/golang/protobuf "$install_path"
-  fi
-  pushd "$install_path"
-  git reset --hard master && git checkout master && git pull
-  git reset --hard $PROTOBUF_VERSION
-  make
-  popd
-}
-
-# install_genproto downloads the Golang protobuf generator that converts the
-# .proto files into Go interface stubs.
-install_genproto() {
-  local install_path="$GOPATH/src/google.golang.org/genproto"
-  if [ ! -d "$install_path" ]; then
-    git clone https://github.com/google/go-genproto "$install_path"
-  fi
-  pushd "$install_path"
-  git reset --hard master && git checkout master && git pull
-  git reset --hard $GENPROTO_VERSION
-  popd
-}
-
-# install_grpc_gateway downloads and installs the gRPC gateway that converts
-# .proto files into REST gateway code.
-install_grpc_gateway() {
-  local install_path="$GOPATH/src/github.com/grpc-ecosystem/grpc-gateway"
-  if [ ! -d "$install_path" ]; then
-    git clone https://github.com/grpc-ecosystem/grpc-gateway "$install_path"
-  fi
-  pushd "$install_path"
-  git reset --hard master && git checkout master && git pull
-  git reset --hard $GRPC_GATEWAY_VERSION
-  GO111MODULE=on go install ./protoc-gen-grpc-gateway ./protoc-gen-swagger
-  popd
-}
-
-install_protoc
-install_protobuf
-install_genproto
-install_grpc_gateway