diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c9a05388..866047dc 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -16,7 +16,6 @@ env: # go needs absolute directories, using the $HOME variable doesn't work here. GOCACHE: /home/runner/work/go/pkg/build GOPATH: /home/runner/work/go - DOWNLOAD_CACHE: /home/runner/work/download_cache BITCOIN_VERSION: 0.20.1 # If you change this value, please change it in the following files as well: @@ -29,10 +28,10 @@ env: jobs: ######################## - # RPC compilation check + # RPC and mobile compilation check ######################## rpc-check: - name: RPC compilation check + name: RPC and mobile compilation check runs-on: ubuntu-latest steps: - name: git checkout @@ -54,20 +53,14 @@ jobs: with: go-version: '~${{ env.GO_VERSION }}' - - name: download cache - uses: actions/cache@v1 - with: - path: /home/runner/work/download_cache - key: lnd-${{ runner.os }}-download-${{ hashFiles('**/install_travis_proto.sh') }} - restore-keys: | - lnd-${{ runner.os }}-download-${{ hashFiles('**/install_travis_proto.sh') }} - lnd-${{ runner.os }}-download- - - - name: install protoc and protobuf libraries - run: ./scripts/install_travis_proto.sh - - name: run check run: make rpc-check + + - name: build mobile RPC bindings + run: make mobile-rpc + + - name: build mobile specific code + run: go build --tags="mobile" ./mobile ######################## # check commits @@ -169,50 +162,6 @@ jobs: - name: build release for all architectures run: make release - ######################## - # mobile compilation - ######################## - mobile-compile: - name: mobile compilation - runs-on: ubuntu-latest - steps: - - name: git checkout - uses: actions/checkout@v2 - - - name: go cache - uses: actions/cache@v1 - with: - path: /home/runner/work/go - key: lnd-${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ github.job }}-${{ hashFiles('**/go.sum') }} - restore-keys: | - lnd-${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ github.job }}-${{ hashFiles('**/go.sum') }} - lnd-${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ github.job }}- - lnd-${{ runner.os }}-go-${{ env.GO_VERSION }}- - lnd-${{ runner.os }}-go- - - - name: setup go ${{ env.GO_VERSION }} - uses: actions/setup-go@v2 - with: - go-version: '~${{ env.GO_VERSION }}' - - - name: download cache - uses: actions/cache@v1 - with: - path: /home/runner/work/download_cache - key: lnd-${{ runner.os }}-download-${{ hashFiles('**/install_travis_proto.sh') }} - restore-keys: | - lnd-${{ runner.os }}-download-${{ hashFiles('**/install_travis_proto.sh') }} - lnd-${{ runner.os }}-download- - - - name: install protoc and protobuf libraries - run: ./scripts/install_travis_proto.sh - - - name: build mobile RPC bindings - run: make mobile-rpc - - - name: build mobile specific code - run: go build --tags="mobile" ./mobile - ######################## # sample configuration check ######################## @@ -239,15 +188,6 @@ jobs: with: go-version: '~${{ env.GO_VERSION }}' - - name: download cache - uses: actions/cache@v1 - with: - path: /home/runner/work/download_cache - key: lnd-${{ runner.os }}-download-${{ hashFiles('**/install_travis_proto.sh') }} - restore-keys: | - lnd-${{ runner.os }}-download-${{ hashFiles('**/install_travis_proto.sh') }} - lnd-${{ runner.os }}-download- - - name: check all command line flags exist in sample-lnd.conf file run: make sample-conf-check diff --git a/.travis.yml b/.travis.yml index b3fcbe0b..942b95c9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,7 +1,6 @@ language: go cache: directories: - - $DOWNLOAD_CACHE - $GOCACHE - $GOPATH/pkg/mod - $GOPATH/src/github.com/btcsuite @@ -27,7 +26,6 @@ go: env: global: - GOCACHE=$HOME/.go-build - - DOWNLOAD_CACHE=$HOME/download_cache - BITCOIN_VERSION=0.20.1 sudo: required @@ -36,11 +34,6 @@ jobs: include: - stage: Sanity Check name: Lint and compile - before_script: - # Install the RPC tools as a before step so Travis collapses the output - # after it's done. - - ./scripts/install_travis_proto.sh - script: # Step 1: Make sure no diff is produced when compiling with the correct # version. diff --git a/Makefile b/Makefile index b2156a22..8cc45fab 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,6 @@ BTCD_PKG := github.com/btcsuite/btcd GOVERALLS_PKG := github.com/mattn/goveralls LINT_PKG := github.com/golangci/golangci-lint/cmd/golangci-lint GOACC_PKG := github.com/ory/go-acc -FALAFEL_PKG := github.com/lightninglabs/falafel GOIMPORTS_PKG := golang.org/x/tools/cmd/goimports GOFUZZ_BUILD_PKG := github.com/dvyukov/go-fuzz/go-fuzz-build GOFUZZ_PKG := github.com/dvyukov/go-fuzz/go-fuzz @@ -37,7 +36,6 @@ BTCD_COMMIT := $(shell cat go.mod | \ LINT_COMMIT := v1.18.0 GOACC_COMMIT := ddc355013f90fea78d83d3a6c71f1d37ac07ecd5 -FALAFEL_COMMIT := v0.7.1 GOFUZZ_COMMIT := 21309f307f61 DEPGET := cd /tmp && GO111MODULE=on go get -v @@ -113,10 +111,6 @@ btcd: @$(call print, "Installing btcd.") $(DEPGET) $(BTCD_PKG)@$(BTCD_COMMIT) -falafel: - @$(call print, "Installing falafel.") - $(DEPGET) $(FALAFEL_PKG)@$(FALAFEL_COMMIT) - goimports: @$(call print, "Installing goimports.") $(DEPGET) $(GOIMPORTS_PKG) @@ -264,7 +258,7 @@ list: rpc: @$(call print, "Compiling protos.") - cd ./lnrpc; ./gen_protos.sh + cd ./lnrpc; ./gen_protos_docker.sh rpc-format: @$(call print, "Formatting protos.") @@ -279,9 +273,9 @@ sample-conf-check: @$(call print, "Making sure every flag has an example in the sample-lnd.conf file") for flag in $$(GO_FLAGS_COMPLETION=1 go run -tags="$(RELEASE_TAGS)" $(PKG)/cmd/lnd -- | grep -v help | cut -c3-); do if ! grep -q $$flag sample-lnd.conf; then echo "Command line flag --$$flag not added to sample-lnd.conf"; exit 1; fi; done -mobile-rpc: falafel goimports +mobile-rpc: @$(call print, "Creating mobile RPC from protos.") - cd ./mobile; ./gen_bindings.sh $(FALAFEL_COMMIT) + cd ./lnrpc; COMPILE_MOBILE=1 ./gen_protos_docker.sh vendor: @$(call print, "Re-creating vendor directory.") @@ -318,7 +312,6 @@ clean: unit \ unit-cover \ unit-race \ - falafel \ goveralls \ travis-race \ travis-cover \ diff --git a/lnrpc/Dockerfile b/lnrpc/Dockerfile new file mode 100644 index 00000000..fa04a78c --- /dev/null +++ b/lnrpc/Dockerfile @@ -0,0 +1,31 @@ +FROM golang:1.15.6-buster + +RUN apt-get update && apt-get install -y \ + git \ + protobuf-compiler='3.6.1*' \ + clang-format='1:7.0*' + +# We don't want any default values for these variables to make sure they're +# explicitly provided by parsing the go.mod file. Otherwise we might forget to +# update them here if we bump the versions. +ARG PROTOC_GEN_VERSION +ARG GRPC_GATEWAY_VERSION + +ENV FALAFEL_VERSION="v0.7.1" +ENV GOCACHE=/tmp/build/.cache +ENV GOMODCACHE=/tmp/build/.modcache + +RUN cd /tmp \ + && mkdir -p /tmp/build/.cache \ + && mkdir -p /tmp/build/.modcache \ + && export GO111MODULE=on \ + && go get github.com/golang/protobuf/protoc-gen-go@${PROTOC_GEN_VERSION} \ + && go get github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway@${GRPC_GATEWAY_VERSION} \ + && go get github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger@${GRPC_GATEWAY_VERSION} \ + && go get github.com/lightninglabs/falafel@${FALAFEL_VERSION} \ + && go get golang.org/x/tools/cmd/goimports \ + && chmod -R 777 /tmp/build/ + +WORKDIR /build + +CMD ["/bin/bash", "/build/lnrpc/gen_protos.sh"] diff --git a/lnrpc/README.md b/lnrpc/README.md index 1e9237c5..8f3b0ec6 100644 --- a/lnrpc/README.md +++ b/lnrpc/README.md @@ -153,58 +153,21 @@ description): ## Generate protobuf definitions -### Linux +To compile the `lnrpc/**/*.proto` files and generate the protobuf definitions, +you need to have [Docker](https://docs.docker.com/get-docker/) and `make` +installed. -For linux there is an easy install script that is also used for the Travis CI -build. Just run the following command (requires `sudo` permissions and the tools -`make`, `go`, `wget` and `unzip` to be installed) from the repository's root -folder: - -```shell -⛰ ./scripts/install_travis_proto.sh -``` - -### MacOS / Unix like systems - -1. Download [v.3.4.0](https://github.com/google/protobuf/releases/tag/v3.4.0) of -`protoc` for your operating system and add it to your `PATH`. -For example, if using macOS: -```shell -⛰ curl -LO https://github.com/google/protobuf/releases/download/v3.4.0/protoc-3.4.0-osx-x86_64.zip -⛰ unzip protoc-3.4.0-osx-x86_64.zip -d protoc -⛰ export PATH=$PWD/protoc/bin:$PATH -``` - -2. Install `golang/protobuf` at version `v1.3.2`. -```shell -⛰ git clone https://github.com/golang/protobuf $GOPATH/src/github.com/golang/protobuf -⛰ cd $GOPATH/src/github.com/golang/protobuf -⛰ git reset --hard v1.3.2 -⛰ make -``` - -3. Install 'genproto' at commit `20e1ac93f88cf06d2b1defb90b9e9e126c7dfff6`. -```shell -⛰ go get google.golang.org/genproto -⛰ cd $GOPATH/src/google.golang.org/genproto -⛰ git reset --hard 20e1ac93f88cf06d2b1defb90b9e9e126c7dfff6 -``` - -4. Install `grpc-ecosystem/grpc-gateway` at version `v1.14.3`. -```shell -⛰ git clone https://github.com/grpc-ecosystem/grpc-gateway $GOPATH/src/github.com/grpc-ecosystem/grpc-gateway -⛰ cd $GOPATH/src/github.com/grpc-ecosystem/grpc-gateway -⛰ git reset --hard v1.14.3 -⛰ go install ./protoc-gen-grpc-gateway ./protoc-gen-swagger -``` - -5. Run [`gen_protos.sh`](https://github.com/lightningnetwork/lnd/blob/master/lnrpc/gen_protos.sh) -or `make rpc` to generate new protobuf definitions. +Simply run `make rpc` to start the compilation process. ## Format .proto files We use `clang-format` to make sure the `.proto` files are formatted correctly. -You can install the formatter on Ubuntu by running `apt install clang-format` or on Mac by running `brew install clang-format`. + +When running the `make rpc` command, the `.proto` files are also formatted. To +format the files without also compiling them, you can install the `clang-format` +formatter on Ubuntu by running `apt install clang-format` or on Mac by running +`brew install clang-format`. +The `make format` command should then produce the correct result. Consult [this page](http://releases.llvm.org/download.html) to find binaries for other operating systems or distributions. @@ -213,7 +176,8 @@ for other operating systems or distributions. The following commands are available with `make`: -* `rpc`: Compile `.proto` files (calls `lnrpc/gen_protos.sh`). +* `rpc`: Compile and format all `.proto` files using Docker (calls + `lnrpc/gen_protos_docker.sh`). * `rpc-format`: Formats all `.proto` files according to our formatting rules. Requires `clang-format`, see previous chapter. * `rpc-check`: Runs both previous commands and makes sure the git work tree is diff --git a/lnrpc/gen_protos.sh b/lnrpc/gen_protos.sh index 78c3270d..13892fee 100755 --- a/lnrpc/gen_protos.sh +++ b/lnrpc/gen_protos.sh @@ -1,28 +1,50 @@ -#!/bin/sh +#!/bin/bash -echo "Generating root gRPC server protos" +set -e -PROTOS="rpc.proto walletunlocker.proto **/*.proto" +# generate compiles the *.pb.go stubs from the *.proto files. +function generate() { + echo "Generating root gRPC server protos" + + PROTOS="rpc.proto walletunlocker.proto **/*.proto" + + # For each of the sub-servers, we then generate their protos, but a restricted + # set as they don't yet require REST proxies, or swagger docs. + for file in $PROTOS; do + DIRECTORY=$(dirname "${file}") + echo "Generating protos from ${file}, into ${DIRECTORY}" + + # Generate the protos. + protoc -I/usr/local/include -I. \ + --go_out=plugins=grpc,paths=source_relative:. \ + "${file}" + + # Generate the REST reverse proxy. + protoc -I/usr/local/include -I. \ + --grpc-gateway_out=logtostderr=true,paths=source_relative,grpc_api_configuration=rest-annotations.yaml:. \ + "${file}" + + + # Finally, generate the swagger file which describes the REST API in detail. + protoc -I/usr/local/include -I. \ + --swagger_out=logtostderr=true,grpc_api_configuration=rest-annotations.yaml:. \ + "${file}" + done +} -# For each of the sub-servers, we then generate their protos, but a restricted -# set as they don't yet require REST proxies, or swagger docs. -for file in $PROTOS; do - DIRECTORY=$(dirname "${file}") - echo "Generating protos from ${file}, into ${DIRECTORY}" +# format formats the *.proto files with the clang-format utility. +function format() { + find . -name "*.proto" -print0 | xargs -0 clang-format --style=file -i +} - # Generate the protos. - protoc -I/usr/local/include -I. \ - --go_out=plugins=grpc,paths=source_relative:. \ - "${file}" +# Compile and format the lnrpc package. +pushd lnrpc +format +generate +popd - # Generate the REST reverse proxy. - protoc -I/usr/local/include -I. \ - --grpc-gateway_out=logtostderr=true,paths=source_relative,grpc_api_configuration=rest-annotations.yaml:. \ - "${file}" - - - # Finally, generate the swagger file which describes the REST API in detail. - protoc -I/usr/local/include -I. \ - --swagger_out=logtostderr=true,grpc_api_configuration=rest-annotations.yaml:. \ - "${file}" -done +if [[ "$COMPILE_MOBILE" == "1" ]]; then + pushd mobile + ./gen_bindings.sh $FALAFEL_VERSION + popd +fi diff --git a/lnrpc/gen_protos_docker.sh b/lnrpc/gen_protos_docker.sh new file mode 100755 index 00000000..deebc4fb --- /dev/null +++ b/lnrpc/gen_protos_docker.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -e + +# Directory of the script file, independent of where it's called from. +DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)" + +PROTOC_GEN_VERSION=$(go list -f '{{.Version}}' -m github.com/golang/protobuf) +GRPC_GATEWAY_VERSION=$(go list -f '{{.Version}}' -m github.com/grpc-ecosystem/grpc-gateway) + +echo "Building protobuf compiler docker image..." +docker build -q -t lnd-protobuf-builder \ + --build-arg PROTOC_GEN_VERSION="$PROTOC_GEN_VERSION" \ + --build-arg GRPC_GATEWAY_VERSION="$GRPC_GATEWAY_VERSION" \ + . + +echo "Compiling and formatting *.proto files..." +docker run \ + --rm \ + --user "$UID:$(id -g)" \ + -e UID=$UID \ + -e COMPILE_MOBILE \ + -v "$DIR/../:/build" \ + lnd-protobuf-builder diff --git a/mobile/gen_bindings.sh b/mobile/gen_bindings.sh index c325ff3f..4b1a1637 100755 --- a/mobile/gen_bindings.sh +++ b/mobile/gen_bindings.sh @@ -47,7 +47,6 @@ for file in $PROTOS; do echo "Generating mobile protos from ${file}" protoc -I/usr/local/include -I. \ - -I$GOPATH/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \ --plugin=protoc-gen-custom=$falafel\ --custom_out=./build \ --custom_opt="$opts" \ @@ -77,7 +76,6 @@ do echo "Generating mobile protos from ${file}, with build tag ${tag}" protoc -I/usr/local/include -I. \ - -I$GOPATH/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \ -I../lnrpc \ --plugin=protoc-gen-custom=$falafel \ --custom_out=./build \ diff --git a/scripts/install_travis_proto.sh b/scripts/install_travis_proto.sh deleted file mode 100755 index 38963f7b..00000000 --- a/scripts/install_travis_proto.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env bash - -# Abort on error (-e) and print commands (-v). -set -ev - -# See README.md in lnrpc why we need these specific versions/commits. -PROTOC_VERSION=3.4.0 -PROTOBUF_VERSION="v1.3.2" -GENPROTO_VERSION="20e1ac93f88cf06d2b1defb90b9e9e126c7dfff6" -GRPC_GATEWAY_VERSION="v1.14.3" - -# This script is specific to Travis CI so we only need to support linux x64. -PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip" -PROTOC_DL_CACHE_DIR="${DOWNLOAD_CACHE:-/tmp/download_cache}/protoc" - -# install_protoc copies the cached protoc binary to the $PATH or downloads it -# if no cached version is found. -install_protoc() { - if [ -f "${PROTOC_DL_CACHE_DIR}/bin/protoc" ]; then - echo "Using cached version of protoc" - else - wget -O /tmp/protoc.zip $PROTOC_URL - mkdir -p "${PROTOC_DL_CACHE_DIR}" - unzip -o /tmp/protoc.zip -d "${PROTOC_DL_CACHE_DIR}" - chmod -R a+rx "${PROTOC_DL_CACHE_DIR}/" - fi - sudo cp "${PROTOC_DL_CACHE_DIR}/bin/protoc" /usr/local/bin - sudo cp -r "${PROTOC_DL_CACHE_DIR}/include" /usr/local -} - -# install_protobuf downloads and compiles the Golang protobuf library that -# encodes/decodes all protobuf messages from/to Go structs. -install_protobuf() { - local install_path="$GOPATH/src/github.com/golang/protobuf" - if [ ! -d "$install_path" ]; then - git clone https://github.com/golang/protobuf "$install_path" - fi - pushd "$install_path" - git reset --hard master && git checkout master && git pull - git reset --hard $PROTOBUF_VERSION - make - popd -} - -# install_genproto downloads the Golang protobuf generator that converts the -# .proto files into Go interface stubs. -install_genproto() { - local install_path="$GOPATH/src/google.golang.org/genproto" - if [ ! -d "$install_path" ]; then - git clone https://github.com/google/go-genproto "$install_path" - fi - pushd "$install_path" - git reset --hard master && git checkout master && git pull - git reset --hard $GENPROTO_VERSION - popd -} - -# install_grpc_gateway downloads and installs the gRPC gateway that converts -# .proto files into REST gateway code. -install_grpc_gateway() { - local install_path="$GOPATH/src/github.com/grpc-ecosystem/grpc-gateway" - if [ ! -d "$install_path" ]; then - git clone https://github.com/grpc-ecosystem/grpc-gateway "$install_path" - fi - pushd "$install_path" - git reset --hard master && git checkout master && git pull - git reset --hard $GRPC_GATEWAY_VERSION - GO111MODULE=on go install ./protoc-gen-grpc-gateway ./protoc-gen-swagger - popd -} - -install_protoc -install_protobuf -install_genproto -install_grpc_gateway