akovalenko/lnd.xprv
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #4911 from guggero/docker-rpc-compile

Browse Source 
lnrpc+mobile: use docker to compile/format protos
This commit is contained in:
Johan T. Halseth 2021-02-01 13:42:42 +01:00 committed by GitHub
commit 315f97ffd5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 123 additions and 233 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
.github/workflows/main.yml vendored
View File

@ -16,7 +16,6 @@ env:
# go needs absolute directories, using the $HOME variable doesn't work here.
GOCACHE: /home/runner/work/go/pkg/build
GOPATH: /home/runner/work/go
DOWNLOAD_CACHE: /home/runner/work/download_cache
BITCOIN_VERSION: 0.20.1
# If you change this value, please change it in the following files as well:
@ -29,10 +28,10 @@ env:
jobs:
########################
# RPC compilation check
# RPC and mobile compilation check
########################
rpc-check:
name: RPC compilation check
name: RPC and mobile compilation check
runs-on: ubuntu-latest
steps:
- name: git checkout
@ -54,21 +53,15 @@ jobs:
with:
go-version: '~${{ env.GO_VERSION }}'
- name: download cache
uses: actions/cache@v1
with:
path: /home/runner/work/download_cache
key: lnd-${{ runner.os }}-download-${{ hashFiles('**/install_travis_proto.sh') }}
restore-keys: |
lnd-${{ runner.os }}-download-${{ hashFiles('**/install_travis_proto.sh') }}
lnd-${{ runner.os }}-download-
- name: install protoc and protobuf libraries
run: ./scripts/install_travis_proto.sh
- name: run check
run: make rpc-check
- name: build mobile RPC bindings
run: make mobile-rpc
- name: build mobile specific code
run: go build --tags="mobile" ./mobile
########################
# check commits
########################
@ -169,50 +162,6 @@ jobs:
- name: build release for all architectures
run: make release
########################
# mobile compilation
########################
mobile-compile:
name: mobile compilation
runs-on: ubuntu-latest
steps:
- name: git checkout
uses: actions/checkout@v2
- name: go cache
uses: actions/cache@v1
with:
path: /home/runner/work/go
key: lnd-${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ github.job }}-${{ hashFiles('**/go.sum') }}
restore-keys: |
lnd-${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ github.job }}-${{ hashFiles('**/go.sum') }}
lnd-${{ runner.os }}-go-${{ env.GO_VERSION }}-${{ github.job }}-
lnd-${{ runner.os }}-go-${{ env.GO_VERSION }}-
lnd-${{ runner.os }}-go-
- name: setup go ${{ env.GO_VERSION }}
uses: actions/setup-go@v2
with:
go-version: '~${{ env.GO_VERSION }}'
- name: download cache
uses: actions/cache@v1
with:
path: /home/runner/work/download_cache
key: lnd-${{ runner.os }}-download-${{ hashFiles('**/install_travis_proto.sh') }}
restore-keys: |
lnd-${{ runner.os }}-download-${{ hashFiles('**/install_travis_proto.sh') }}
lnd-${{ runner.os }}-download-
- name: install protoc and protobuf libraries
run: ./scripts/install_travis_proto.sh
- name: build mobile RPC bindings
run: make mobile-rpc
- name: build mobile specific code
run: go build --tags="mobile" ./mobile
########################
# sample configuration check
########################
@ -239,15 +188,6 @@ jobs:
with:
go-version: '~${{ env.GO_VERSION }}'
- name: download cache
uses: actions/cache@v1
with:
path: /home/runner/work/download_cache
key: lnd-${{ runner.os }}-download-${{ hashFiles('**/install_travis_proto.sh') }}
restore-keys: |
lnd-${{ runner.os }}-download-${{ hashFiles('**/install_travis_proto.sh') }}
lnd-${{ runner.os }}-download-
- name: check all command line flags exist in sample-lnd.conf file
run: make sample-conf-check

7
.travis.yml
View File

@ -1,7 +1,6 @@
language: go
cache:
directories:
- $DOWNLOAD_CACHE
- $GOCACHE
- $GOPATH/pkg/mod
- $GOPATH/src/github.com/btcsuite
@ -27,7 +26,6 @@ go:
env:
global:
- GOCACHE=$HOME/.go-build
- DOWNLOAD_CACHE=$HOME/download_cache
- BITCOIN_VERSION=0.20.1
sudo: required
@ -36,11 +34,6 @@ jobs:
include:
- stage: Sanity Check
name: Lint and compile
before_script:
# Install the RPC tools as a before step so Travis collapses the output
# after it's done.
- ./scripts/install_travis_proto.sh
script:
# Step 1: Make sure no diff is produced when compiling with the correct
# version.

13
Makefile
View File

@ -6,7 +6,6 @@ BTCD_PKG := github.com/btcsuite/btcd
GOVERALLS_PKG := github.com/mattn/goveralls
LINT_PKG := github.com/golangci/golangci-lint/cmd/golangci-lint
GOACC_PKG := github.com/ory/go-acc
FALAFEL_PKG := github.com/lightninglabs/falafel
GOIMPORTS_PKG := golang.org/x/tools/cmd/goimports
GOFUZZ_BUILD_PKG := github.com/dvyukov/go-fuzz/go-fuzz-build
GOFUZZ_PKG := github.com/dvyukov/go-fuzz/go-fuzz
@ -37,7 +36,6 @@ BTCD_COMMIT := $(shell cat go.mod | \
LINT_COMMIT := v1.18.0
GOACC_COMMIT := ddc355013f90fea78d83d3a6c71f1d37ac07ecd5
FALAFEL_COMMIT := v0.7.1
GOFUZZ_COMMIT := 21309f307f61
DEPGET := cd /tmp && GO111MODULE=on go get -v
@ -113,10 +111,6 @@ btcd:
@$(call print, "Installing btcd.")
$(DEPGET) $(BTCD_PKG)@$(BTCD_COMMIT)
falafel:
@$(call print, "Installing falafel.")
$(DEPGET) $(FALAFEL_PKG)@$(FALAFEL_COMMIT)
goimports:
@$(call print, "Installing goimports.")
$(DEPGET) $(GOIMPORTS_PKG)
@ -264,7 +258,7 @@ list:
rpc:
@$(call print, "Compiling protos.")
cd ./lnrpc; ./gen_protos.sh
cd ./lnrpc; ./gen_protos_docker.sh
rpc-format:
@$(call print, "Formatting protos.")
@ -279,9 +273,9 @@ sample-conf-check:
@$(call print, "Making sure every flag has an example in the sample-lnd.conf file")
for flag in $$(GO_FLAGS_COMPLETION=1 go run -tags="$(RELEASE_TAGS)" $(PKG)/cmd/lnd -- | grep -v help | cut -c3-); do if ! grep -q $$flag sample-lnd.conf; then echo "Command line flag --$$flag not added to sample-lnd.conf"; exit 1; fi; done
mobile-rpc: falafel goimports
mobile-rpc:
@$(call print, "Creating mobile RPC from protos.")
cd ./mobile; ./gen_bindings.sh $(FALAFEL_COMMIT)
cd ./lnrpc; COMPILE_MOBILE=1 ./gen_protos_docker.sh
vendor:
@$(call print, "Re-creating vendor directory.")
@ -318,7 +312,6 @@ clean:
unit \
unit-cover \
unit-race \
falafel \
goveralls \
travis-race \
travis-cover \

31
lnrpc/Dockerfile Normal file
View File

@ -0,0 +1,31 @@
FROM golang:1.15.6-buster
RUN apt-get update && apt-get install -y \
git \
protobuf-compiler='3.6.1*' \
clang-format='1:7.0*'
# We don't want any default values for these variables to make sure they're
# explicitly provided by parsing the go.mod file. Otherwise we might forget to
# update them here if we bump the versions.
ARG PROTOC_GEN_VERSION
ARG GRPC_GATEWAY_VERSION
ENV FALAFEL_VERSION="v0.7.1"
ENV GOCACHE=/tmp/build/.cache
ENV GOMODCACHE=/tmp/build/.modcache
RUN cd /tmp \
&& mkdir -p /tmp/build/.cache \
&& mkdir -p /tmp/build/.modcache \
&& export GO111MODULE=on \
&& go get github.com/golang/protobuf/protoc-gen-go@${PROTOC_GEN_VERSION} \
&& go get github.com/grpc-ecosystem/grpc-gateway/protoc-gen-grpc-gateway@${GRPC_GATEWAY_VERSION} \
&& go get github.com/grpc-ecosystem/grpc-gateway/protoc-gen-swagger@${GRPC_GATEWAY_VERSION} \
&& go get github.com/lightninglabs/falafel@${FALAFEL_VERSION} \
&& go get golang.org/x/tools/cmd/goimports \
&& chmod -R 777 /tmp/build/
WORKDIR /build
CMD ["/bin/bash", "/build/lnrpc/gen_protos.sh"]

60
lnrpc/README.md
View File

@ -153,58 +153,21 @@ description):
## Generate protobuf definitions
### Linux
To compile the `lnrpc/**/*.proto` files and generate the protobuf definitions,
you need to have [Docker](https://docs.docker.com/get-docker/) and `make`
installed.
For linux there is an easy install script that is also used for the Travis CI
build. Just run the following command (requires `sudo` permissions and the tools
`make`, `go`, `wget` and `unzip` to be installed) from the repository's root
folder:
```shell
⛰ ./scripts/install_travis_proto.sh
```
### MacOS / Unix like systems
1. Download [v.3.4.0](https://github.com/google/protobuf/releases/tag/v3.4.0) of
`protoc` for your operating system and add it to your `PATH`.
For example, if using macOS:
```shell
⛰ curl -LO https://github.com/google/protobuf/releases/download/v3.4.0/protoc-3.4.0-osx-x86_64.zip
⛰ unzip protoc-3.4.0-osx-x86_64.zip -d protoc
⛰ export PATH=$PWD/protoc/bin:$PATH
```
2. Install `golang/protobuf` at version `v1.3.2`.
```shell
⛰ git clone https://github.com/golang/protobuf $GOPATH/src/github.com/golang/protobuf
⛰ cd $GOPATH/src/github.com/golang/protobuf
⛰ git reset --hard v1.3.2
⛰ make
```
3. Install 'genproto' at commit `20e1ac93f88cf06d2b1defb90b9e9e126c7dfff6`.
```shell
⛰ go get google.golang.org/genproto
⛰ cd $GOPATH/src/google.golang.org/genproto
⛰ git reset --hard 20e1ac93f88cf06d2b1defb90b9e9e126c7dfff6
```
4. Install `grpc-ecosystem/grpc-gateway` at version `v1.14.3`.
```shell
⛰ git clone https://github.com/grpc-ecosystem/grpc-gateway $GOPATH/src/github.com/grpc-ecosystem/grpc-gateway
⛰ cd $GOPATH/src/github.com/grpc-ecosystem/grpc-gateway
⛰ git reset --hard v1.14.3
⛰ go install ./protoc-gen-grpc-gateway ./protoc-gen-swagger
```
5. Run [`gen_protos.sh`](https://github.com/lightningnetwork/lnd/blob/master/lnrpc/gen_protos.sh)
or `make rpc` to generate new protobuf definitions.
Simply run `make rpc` to start the compilation process.
## Format .proto files
We use `clang-format` to make sure the `.proto` files are formatted correctly.
You can install the formatter on Ubuntu by running `apt install clang-format` or on Mac by running `brew install clang-format`.
When running the `make rpc` command, the `.proto` files are also formatted. To
format the files without also compiling them, you can install the `clang-format`
formatter on Ubuntu by running `apt install clang-format` or on Mac by running
`brew install clang-format`.
The `make format` command should then produce the correct result.
Consult [this page](http://releases.llvm.org/download.html) to find binaries
for other operating systems or distributions.
@ -213,7 +176,8 @@ for other operating systems or distributions.
The following commands are available with `make`:
* `rpc`: Compile `.proto` files (calls `lnrpc/gen_protos.sh`).
* `rpc`: Compile and format all `.proto` files using Docker (calls
`lnrpc/gen_protos_docker.sh`).
* `rpc-format`: Formats all `.proto` files according to our formatting rules.
Requires `clang-format`, see previous chapter.
* `rpc-check`: Runs both previous commands and makes sure the git work tree is

64
lnrpc/gen_protos.sh
View File

@ -1,28 +1,50 @@
#!/bin/sh
#!/bin/bash
echo "Generating root gRPC server protos"
set -e
PROTOS="rpc.proto walletunlocker.proto **/*.proto"
# generate compiles the *.pb.go stubs from the *.proto files.
function generate() {
echo "Generating root gRPC server protos"
# For each of the sub-servers, we then generate their protos, but a restricted
# set as they don't yet require REST proxies, or swagger docs.
for file in $PROTOS; do
DIRECTORY=$(dirname "${file}")
echo "Generating protos from ${file}, into ${DIRECTORY}"
PROTOS="rpc.proto walletunlocker.proto **/*.proto"
# Generate the protos.
protoc -I/usr/local/include -I. \
--go_out=plugins=grpc,paths=source_relative:. \
"${file}"
# For each of the sub-servers, we then generate their protos, but a restricted
# set as they don't yet require REST proxies, or swagger docs.
for file in $PROTOS; do
DIRECTORY=$(dirname "${file}")
echo "Generating protos from ${file}, into ${DIRECTORY}"
# Generate the REST reverse proxy.
protoc -I/usr/local/include -I. \
--grpc-gateway_out=logtostderr=true,paths=source_relative,grpc_api_configuration=rest-annotations.yaml:. \
"${file}"
# Generate the protos.
protoc -I/usr/local/include -I. \
--go_out=plugins=grpc,paths=source_relative:. \
"${file}"
# Generate the REST reverse proxy.
protoc -I/usr/local/include -I. \
--grpc-gateway_out=logtostderr=true,paths=source_relative,grpc_api_configuration=rest-annotations.yaml:. \
"${file}"
# Finally, generate the swagger file which describes the REST API in detail.
protoc -I/usr/local/include -I. \
--swagger_out=logtostderr=true,grpc_api_configuration=rest-annotations.yaml:. \
"${file}"
done
# Finally, generate the swagger file which describes the REST API in detail.
protoc -I/usr/local/include -I. \
--swagger_out=logtostderr=true,grpc_api_configuration=rest-annotations.yaml:. \
"${file}"
done
}
# format formats the *.proto files with the clang-format utility.
function format() {
find . -name "*.proto" -print0 | xargs -0 clang-format --style=file -i
}
# Compile and format the lnrpc package.
pushd lnrpc
format
generate
popd
if [[ "$COMPILE_MOBILE" == "1" ]]; then
pushd mobile
./gen_bindings.sh $FALAFEL_VERSION
popd
fi

24
lnrpc/gen_protos_docker.sh Executable file
View File

@ -0,0 +1,24 @@
#!/bin/bash
set -e
# Directory of the script file, independent of where it's called from.
DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"
PROTOC_GEN_VERSION=$(go list -f '{{.Version}}' -m github.com/golang/protobuf)
GRPC_GATEWAY_VERSION=$(go list -f '{{.Version}}' -m github.com/grpc-ecosystem/grpc-gateway)
echo "Building protobuf compiler docker image..."
docker build -q -t lnd-protobuf-builder \
--build-arg PROTOC_GEN_VERSION="$PROTOC_GEN_VERSION" \
--build-arg GRPC_GATEWAY_VERSION="$GRPC_GATEWAY_VERSION" \
.
echo "Compiling and formatting *.proto files..."
docker run \
--rm \
--user "$UID:$(id -g)" \
-e UID=$UID \
-e COMPILE_MOBILE \
-v "$DIR/../:/build" \
lnd-protobuf-builder

2
mobile/gen_bindings.sh
View File

@ -47,7 +47,6 @@ for file in $PROTOS; do
echo "Generating mobile protos from ${file}"
protoc -I/usr/local/include -I. \
-I$GOPATH/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \
--plugin=protoc-gen-custom=$falafel\
--custom_out=./build \
--custom_opt="$opts" \
@ -77,7 +76,6 @@ do
echo "Generating mobile protos from ${file}, with build tag ${tag}"
protoc -I/usr/local/include -I. \
-I$GOPATH/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \
-I../lnrpc \
--plugin=protoc-gen-custom=$falafel \
--custom_out=./build \

75
scripts/install_travis_proto.sh
View File

@ -1,75 +0,0 @@
#!/usr/bin/env bash
# Abort on error (-e) and print commands (-v).
set -ev
# See README.md in lnrpc why we need these specific versions/commits.
PROTOC_VERSION=3.4.0
PROTOBUF_VERSION="v1.3.2"
GENPROTO_VERSION="20e1ac93f88cf06d2b1defb90b9e9e126c7dfff6"
GRPC_GATEWAY_VERSION="v1.14.3"
# This script is specific to Travis CI so we only need to support linux x64.
PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip"
PROTOC_DL_CACHE_DIR="${DOWNLOAD_CACHE:-/tmp/download_cache}/protoc"
# install_protoc copies the cached protoc binary to the $PATH or downloads it
# if no cached version is found.
install_protoc() {
if [ -f "${PROTOC_DL_CACHE_DIR}/bin/protoc" ]; then
echo "Using cached version of protoc"
else
wget -O /tmp/protoc.zip $PROTOC_URL
mkdir -p "${PROTOC_DL_CACHE_DIR}"
unzip -o /tmp/protoc.zip -d "${PROTOC_DL_CACHE_DIR}"
chmod -R a+rx "${PROTOC_DL_CACHE_DIR}/"
fi
sudo cp "${PROTOC_DL_CACHE_DIR}/bin/protoc" /usr/local/bin
sudo cp -r "${PROTOC_DL_CACHE_DIR}/include" /usr/local
}
# install_protobuf downloads and compiles the Golang protobuf library that
# encodes/decodes all protobuf messages from/to Go structs.
install_protobuf() {
local install_path="$GOPATH/src/github.com/golang/protobuf"
if [ ! -d "$install_path" ]; then
git clone https://github.com/golang/protobuf "$install_path"
fi
pushd "$install_path"
git reset --hard master && git checkout master && git pull
git reset --hard $PROTOBUF_VERSION
make
popd
}
# install_genproto downloads the Golang protobuf generator that converts the
# .proto files into Go interface stubs.
install_genproto() {
local install_path="$GOPATH/src/google.golang.org/genproto"
if [ ! -d "$install_path" ]; then
git clone https://github.com/google/go-genproto "$install_path"
fi
pushd "$install_path"
git reset --hard master && git checkout master && git pull
git reset --hard $GENPROTO_VERSION
popd
}
# install_grpc_gateway downloads and installs the gRPC gateway that converts
# .proto files into REST gateway code.
install_grpc_gateway() {
local install_path="$GOPATH/src/github.com/grpc-ecosystem/grpc-gateway"
if [ ! -d "$install_path" ]; then
git clone https://github.com/grpc-ecosystem/grpc-gateway "$install_path"
fi
pushd "$install_path"
git reset --hard master && git checkout master && git pull
git reset --hard $GRPC_GATEWAY_VERSION
GO111MODULE=on go install ./protoc-gen-grpc-gateway ./protoc-gen-swagger
popd
}
install_protoc
install_protobuf
install_genproto
install_grpc_gateway