Merge pull request #3282 from halseth/mobile-rpcs

[mobile] Mobile RPCs

.gitignore

@@ -43,6 +43,10 @@ cmd/cmd
 
 cmd/lncli/lncli
 
+# Files from mobile build.
+mobile/build
+mobile/*_generated.go
+
 # vim
 *.swp
 

Makefile

@@ -1,5 +1,6 @@
 PKG := github.com/lightningnetwork/lnd
 ESCPKG := github.com\/lightningnetwork\/lnd
+MOBILE_PKG := $(PKG)/mobile
 
 BTCD_PKG := github.com/btcsuite/btcd
 GOVERALLS_PKG := github.com/mattn/goveralls
@@ -8,11 +9,17 @@ GOACC_PKG := github.com/ory/go-acc
 
 GO_BIN := ${GOPATH}/bin
 BTCD_BIN := $(GO_BIN)/btcd
+GOMOBILE_BIN := GO111MODULE=off $(GO_BIN)/gomobile
 GOVERALLS_BIN := $(GO_BIN)/goveralls
 LINT_BIN := $(GO_BIN)/golangci-lint
 GOACC_BIN := $(GO_BIN)/go-acc
 
 BTCD_DIR :=${GOPATH}/src/$(BTCD_PKG)
+MOBILE_BUILD_DIR :=${GOPATH}/src/$(MOBILE_PKG)/build
+IOS_BUILD_DIR := $(MOBILE_BUILD_DIR)/ios
+IOS_BUILD := $(IOS_BUILD_DIR)/Lndmobile.framework
+ANDROID_BUILD_DIR := $(MOBILE_BUILD_DIR)/android
+ANDROID_BUILD := $(ANDROID_BUILD_DIR)/Lndmobile.aar
 
 COMMIT := $(shell git describe --abbrev=40 --dirty)
 LDFLAGS := -ldflags "-X $(PKG)/build.Commit=$(COMMIT)"
@@ -170,6 +177,26 @@ rpc:
 	@$(call print, "Compiling protos.")
 	cd ./lnrpc; ./gen_protos.sh
 
+mobile-rpc:
+	@$(call print, "Creating mobile RPC from protos.")
+	cd ./mobile; ./gen_bindings.sh
+
+vendor:
+	@$(call print, "Re-creating vendor directory.")
+	rm -r vendor/; GO111MODULE=on go mod vendor
+
+ios: vendor mobile-rpc
+	@$(call print, "Building iOS framework ($(IOS_BUILD)).")
+	mkdir -p $(IOS_BUILD_DIR)
+	$(GOMOBILE_BIN) bind -target=ios -tags="ios $(DEV_TAGS) autopilotrpc experimental" $(LDFLAGS) -v -o $(IOS_BUILD) $(MOBILE_PKG)
+
+android: vendor mobile-rpc
+	@$(call print, "Building Android library ($(ANDROID_BUILD)).")
+	mkdir -p $(ANDROID_BUILD_DIR)
+	$(GOMOBILE_BIN) bind -target=android -tags="android $(DEV_TAGS) autopilotrpc experimental" $(LDFLAGS) -v -o $(ANDROID_BUILD) $(MOBILE_PKG)
+
+mobile: ios android
+
 clean:
 	@$(call print, "Cleaning source.$(NC)")
 	$(RM) ./lnd-debug ./lncli-debug
@@ -199,4 +226,9 @@ clean:
 	lint \
 	list \
 	rpc \
+	mobile-rpc \
+	vendor \
+	ios \
+	android \
+	mobile \
 	clean

cmd/lnd/main.go

@@ -11,7 +11,7 @@ import (
 func main() {
 	// Call the "real" main in a nested manner so the defers will properly
 	// be executed in the case of a graceful shutdown.
-	if err := lnd.Main(); err != nil {
+	if err := lnd.Main(lnd.ListenerCfg{}); err != nil {
 		if e, ok := err.(*flags.Error); ok && e.Type == flags.ErrHelp {
 		} else {
 			fmt.Fprintln(os.Stderr, err)

lnd.go

@@ -93,10 +93,29 @@ var (
 	}
 )
 
+// ListenerCfg is a wrapper around custom listeners that can be passed to lnd
+// when calling its main method.
+type ListenerCfg struct {
+	// WalletUnlocker can be set to the listener to use for the wallet
+	// unlocker. If nil a regular network listener will be created.
+	WalletUnlocker net.Listener
+
+	// RPCListener can be set to the listener to use for the RPC server. If
+	// nil a regular network listener will be created.
+	RPCListener net.Listener
+}
+
+// rpcListeners is a function type used for closures that fetches a set of RPC
+// listeners for the current configuration, and the GRPC server options to use
+// with these listeners. If no custom listeners are present, this should return
+// normal listeners from the RPC endpoints defined in the config, and server
+// options specifying TLS.
+type rpcListeners func() ([]net.Listener, func(), []grpc.ServerOption, error)
+
 // Main is the true entry point for lnd. This function is required since defers
 // created in the top-level scope of a main method aren't executed if os.Exit()
 // is called.
-func Main() error {
+func Main(lisCfg ListenerCfg) error {
 	// Load the configuration, and parse any command line options. This
 	// function will also set up logging properly.
 	loadedConfig, err := loadConfig()
@@ -240,13 +259,60 @@ func Main() error {
 	// this information.
 	walletInitParams.Birthday = time.Now()
 
+	// getListeners is a closure that creates listeners from the
+	// RPCListeners defined in the config. It also returns a cleanup
+	// closure and the server options to use for the GRPC server.
+	getListeners := func() ([]net.Listener, func(), []grpc.ServerOption,
+		error) {
+
+		var grpcListeners []net.Listener
+		for _, grpcEndpoint := range cfg.RPCListeners {
+			// Start a gRPC server listening for HTTP/2
+			// connections.
+			lis, err := lncfg.ListenOnAddress(grpcEndpoint)
+			if err != nil {
+				ltndLog.Errorf("unable to listen on %s",
+					grpcEndpoint)
+				return nil, nil, nil, err
+			}
+			grpcListeners = append(grpcListeners, lis)
+		}
+
+		cleanup := func() {
+			for _, lis := range grpcListeners {
+				lis.Close()
+			}
+		}
+		return grpcListeners, cleanup, serverOpts, nil
+	}
+
+	// walletUnlockerListeners is a closure we'll hand to the wallet
+	// unlocker, that will be called when it needs listeners for its GPRC
+	// server.
+	walletUnlockerListeners := func() ([]net.Listener, func(),
+		[]grpc.ServerOption, error) {
+
+		// If we have chosen to start with a dedicated listener for the
+		// wallet unlocker, we return it directly, and empty server
+		// options to deactivate TLS.
+		// TODO(halseth): any point in adding TLS support for custom
+		// listeners?
+		if lisCfg.WalletUnlocker != nil {
+			return []net.Listener{lisCfg.WalletUnlocker}, func() {},
+				[]grpc.ServerOption{}, nil
+		}
+
+		// Otherwise we'll return the regular listeners.
+		return getListeners()
+	}
+
 	// We wait until the user provides a password over RPC. In case lnd is
 	// started with the --noseedbackup flag, we use the default password
 	// for wallet encryption.
 	if !cfg.NoSeedBackup {
 		params, err := waitForWalletPassword(
-			cfg.RPCListeners, cfg.RESTListeners, serverOpts,
+			cfg.RESTListeners, restDialOpts, restProxyDest, tlsCfg,
-			restDialOpts, restProxyDest, tlsCfg,
+			walletUnlockerListeners,
 		)
 		if err != nil {
 			err := fmt.Errorf("Unable to set up wallet password "+
@@ -457,12 +523,31 @@ func Main() error {
 	}
 	defer atplManager.Stop()
 
+	// rpcListeners is a closure we'll hand to the rpc server, that will be
+	// called when it needs listeners for its GPRC server.
+	rpcListeners := func() ([]net.Listener, func(), []grpc.ServerOption,
+		error) {
+
+		// If we have chosen to start with a dedicated listener for the
+		// rpc server, we return it directly, and empty server options
+		// to deactivate TLS.
+		// TODO(halseth): any point in adding TLS support for custom
+		// listeners?
+		if lisCfg.RPCListener != nil {
+			return []net.Listener{lisCfg.RPCListener}, func() {},
+				[]grpc.ServerOption{}, nil
+		}
+
+		// Otherwise we'll return the regular listeners.
+		return getListeners()
+	}
+
 	// Initialize, and register our implementation of the gRPC interface
 	// exported by the rpcServer.
 	rpcServer, err := newRPCServer(
-		server, macaroonService, cfg.SubRPCServers, serverOpts,
+		server, macaroonService, cfg.SubRPCServers, restDialOpts,
-		restDialOpts, restProxyDest, atplManager, server.invoices,
+		restProxyDest, atplManager, server.invoices, tower, tlsCfg,
-		tower, tlsCfg,
+		rpcListeners,
 	)
 	if err != nil {
 		err := fmt.Errorf("Unable to create RPC server: %v", err)
@@ -880,9 +965,18 @@ type WalletUnlockParams struct {
 // waitForWalletPassword will spin up gRPC and REST endpoints for the
 // WalletUnlocker server, and block until a password is provided by
 // the user to this RPC server.
-func waitForWalletPassword(grpcEndpoints, restEndpoints []net.Addr,
+func waitForWalletPassword(restEndpoints []net.Addr,
-	serverOpts []grpc.ServerOption, restDialOpts []grpc.DialOption,
+	restDialOpts []grpc.DialOption, restProxyDest string,
-	restProxyDest string, tlsConf *tls.Config) (*WalletUnlockParams, error) {
+	tlsConf *tls.Config, getListeners rpcListeners) (
+	*WalletUnlockParams, error) {
+
+	// Start a gRPC server listening for HTTP/2 connections, solely used
+	// for getting the encryption password from the client.
+	listeners, cleanup, serverOpts, err := getListeners()
+	if err != nil {
+		return nil, err
+	}
+	defer cleanup()
 
 	// Set up a new PasswordService, which will listen for passwords
 	// provided over RPC.
@@ -911,28 +1005,14 @@ func waitForWalletPassword(grpcEndpoints, restEndpoints []net.Addr,
 	// password is the last thing to be printed to the console.
 	var wg sync.WaitGroup
 
-	for _, grpcEndpoint := range grpcEndpoints {
+	for _, lis := range listeners {
-		// Start a gRPC server listening for HTTP/2 connections, solely
-		// used for getting the encryption password from the client.
-		lis, err := lncfg.ListenOnAddress(grpcEndpoint)
-		if err != nil {
-			ltndLog.Errorf(
-				"password RPC server unable to listen on %s",
-				grpcEndpoint,
-			)
-			return nil, err
-		}
-		defer lis.Close()
-
 		wg.Add(1)
-		go func() {
+		go func(lis net.Listener) {
-			rpcsLog.Infof(
+			rpcsLog.Infof("password RPC server listening on %s",
-				"password RPC server listening on %s",
+				lis.Addr())
-				lis.Addr(),
-			)
 			wg.Done()
 			grpcServer.Serve(lis)
-		}()
+		}(lis)
 	}
 
 	// Start a REST proxy for our gRPC server above.
@@ -942,7 +1022,7 @@ func waitForWalletPassword(grpcEndpoints, restEndpoints []net.Addr,
 
 	mux := proxy.NewServeMux()
 
-	err := lnrpc.RegisterWalletUnlockerHandlerFromEndpoint(
+	err = lnrpc.RegisterWalletUnlockerHandlerFromEndpoint(
 		ctx, mux, restProxyDest, restDialOpts,
 	)
 	if err != nil {

mobile/README.md

@@ -0,0 +1,55 @@
+## Building mobile libraries
+
+### Prerequisites
+#### protoc
+Install the dependencies for genarating protobuf definitions as stated in [lnrpc docs](
+../lnrpc/README.md#generate-protobuf-definitions)
+
+#### gomobile
+Follow [gomobile](https://github.com/golang/go/wiki/Mobile) in order to intall `gomobile` and dependencies.
+
+Remember to run `gomobile init` (otherwise the `lnd` build might just hang).
+
+Note that `gomobile` only supports building projects from `GOPATH` at this point.
+
+#### falafel
+Install [`falafel`](https://github.com/halseth/falafel):
+```
+go get -u -v github.com/halseth/falafel
+```
+
+### Building `lnd` for iOS
+```
+make ios
+```
+
+### Building `lnd` for Android
+```
+make android
+```
+
+`make mobile` will build both iOS and Android libs.
+
+### Libraries
+After the build has succeeded, the libraries will be found in `mobile/build/ios/Lndmobile.framework` and `mobile/build/android/Lndmobile.aar`. Reference your platforms' SDK documentation for how to add the library to your project.
+
+#### Generating proto definitions for your language.
+In order to call the methods in the generated library, the serialized proto for the given RPC call must be provided. Similarly, the response will be a serialized proto.
+
+In order to generate protobuf definitions for your language of choice, add the proto plugin to the `protoc` invocations found in [`gen_protos.sh`](../lnrpc/gen_protos.sh). For instance to generate protos for Swift, add `--swift_out=.` and run `make rpc`.
+
+### Options
+Similar to lnd, subservers can be conditionally compiled with the build by setting the tags argument:
+
+```
+make ios tags="routerrpc"
+```
+
+To support subservers that have APIs with name conflicts, pass the "prefix" flag. This will add the subserver name as a prefix to each method name:
+
+```
+make ios tags="routerrpc" prefix=1
+```
+
+### API docs
+TODO(halseth)

mobile/bindings.go

@@ -0,0 +1,61 @@
+// +build ios android
+
+package lndmobile
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	flags "github.com/jessevdk/go-flags"
+	"github.com/lightningnetwork/lnd"
+)
+
+// Start starts lnd in a new goroutine.
+//
+// extraArgs can be used to pass command line arguments to lnd that will
+// override what is found in the config file. Example:
+//	extraArgs = "--bitcoin.testnet --lnddir=\"/tmp/folder name/\" --profile=5050"
+func Start(extraArgs string, callback Callback) {
+	// Split the argument string on "--" to get separated command line
+	// arguments.
+	var splitArgs []string
+	for _, a := range strings.Split(extraArgs, "--") {
+		if a == "" {
+			continue
+		}
+		// Finally we prefix any non-empty string with --, and trim
+		// whitespace to mimic the regular command line arguments.
+		splitArgs = append(splitArgs, strings.TrimSpace("--"+a))
+	}
+
+	// Add the extra arguments to os.Args, as that will be parsed during
+	// startup.
+	os.Args = append(os.Args, splitArgs...)
+
+	// We call the main method with the custom in-memory listeners called
+	// by the mobile APIs, such that the grpc server will use these.
+	cfg := lnd.ListenerCfg{
+		WalletUnlocker: walletUnlockerLis,
+		RPCListener:    lightningLis,
+	}
+
+	// Call the "real" main in a nested manner so the defers will properly
+	// be executed in the case of a graceful shutdown.
+	go func() {
+		if err := lnd.Main(cfg); err != nil {
+			if e, ok := err.(*flags.Error); ok &&
+				e.Type == flags.ErrHelp {
+			} else {
+				fmt.Fprintln(os.Stderr, err)
+			}
+			os.Exit(1)
+		}
+	}()
+
+	// TODO(halseth): callback when RPC server is actually running. Since
+	// the RPC server might take a while to start up, the client might
+	// assume it is ready to accept calls when this callback is sent, while
+	// it's not.
+	callback.OnResponse([]byte("started"))
+}

mobile/gen_bindings.sh

@@ -0,0 +1,64 @@
+#!/bin/sh
+
+mkdir -p build
+
+# Check falafel version.
+falafelVersion="0.5"
+falafel=$(which falafel)
+if [ $falafel ]
+then
+        version=$($falafel -v)
+        if [ $version != $falafelVersion ]
+        then
+                echo "falafel version $falafelVersion required"
+                exit 1
+        fi
+        echo "Using plugin $falafel $version"
+else
+        echo "falafel not found"
+        exit 1
+fi
+
+pkg="lndmobile"
+target_pkg="github.com/lightningnetwork/lnd/lnrpc"
+
+# Generate APIs by passing the parsed protos to the falafel plugin.
+opts="package_name=$pkg,target_package=$target_pkg,listeners=lightning=lightningLis walletunlocker=walletUnlockerLis,mem_rpc=1"
+protoc -I/usr/local/include -I. \
+       -I$GOPATH/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \
+       --plugin=protoc-gen-custom=$falafel\
+       --custom_out=./build \
+       --custom_opt="$opts" \
+       --proto_path=../lnrpc \
+       rpc.proto
+
+# If prefix=1 is specified, prefix the generated methods with subserver name.
+# This must be enabled to support subservers with name conflicts.
+use_prefix="0"
+if [[ $prefix = "1" ]]
+then
+    echo "Prefixing methods with subserver name"
+    use_prefix="1"
+fi
+
+# Find all subservers.
+for file in ../lnrpc/**/*.proto
+do
+    DIRECTORY=$(dirname ${file})
+    tag=$(basename ${DIRECTORY})
+    build_tags="// +build $tag"
+    lis="lightningLis"
+
+    opts="package_name=$pkg,target_package=$target_pkg/$tag,build_tags=$build_tags,api_prefix=$use_prefix,defaultlistener=$lis"
+
+    echo "Generating mobile protos from ${file}, with build tag ${tag}"
+
+    protoc -I/usr/local/include -I. \
+           -I$GOPATH/src/github.com/grpc-ecosystem/grpc-gateway/third_party/googleapis \
+           -I../lnrpc \
+           --plugin=protoc-gen-custom=$falafel \
+           --custom_out=./build \
+           --custom_opt="$opts" \
+           --proto_path=${DIRECTORY} \
+           ${file}
+done

mobile/sample_lnd.conf

@@ -0,0 +1,13 @@
+[Application Options]
+debuglevel=info
+no-macaroons=1
+maxbackoff=2s
+nolisten=1
+
+[Routing]
+routing.assumechanvalid=1
+
+[Bitcoin]
+bitcoin.active=1
+bitcoin.testnet=1
+bitcoin.node=neutrino

rpcserver.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"math"
+	"net"
 	"net/http"
 	"sort"
 	"strings"
@@ -409,6 +410,11 @@ type rpcServer struct {
 	// requests from.
 	grpcServer *grpc.Server
 
+	// listeners is a list of listeners to use when starting the grpc
+	// server. We make it configurable such that the grpc server can listen
+	// on custom interfaces.
+	listeners []net.Listener
+
 	// listenerCleanUp are a set of closures functions that will allow this
 	// main RPC server to clean up all the listening socket created for the
 	// server.
@@ -442,10 +448,10 @@ var _ lnrpc.LightningServer = (*rpcServer)(nil)
 // base level options passed to the grPC server. This typically includes things
 // like requiring TLS, etc.
 func newRPCServer(s *server, macService *macaroons.Service,
-	subServerCgs *subRPCServerConfigs, serverOpts []grpc.ServerOption,
+	subServerCgs *subRPCServerConfigs, restDialOpts []grpc.DialOption,
-	restDialOpts []grpc.DialOption, restProxyDest string,
+	restProxyDest string, atpl *autopilot.Manager,
-	atpl *autopilot.Manager, invoiceRegistry *invoices.InvoiceRegistry,
+	invoiceRegistry *invoices.InvoiceRegistry, tower *watchtower.Standalone,
-	tower *watchtower.Standalone, tlsCfg *tls.Config) (*rpcServer, error) {
+	tlsCfg *tls.Config, getListeners rpcListeners) (*rpcServer, error) {
 
 	// Set up router rpc backend.
 	channelGraph := s.chanDB.ChannelGraph()
@@ -570,6 +576,12 @@ func newRPCServer(s *server, macService *macaroons.Service,
 		strmInterceptors, errorLogStreamServerInterceptor(rpcsLog),
 	)
 
+	// Get the listeners and server options to use for this rpc server.
+	listeners, cleanup, serverOpts, err := getListeners()
+	if err != nil {
+		return nil, err
+	}
+
 	// If any interceptors have been set up, add them to the server options.
 	if len(unaryInterceptors) != 0 && len(strmInterceptors) != 0 {
 		chainedUnary := grpc_middleware.WithUnaryServerChain(
@@ -585,14 +597,16 @@ func newRPCServer(s *server, macService *macaroons.Service,
 	// gRPC server, and register the main lnrpc server along side.
 	grpcServer := grpc.NewServer(serverOpts...)
 	rootRPCServer := &rpcServer{
-		restDialOpts:  restDialOpts,
+		restDialOpts:    restDialOpts,
-		restProxyDest: restProxyDest,
+		listeners:       listeners,
-		subServers:    subServers,
+		listenerCleanUp: []func(){cleanup},
-		tlsCfg:        tlsCfg,
+		restProxyDest:   restProxyDest,
-		grpcServer:    grpcServer,
+		subServers:      subServers,
-		server:        s,
+		tlsCfg:          tlsCfg,
-		routerBackend: routerBackend,
+		grpcServer:      grpcServer,
-		quit:          make(chan struct{}, 1),
+		server:          s,
+		routerBackend:   routerBackend,
+		quit:            make(chan struct{}, 1),
 	}
 	lnrpc.RegisterLightningServer(grpcServer, rootRPCServer)
 
@@ -632,23 +646,11 @@ func (r *rpcServer) Start() error {
 
 	// With all the sub-servers started, we'll spin up the listeners for
 	// the main RPC server itself.
-	for _, listener := range cfg.RPCListeners {
+	for _, lis := range r.listeners {
-		lis, err := lncfg.ListenOnAddress(listener)
+		go func(lis net.Listener) {
-		if err != nil {
-			ltndLog.Errorf(
-				"RPC server unable to listen on %s", listener,
-			)
-			return err
-		}
-
-		r.listenerCleanUp = append(r.listenerCleanUp, func() {
-			lis.Close()
-		})
-
-		go func() {
 			rpcsLog.Infof("RPC server listening on %s", lis.Addr())
 			r.grpcServer.Serve(lis)
-		}()
+		}(lis)
 	}
 
 	// If Prometheus monitoring is enabled, start the Prometheus exporter.