From 0bb449c8fe2713b34174897191c78005324bfd3d Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Mon, 4 May 2020 09:52:49 +0200 Subject: [PATCH 1/8] docs+python: fix headings, make numbered list --- docs/grpc/python.md | 64 ++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/docs/grpc/python.md b/docs/grpc/python.md index 14d87701..4f6bae04 100644 --- a/docs/grpc/python.md +++ b/docs/grpc/python.md @@ -3,46 +3,46 @@ This section enumerates what you need to do to write a client that communicates with `lnd` in Python. -### Setup and Installation +## Setup and Installation Lnd uses the gRPC protocol for communication with clients like lncli. gRPC is based on protocol buffers and as such, you will need to compile the lnd proto file in Python before you can use it to communicate with lnd. -* Create a virtual environment for your project -``` -$ virtualenv lnd -``` -* Activate the virtual environment -``` -$ source lnd/bin/activate -``` -* Install dependencies (googleapis-common-protos is required due to the use of +1. Create a virtual environment for your project + ``` + $ virtualenv lnd + ``` +2. Activate the virtual environment + ``` + $ source lnd/bin/activate + ``` +3. Install dependencies (googleapis-common-protos is required due to the use of google/api/annotations.proto) -``` -(lnd)$ pip install grpcio grpcio-tools googleapis-common-protos -``` -* Clone the google api's repository (required due to the use of + ``` + (lnd)$ pip install grpcio grpcio-tools googleapis-common-protos + ``` +4. Clone the google api's repository (required due to the use of google/api/annotations.proto) -``` -(lnd)$ git clone https://github.com/googleapis/googleapis.git -``` -* Copy the lnd rpc.proto file (you'll find this at + ``` + (lnd)$ git clone https://github.com/googleapis/googleapis.git + ``` +5. Copy the lnd rpc.proto file (you'll find this at [lnrpc/rpc.proto](https://github.com/lightningnetwork/lnd/blob/master/lnrpc/rpc.proto)) or just download it -``` -(lnd)$ curl -o rpc.proto -s https://raw.githubusercontent.com/lightningnetwork/lnd/master/lnrpc/rpc.proto -``` -* Compile the proto file -``` -(lnd)$ python -m grpc_tools.protoc --proto_path=googleapis:. --python_out=. --grpc_python_out=. rpc.proto -``` + ``` + (lnd)$ curl -o rpc.proto -s https://raw.githubusercontent.com/lightningnetwork/lnd/master/lnrpc/rpc.proto + ``` +6. Compile the proto file + ``` + (lnd)$ python -m grpc_tools.protoc --proto_path=googleapis:. --python_out=. --grpc_python_out=. rpc.proto + ``` After following these steps, two files `rpc_pb2.py` and `rpc_pb2_grpc.py` will be generated. These files will be imported in your project anytime you use Python gRPC. -#### Imports and Client +### Imports and Client Every time you use Python gRPC, you will have to import the generated rpc modules and set up a channel and stub to your connect to your `lnd` node: @@ -66,13 +66,13 @@ channel = grpc.secure_channel('localhost:10009', creds) stub = lnrpc.LightningStub(channel) ``` -### Examples +## Examples Let's walk through some examples of Python gRPC clients. These examples assume that you have at least two `lnd` nodes running, the RPC location of one of which is at the default `localhost:10009`, with an open channel between the two nodes. -#### Simple RPC +### Simple RPC ```python # Retrieve and display the wallet balance @@ -80,7 +80,7 @@ response = stub.WalletBalance(ln.WalletBalanceRequest()) print(response.total_balance) ``` -#### Response-streaming RPC +### Response-streaming RPC ```python request = ln.InvoiceSubscription() @@ -102,7 +102,7 @@ $ lncli sendpayment --pay_req= Your Python console should now display the details of the recently satisfied invoice. -#### Bidirectional-streaming RPC +### Bidirectional-streaming RPC ```python from time import sleep @@ -133,7 +133,7 @@ for payment in stub.SendPayment(request_iterable): ``` This example will send a payment of 100 satoshis every 2 seconds. -#### Using Macaroons +### Using Macaroons To authenticate using macaroons you need to include the macaroon in the metadata of the request. @@ -180,7 +180,7 @@ stub.GetInfo(ln.GetInfoRequest()) ``` -### Conclusion +## Conclusion With the above, you should have all the `lnd` related `gRPC` dependencies installed locally into your virtual environment. In order to get up to speed From d1629e9cf4f6bfd92b15bb1e7ebea7d582332a8e Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Mon, 4 May 2020 09:53:07 +0200 Subject: [PATCH 2/8] docs+python: add build instructions for subserver modules --- docs/grpc/python.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/docs/grpc/python.md b/docs/grpc/python.md index 4f6bae04..b1b09cd0 100644 --- a/docs/grpc/python.md +++ b/docs/grpc/python.md @@ -42,6 +42,21 @@ After following these steps, two files `rpc_pb2.py` and `rpc_pb2_grpc.py` will be generated. These files will be imported in your project anytime you use Python gRPC. +### Generating RPC modules for subservers + +If you want to use any of the subservers' functionality, you also need to +generate the python modules for them. + +For example, if you want to generate the RPC modules for the `Router` subserver +(located/defined in `routerrpc/router.proto`), you need to run the following two +extra steps (after completing all 6 step described above) to get the +`router_pb2.py` and `router_pb2_grpc.py`: + +``` +(lnd)$ curl -o router.proto -s https://raw.githubusercontent.com/lightningnetwork/lnd/master/lnrpc/routerrpc/router.proto +(lnd)$ python -m grpc_tools.protoc --proto_path=googleapis:. --python_out=. --grpc_python_out=. router.proto +``` + ### Imports and Client Every time you use Python gRPC, you will have to import the generated rpc modules From 26998e13de794ed6287f3feb243cbc6249ce5268 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Mon, 4 May 2020 09:54:50 +0200 Subject: [PATCH 3/8] docs+python: add link to API documentation --- docs/grpc/python.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/grpc/python.md b/docs/grpc/python.md index b1b09cd0..55452f28 100644 --- a/docs/grpc/python.md +++ b/docs/grpc/python.md @@ -204,3 +204,9 @@ Python](https://developers.google.com/protocol-buffers/docs/pythontutorial). Additionally, [this official gRPC resource](http://www.grpc.io/docs/tutorials/basic/python.html) provides more details around how to drive `gRPC` from Python. + +## API documentation + +There is an [online API documentation](https://api.lightning.community?python) +available that shows all currently existing RPC methods, including code snippets +on how to use them. From af0bfd015ad8d1e4504e1e14262d7f6ca96cbd2d Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Mon, 4 May 2020 09:56:12 +0200 Subject: [PATCH 4/8] docs+javascript: fix headings --- docs/grpc/javascript.md | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/docs/grpc/javascript.md b/docs/grpc/javascript.md index 6c69253a..5555aa2e 100644 --- a/docs/grpc/javascript.md +++ b/docs/grpc/javascript.md @@ -1,6 +1,6 @@ # How to write a simple `lnd` client in Javascript using `node.js` -### Setup and Installation +## Setup and Installation First, you'll need to initialize a simple nodejs project: ``` @@ -25,7 +25,7 @@ comment out the following line: //import "google/api/annotations.proto"; ``` -#### Imports and Client +### Imports and Client Every time you work with Javascript gRPC, you will have to import `grpc`, load `rpc.proto`, and create a connection to your client like so: @@ -48,14 +48,14 @@ var lnrpc = lnrpcDescriptor.lnrpc; var lightning = new lnrpc.Lightning('localhost:10009', credentials); ``` -### Examples +## Examples Let's walk through some examples of Javascript gRPC clients. These examples assume that you have at least two `lnd` nodes running, the RPC location of one of which is at the default `localhost:10009`, with an open channel between the two nodes. -#### Simple RPC +### Simple RPC ```js > lightning.getInfo({}, function(err, response) { @@ -79,7 +79,7 @@ GetInfo: { identity_pubkey: '03c892e3f3f077ea1e381c081abb36491a2502bc43ed37ffb82 chains: [ 'bitcoin' ] } ``` -#### Response-streaming RPC +### Response-streaming RPC ```js var call = lightning.subscribeInvoices({}); @@ -108,7 +108,7 @@ $ lncli sendpayment --pay_req= Your Javascript console should now display the details of the recently satisfied invoice. -#### Bidirectional-streaming RPC +### Bidirectional-streaming RPC This example has a few dependencies: ```shell @@ -165,7 +165,7 @@ async.series(payment_senders, function() { This example will send a payment of 100 satoshis every 2 seconds. -#### Using Macaroons +### Using Macaroons To authenticate using macaroons you need to include the macaroon in the metadata of the request. @@ -224,8 +224,7 @@ var client = new lnrpc.Lightning('some.address:10009', credentials); client.getInfo({}, (err, res) => { ... }); ``` - -### Conclusion +## Conclusion With the above, you should have all the `lnd` related `gRPC` dependencies installed locally in your project. In order to get up to speed with `protofbuf` From 192bcc15b75d4bf6c55c5263c0f13e6e0f1e0921 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Mon, 4 May 2020 09:59:59 +0200 Subject: [PATCH 5/8] docs+javascript: uset let and const, fix example formatting --- docs/grpc/javascript.md | 88 ++++++++++++++++++++++------------------- 1 file changed, 48 insertions(+), 40 deletions(-) diff --git a/docs/grpc/javascript.md b/docs/grpc/javascript.md index 5555aa2e..b596131e 100644 --- a/docs/grpc/javascript.md +++ b/docs/grpc/javascript.md @@ -31,8 +31,8 @@ Every time you work with Javascript gRPC, you will have to import `grpc`, load `rpc.proto`, and create a connection to your client like so: ```js -var grpc = require('grpc'); -var fs = require("fs"); +const grpc = require('grpc'); +const fs = require("fs"); // Due to updated ECDSA generated tls.cert we need to let gprc know that // we need to use that cipher suite otherwise there will be a handhsake @@ -41,11 +41,11 @@ process.env.GRPC_SSL_CIPHER_SUITES = 'HIGH+ECDSA' // Lnd cert is at ~/.lnd/tls.cert on Linux and // ~/Library/Application Support/Lnd/tls.cert on Mac -var lndCert = fs.readFileSync("~/.lnd/tls.cert"); -var credentials = grpc.credentials.createSsl(lndCert); -var lnrpcDescriptor = grpc.load("rpc.proto"); -var lnrpc = lnrpcDescriptor.lnrpc; -var lightning = new lnrpc.Lightning('localhost:10009', credentials); +let lndCert = fs.readFileSync("~/.lnd/tls.cert"); +let credentials = grpc.credentials.createSsl(lndCert); +let lnrpcDescriptor = grpc.load("rpc.proto"); +let lnrpc = lnrpcDescriptor.lnrpc; +let lightning = new lnrpc.Lightning('localhost:10009', credentials); ``` ## Examples @@ -58,9 +58,12 @@ two nodes. ### Simple RPC ```js -> lightning.getInfo({}, function(err, response) { - console.log('GetInfo:', response); - }); +lightning.getInfo({}, function(err, response) { + if (err) { + console.log('Error: ' + err); + } + console.log('GetInfo:', response); +}); ``` You should get something like this in your console: @@ -82,7 +85,7 @@ GetInfo: { identity_pubkey: '03c892e3f3f077ea1e381c081abb36491a2502bc43ed37ffb82 ### Response-streaming RPC ```js -var call = lightning.subscribeInvoices({}); +let call = lightning.subscribeInvoices({}); call.on('data', function(invoice) { console.log(invoice); }) @@ -120,15 +123,15 @@ You can run the following in your shell or put it in a program and run it like ```js // Load some libraries specific to this example -var async = require('async'); -var _ = require('lodash'); -var ByteBuffer = require('bytebuffer'); +const async = require('async'); +const _ = require('lodash'); +const ByteBuffer = require('bytebuffer'); -var dest_pubkey = ; -var dest_pubkey_bytes = ByteBuffer.fromHex(dest_pubkey); +let dest_pubkey = ; +let dest_pubkey_bytes = ByteBuffer.fromHex(dest_pubkey); // Set a listener on the bidirectional stream -var call = lightning.sendPayment(); +let call = lightning.sendPayment(); call.on('data', function(payment) { console.log("Payment sent:"); console.log(payment); @@ -153,8 +156,8 @@ function paymentSender(destination, amount) { _.delay(callback, 2000); }; } -var payment_senders = []; -for (var i = 0; i < 10; i++) { +let payment_senders = []; +for (let i = 0; i < 10; i++) { payment_senders[i] = paymentSender(dest_pubkey_bytes, 100); } async.series(payment_senders, function() { @@ -170,20 +173,20 @@ This example will send a payment of 100 satoshis every 2 seconds. To authenticate using macaroons you need to include the macaroon in the metadata of the request. ```js -var fs = require('fs'); -var grpc = require('grpc'); +const fs = require('fs'); +const grpc = require('grpc'); process.env.GRPC_SSL_CIPHER_SUITES = 'HIGH+ECDSA' // Lnd admin macaroon is at ~/.lnd/data/chain/bitcoin/simnet/admin.macaroon on Linux and // ~/Library/Application Support/Lnd/data/chain/bitcoin/simnet/admin.macaroon on Mac -var m = fs.readFileSync('~/.lnd/data/chain/bitcoin/simnet/admin.macaroon'); -var macaroon = m.toString('hex'); -var meta = new grpc.Metadata().add('macaroon', macaroon); +let m = fs.readFileSync('~/.lnd/data/chain/bitcoin/simnet/admin.macaroon'); +let macaroon = m.toString('hex'); +let meta = new grpc.Metadata().add('macaroon', macaroon); -var lnrpcDescriptor = grpc.load("rpc.proto"); -var lnrpc = lnrpcDescriptor.lnrpc; -var client = new lnrpc.Lightning('some.address:10009', grpc.credentials.createInsecure()); +let lnrpcDescriptor = grpc.load("rpc.proto"); +let lnrpc = lnrpcDescriptor.lnrpc; +let client = new lnrpc.Lightning('some.address:10009', grpc.credentials.createInsecure()); client.getInfo({}, meta); ``` @@ -191,37 +194,42 @@ client.getInfo({}, meta); However, this can get tiresome to do for each request, so to avoid explicitly including the macaroon we can update the credentials to include it automatically. ```js -var fs = require('fs'); -var grpc = require('grpc'); +const fs = require('fs'); +const grpc = require('grpc'); process.env.GRPC_SSL_CIPHER_SUITES = 'HIGH+ECDSA' // Lnd admin macaroon is at ~/.lnd/data/chain/bitcoin/simnet/admin.macaroon on Linux and // ~/Library/Application Support/Lnd/data/chain/bitcoin/simnet/admin.macaroon on Mac -var m = fs.readFileSync('~/.lnd/data/chain/bitcoin/simnet/admin.macaroon'); -var macaroon = m.toString('hex'); +let m = fs.readFileSync('~/.lnd/data/chain/bitcoin/simnet/admin.macaroon'); +let macaroon = m.toString('hex'); // build meta data credentials -var metadata = new grpc.Metadata() +let metadata = new grpc.Metadata() metadata.add('macaroon', macaroon) -var macaroonCreds = grpc.credentials.createFromMetadataGenerator((_args, callback) => { +let macaroonCreds = grpc.credentials.createFromMetadataGenerator((_args, callback) => { callback(null, metadata); }); // build ssl credentials using the cert the same as before -var lndCert = fs.readFileSync("~/.lnd/tls.cert"); -var sslCreds = grpc.credentials.createSsl(lndCert); +let lndCert = fs.readFileSync("~/.lnd/tls.cert"); +let sslCreds = grpc.credentials.createSsl(lndCert); // combine the cert credentials and the macaroon auth credentials // such that every call is properly encrypted and authenticated -var credentials = grpc.credentials.combineChannelCredentials(sslCreds, macaroonCreds); +let credentials = grpc.credentials.combineChannelCredentials(sslCreds, macaroonCreds); // Pass the crendentials when creating a channel -var lnrpcDescriptor = grpc.load("rpc.proto"); -var lnrpc = lnrpcDescriptor.lnrpc; -var client = new lnrpc.Lightning('some.address:10009', credentials); +let lnrpcDescriptor = grpc.load("rpc.proto"); +let lnrpc = lnrpcDescriptor.lnrpc; +let client = new lnrpc.Lightning('some.address:10009', credentials); -client.getInfo({}, (err, res) => { ... }); +client.getInfo({}, (err, response) => { + if (err) { + console.log('Error: ' + err); + } + console.log('GetInfo:', response); +}); ``` ## Conclusion From d950b7e4b1d69afa1612e18b78c43220c9939707 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Mon, 4 May 2020 10:07:54 +0200 Subject: [PATCH 6/8] docs+javascript: remove broken macaroon example --- docs/grpc/javascript.md | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/docs/grpc/javascript.md b/docs/grpc/javascript.md index b596131e..357da539 100644 --- a/docs/grpc/javascript.md +++ b/docs/grpc/javascript.md @@ -170,28 +170,10 @@ This example will send a payment of 100 satoshis every 2 seconds. ### Using Macaroons -To authenticate using macaroons you need to include the macaroon in the metadata of the request. +To authenticate using macaroons you need to include the macaroon in the metadata +of each request. -```js -const fs = require('fs'); -const grpc = require('grpc'); - -process.env.GRPC_SSL_CIPHER_SUITES = 'HIGH+ECDSA' - -// Lnd admin macaroon is at ~/.lnd/data/chain/bitcoin/simnet/admin.macaroon on Linux and -// ~/Library/Application Support/Lnd/data/chain/bitcoin/simnet/admin.macaroon on Mac -let m = fs.readFileSync('~/.lnd/data/chain/bitcoin/simnet/admin.macaroon'); -let macaroon = m.toString('hex'); -let meta = new grpc.Metadata().add('macaroon', macaroon); - -let lnrpcDescriptor = grpc.load("rpc.proto"); -let lnrpc = lnrpcDescriptor.lnrpc; -let client = new lnrpc.Lightning('some.address:10009', grpc.credentials.createInsecure()); - -client.getInfo({}, meta); -``` - -However, this can get tiresome to do for each request, so to avoid explicitly including the macaroon we can update the credentials to include it automatically. +The following snippet will add the macaroon to every request automatically: ```js const fs = require('fs'); From 92abf3ed8793af4e663235e90cbcd753a81ea435 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Mon, 4 May 2020 09:57:18 +0200 Subject: [PATCH 7/8] docs+javascript: switch to proto-loader --- docs/grpc/javascript.md | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/docs/grpc/javascript.md b/docs/grpc/javascript.md index 357da539..0f5e4346 100644 --- a/docs/grpc/javascript.md +++ b/docs/grpc/javascript.md @@ -7,9 +7,10 @@ First, you'll need to initialize a simple nodejs project: npm init (or npm init -f if you want to use the default values without prompt) ``` -Then you need to install the Javascript grpc library dependency: +Then you need to install the Javascript grpc and proto loader library +dependencies: ``` -npm install grpc --save +npm install grpc @grpc/proto-loader --save ``` You also need to copy the `lnd` `rpc.proto` file in your project directory (or @@ -18,13 +19,6 @@ at least somewhere reachable by your Javascript code). The `rpc.proto` file is [located in the `lnrpc` directory of the `lnd` sources](https://github.com/lightningnetwork/lnd/blob/master/lnrpc/rpc.proto). -In order for the auto-generated code to compile successfully, you'll need to -comment out the following line: - -``` -//import "google/api/annotations.proto"; -``` - ### Imports and Client Every time you work with Javascript gRPC, you will have to import `grpc`, load @@ -32,6 +26,7 @@ Every time you work with Javascript gRPC, you will have to import `grpc`, load ```js const grpc = require('grpc'); +const protoLoader = require('@grpc/proto-loader'); const fs = require("fs"); // Due to updated ECDSA generated tls.cert we need to let gprc know that @@ -39,11 +34,22 @@ const fs = require("fs"); // error when we communicate with the lnd rpc server. process.env.GRPC_SSL_CIPHER_SUITES = 'HIGH+ECDSA' +// We need to give the proto loader some extra options, otherwise the code won't +// fully work with lnd. +const loaderOptions = { + keepCase: true, + longs: String, + enums: String, + defaults: true, + oneofs: true +}; +const packageDefinition = protoLoader.loadSync('rpc.proto', loaderOptions); + // Lnd cert is at ~/.lnd/tls.cert on Linux and // ~/Library/Application Support/Lnd/tls.cert on Mac let lndCert = fs.readFileSync("~/.lnd/tls.cert"); let credentials = grpc.credentials.createSsl(lndCert); -let lnrpcDescriptor = grpc.load("rpc.proto"); +let lnrpcDescriptor = grpc.loadPackageDefinition(packageDefinition); let lnrpc = lnrpcDescriptor.lnrpc; let lightning = new lnrpc.Lightning('localhost:10009', credentials); ``` @@ -178,6 +184,15 @@ The following snippet will add the macaroon to every request automatically: ```js const fs = require('fs'); const grpc = require('grpc'); +const protoLoader = require('@grpc/proto-loader'); +const loaderOptions = { + keepCase: true, + longs: String, + enums: String, + defaults: true, + oneofs: true +}; +const packageDefinition = protoLoader.loadSync('rpc.proto', loaderOptions); process.env.GRPC_SSL_CIPHER_SUITES = 'HIGH+ECDSA' @@ -202,7 +217,7 @@ let sslCreds = grpc.credentials.createSsl(lndCert); let credentials = grpc.credentials.combineChannelCredentials(sslCreds, macaroonCreds); // Pass the crendentials when creating a channel -let lnrpcDescriptor = grpc.load("rpc.proto"); +let lnrpcDescriptor = grpc.loadPackageDefinition(packageDefinition); let lnrpc = lnrpcDescriptor.lnrpc; let client = new lnrpc.Lightning('some.address:10009', credentials); From b7ff6ab4ef107a4d95c11ff4c56e34bcfdac1f8e Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Mon, 4 May 2020 09:58:18 +0200 Subject: [PATCH 8/8] docs+javascript: add link to API documentation --- docs/grpc/javascript.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/grpc/javascript.md b/docs/grpc/javascript.md index 0f5e4346..d4dc0a3a 100644 --- a/docs/grpc/javascript.md +++ b/docs/grpc/javascript.md @@ -238,3 +238,9 @@ Javascript](https://developers.google.com/protocol-buffers/docs/reference/javasc Additionally, [this official gRPC resource](http://www.grpc.io/docs/tutorials/basic/node.html) provides more details around how to drive `gRPC` from `node.js`. + +## API documentation + +There is an [online API documentation](https://api.lightning.community?javascript) +available that shows all currently existing RPC methods, including code snippets +on how to use them.