diff --git a/config.go b/config.go index 11916a1a..b23bb5e4 100644 --- a/config.go +++ b/config.go @@ -159,6 +159,7 @@ type Config struct { RawExternalIPs []string `long:"externalip" description:"Add an ip:port to the list of local addresses we claim to listen on to peers. If a port is not specified, the default (9735) will be used regardless of other parameters"` RPCListeners []net.Addr RESTListeners []net.Addr + RestCORS []string `long:"restcors" description:"Add an ip:port/hostname to allow cross origin access from. To allow all origins, set as \"*\"."` Listeners []net.Addr ExternalIPs []net.Addr DisableListen bool `long:"nolisten" description:"Disable listening for incoming peer connections"` diff --git a/go.mod b/go.mod index ff2108a6..87da6a0b 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.3.2 github.com/google/btree v1.0.0 // indirect - github.com/gorilla/websocket v1.4.1 // indirect + github.com/gorilla/websocket v1.4.2 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/grpc-ecosystem/grpc-gateway v1.14.3 diff --git a/go.sum b/go.sum index 578baa56..060921bd 100644 --- a/go.sum +++ b/go.sum @@ -132,8 +132,8 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= -github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= diff --git a/lnrpc/rpc.pb.go b/lnrpc/rpc.pb.go index f6cc76fc..3d81001b 100644 --- a/lnrpc/rpc.pb.go +++ b/lnrpc/rpc.pb.go @@ -12367,6 +12367,11 @@ type LightningClient interface { // lncli: `estimatefee` //EstimateFee asks the chain backend to estimate the fee rate and total fees //for a transaction that pays to multiple specified outputs. + // + //When using REST, the `AddrToAmount` map type can be set by appending + //`&AddrToAmount[
]=` to the URL. Unfortunately this + //map type doesn't appear in the REST API documentation because of a bug in + //the grpc-gateway library. EstimateFee(ctx context.Context, in *EstimateFeeRequest, opts ...grpc.CallOption) (*EstimateFeeResponse, error) // lncli: `sendcoins` //SendCoins executes a request to send coins to a particular address. Unlike @@ -12590,6 +12595,11 @@ type LightningClient interface { //satoshis. The returned route contains the full details required to craft and //send an HTLC, also including the necessary information that should be //present within the Sphinx packet encapsulated within the HTLC. + // + //When using REST, the `dest_custom_records` map type can be set by appending + //`&dest_custom_records[]=` + //to the URL. Unfortunately this map type doesn't appear in the REST API + //documentation because of a bug in the grpc-gateway library. QueryRoutes(ctx context.Context, in *QueryRoutesRequest, opts ...grpc.CallOption) (*QueryRoutesResponse, error) // lncli: `getnetworkinfo` //GetNetworkInfo returns some basic stats about the known channel graph from @@ -13448,6 +13458,11 @@ type LightningServer interface { // lncli: `estimatefee` //EstimateFee asks the chain backend to estimate the fee rate and total fees //for a transaction that pays to multiple specified outputs. + // + //When using REST, the `AddrToAmount` map type can be set by appending + //`&AddrToAmount[
]=` to the URL. Unfortunately this + //map type doesn't appear in the REST API documentation because of a bug in + //the grpc-gateway library. EstimateFee(context.Context, *EstimateFeeRequest) (*EstimateFeeResponse, error) // lncli: `sendcoins` //SendCoins executes a request to send coins to a particular address. Unlike @@ -13671,6 +13686,11 @@ type LightningServer interface { //satoshis. The returned route contains the full details required to craft and //send an HTLC, also including the necessary information that should be //present within the Sphinx packet encapsulated within the HTLC. + // + //When using REST, the `dest_custom_records` map type can be set by appending + //`&dest_custom_records[]=` + //to the URL. Unfortunately this map type doesn't appear in the REST API + //documentation because of a bug in the grpc-gateway library. QueryRoutes(context.Context, *QueryRoutesRequest) (*QueryRoutesResponse, error) // lncli: `getnetworkinfo` //GetNetworkInfo returns some basic stats about the known channel graph from diff --git a/lnrpc/rpc.proto b/lnrpc/rpc.proto index bd681142..f69b768f 100644 --- a/lnrpc/rpc.proto +++ b/lnrpc/rpc.proto @@ -46,6 +46,11 @@ service Lightning { /* lncli: `estimatefee` EstimateFee asks the chain backend to estimate the fee rate and total fees for a transaction that pays to multiple specified outputs. + + When using REST, the `AddrToAmount` map type can be set by appending + `&AddrToAmount[
]=` to the URL. Unfortunately this + map type doesn't appear in the REST API documentation because of a bug in + the grpc-gateway library. */ rpc EstimateFee (EstimateFeeRequest) returns (EstimateFeeResponse); @@ -355,6 +360,11 @@ service Lightning { satoshis. The returned route contains the full details required to craft and send an HTLC, also including the necessary information that should be present within the Sphinx packet encapsulated within the HTLC. + + When using REST, the `dest_custom_records` map type can be set by appending + `&dest_custom_records[]=` + to the URL. Unfortunately this map type doesn't appear in the REST API + documentation because of a bug in the grpc-gateway library. */ rpc QueryRoutes (QueryRoutesRequest) returns (QueryRoutesResponse); diff --git a/lnrpc/rpc.swagger.json b/lnrpc/rpc.swagger.json index 584e5c56..cd36a870 100644 --- a/lnrpc/rpc.swagger.json +++ b/lnrpc/rpc.swagger.json @@ -1001,6 +1001,7 @@ "/v1/graph/routes/{pub_key}/{amt}": { "get": { "summary": "lncli: `queryroutes`\nQueryRoutes attempts to query the daemon's Channel Router for a possible\nroute to a target destination capable of carrying a specific amount of\nsatoshis. The returned route contains the full details required to craft and\nsend an HTLC, also including the necessary information that should be\npresent within the Sphinx packet encapsulated within the HTLC.", + "description": "When using REST, the `dest_custom_records` map type can be set by appending\n`\u0026dest_custom_records[\u003crecord_number\u003e]=\u003crecord_data_base64_url_encoded\u003e`\nto the URL. Unfortunately this map type doesn't appear in the REST API\ndocumentation because of a bug in the grpc-gateway library.", "operationId": "QueryRoutes", "responses": { "200": { @@ -1854,6 +1855,7 @@ "/v1/transactions/fee": { "get": { "summary": "lncli: `estimatefee`\nEstimateFee asks the chain backend to estimate the fee rate and total fees\nfor a transaction that pays to multiple specified outputs.", + "description": "When using REST, the `AddrToAmount` map type can be set by appending\n`\u0026AddrToAmount[\u003caddress\u003e]=\u003camount_to_send\u003e` to the URL. Unfortunately this\nmap type doesn't appear in the REST API documentation because of a bug in\nthe grpc-gateway library.", "operationId": "EstimateFee", "responses": { "200": { diff --git a/lnrpc/websocket_proxy.go b/lnrpc/websocket_proxy.go new file mode 100644 index 00000000..921b2171 --- /dev/null +++ b/lnrpc/websocket_proxy.go @@ -0,0 +1,305 @@ +// The code in this file is a heavily modified version of +// https://github.com/tmc/grpc-websocket-proxy/ + +package lnrpc + +import ( + "bufio" + "io" + "net/http" + "net/textproto" + "strings" + + "github.com/btcsuite/btclog" + "github.com/gorilla/websocket" + "golang.org/x/net/context" +) + +const ( + // MethodOverrideParam is the GET query parameter that specifies what + // HTTP request method should be used for the forwarded REST request. + // This is necessary because the WebSocket API specifies that a + // handshake request must always be done through a GET request. + MethodOverrideParam = "method" +) + +var ( + // defaultHeadersToForward is a map of all HTTP header fields that are + // forwarded by default. The keys must be in the canonical MIME header + // format. + defaultHeadersToForward = map[string]bool{ + "Origin": true, + "Referer": true, + "Grpc-Metadata-Macaroon": true, + } +) + +// NewWebSocketProxy attempts to expose the underlying handler as a response- +// streaming WebSocket stream with newline-delimited JSON as the content +// encoding. +func NewWebSocketProxy(h http.Handler, logger btclog.Logger) http.Handler { + p := &WebsocketProxy{ + backend: h, + logger: logger, + upgrader: &websocket.Upgrader{ + ReadBufferSize: 1024, + WriteBufferSize: 1024, + CheckOrigin: func(r *http.Request) bool { + return true + }, + }, + } + return p +} + +// WebsocketProxy provides websocket transport upgrade to compatible endpoints. +type WebsocketProxy struct { + backend http.Handler + logger btclog.Logger + upgrader *websocket.Upgrader +} + +// ServeHTTP handles the incoming HTTP request. If the request is an +// "upgradeable" WebSocket request (identified by header fields), then the +// WS proxy handles the request. Otherwise the request is passed directly to the +// underlying REST proxy. +func (p *WebsocketProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if !websocket.IsWebSocketUpgrade(r) { + p.backend.ServeHTTP(w, r) + return + } + p.upgradeToWebSocketProxy(w, r) +} + +// upgradeToWebSocketProxy upgrades the incoming request to a WebSocket, reads +// one incoming message then streams all responses until either the client or +// server quit the connection. +func (p *WebsocketProxy) upgradeToWebSocketProxy(w http.ResponseWriter, + r *http.Request) { + + conn, err := p.upgrader.Upgrade(w, r, nil) + if err != nil { + p.logger.Errorf("error upgrading websocket:", err) + return + } + defer func() { + err := conn.Close() + if err != nil && !IsClosedConnError(err) { + p.logger.Errorf("WS: error closing upgraded conn: %v", + err) + } + }() + + ctx, cancelFn := context.WithCancel(context.Background()) + defer cancelFn() + + requestForwarder := newRequestForwardingReader() + request, err := http.NewRequestWithContext( + r.Context(), r.Method, r.URL.String(), requestForwarder, + ) + if err != nil { + p.logger.Errorf("WS: error preparing request:", err) + return + } + for header := range r.Header { + headerName := textproto.CanonicalMIMEHeaderKey(header) + forward, ok := defaultHeadersToForward[headerName] + if ok && forward { + request.Header.Set(headerName, r.Header.Get(header)) + } + } + if m := r.URL.Query().Get(MethodOverrideParam); m != "" { + request.Method = m + } + + responseForwarder := newResponseForwardingWriter() + go func() { + <-ctx.Done() + responseForwarder.Close() + }() + + go func() { + defer cancelFn() + p.backend.ServeHTTP(responseForwarder, request) + }() + + // Read loop: Take messages from websocket and write to http request. + go func() { + defer cancelFn() + for { + select { + case <-ctx.Done(): + return + default: + } + + _, payload, err := conn.ReadMessage() + if err != nil { + if IsClosedConnError(err) { + p.logger.Tracef("WS: socket "+ + "closed: %v", err) + return + } + p.logger.Errorf("error reading message: %v", + err) + return + } + _, err = requestForwarder.Write(payload) + if err != nil { + p.logger.Errorf("WS: error writing message "+ + "to upstream http server: %v", err) + return + } + _, _ = requestForwarder.Write([]byte{'\n'}) + + // We currently only support server-streaming messages. + // Therefore we close the request body after the first + // incoming message to trigger a response. + requestForwarder.CloseWriter() + } + }() + + // Write loop: Take messages from the response forwarder and write them + // to the WebSocket. + for responseForwarder.Scan() { + if len(responseForwarder.Bytes()) == 0 { + p.logger.Errorf("WS: empty scan: %v", + responseForwarder.Err()) + + continue + } + + err = conn.WriteMessage( + websocket.TextMessage, responseForwarder.Bytes(), + ) + if err != nil { + p.logger.Errorf("WS: error writing message: %v", err) + return + } + } + if err := responseForwarder.Err(); err != nil && !IsClosedConnError(err) { + p.logger.Errorf("WS: scanner err: %v", err) + } +} + +// newRequestForwardingReader creates a new request forwarding pipe. +func newRequestForwardingReader() *requestForwardingReader { + r, w := io.Pipe() + return &requestForwardingReader{ + Reader: r, + Writer: w, + pipeR: r, + pipeW: w, + } +} + +// requestForwardingReader is a wrapper around io.Pipe that embeds both the +// io.Reader and io.Writer interface and can be closed. +type requestForwardingReader struct { + io.Reader + io.Writer + + pipeR *io.PipeReader + pipeW *io.PipeWriter +} + +// CloseWriter closes the underlying pipe writer. +func (r *requestForwardingReader) CloseWriter() { + _ = r.pipeW.CloseWithError(io.EOF) +} + +// newResponseForwardingWriter creates a new http.ResponseWriter that intercepts +// what's written to it and presents it through a bufio.Scanner interface. +func newResponseForwardingWriter() *responseForwardingWriter { + r, w := io.Pipe() + return &responseForwardingWriter{ + Writer: w, + Scanner: bufio.NewScanner(r), + pipeR: r, + pipeW: w, + header: http.Header{}, + closed: make(chan bool, 1), + } +} + +// responseForwardingWriter is a type that implements the http.ResponseWriter +// interface but internally forwards what's written to the writer through a pipe +// so it can easily be read again through the bufio.Scanner interface. +type responseForwardingWriter struct { + io.Writer + *bufio.Scanner + + pipeR *io.PipeReader + pipeW *io.PipeWriter + + header http.Header + code int + closed chan bool +} + +// Write writes the given bytes to the internal pipe. +// +// NOTE: This is part of the http.ResponseWriter interface. +func (w *responseForwardingWriter) Write(b []byte) (int, error) { + return w.Writer.Write(b) +} + +// Header returns the HTTP header fields intercepted so far. +// +// NOTE: This is part of the http.ResponseWriter interface. +func (w *responseForwardingWriter) Header() http.Header { + return w.header +} + +// WriteHeader indicates that the header part of the response is now finished +// and sets the response code. +// +// NOTE: This is part of the http.ResponseWriter interface. +func (w *responseForwardingWriter) WriteHeader(code int) { + w.code = code +} + +// CloseNotify returns a channel that indicates if a connection was closed. +// +// NOTE: This is part of the http.CloseNotifier interface. +func (w *responseForwardingWriter) CloseNotify() <-chan bool { + return w.closed +} + +// Flush empties all buffers. We implement this to indicate to our backend that +// we support flushing our content. There is no actual implementation because +// all writes happen immediately, there is no internal buffering. +// +// NOTE: This is part of the http.Flusher interface. +func (w *responseForwardingWriter) Flush() {} + +func (w *responseForwardingWriter) Close() { + _ = w.pipeR.CloseWithError(io.EOF) + _ = w.pipeW.CloseWithError(io.EOF) + w.closed <- true +} + +// IsClosedConnError is a helper function that returns true if the given error +// is an error indicating we are using a closed connection. +func IsClosedConnError(err error) bool { + if err == nil { + return false + } + if err == http.ErrServerClosed { + return true + } + + str := err.Error() + if strings.Contains(str, "use of closed network connection") { + return true + } + if strings.Contains(str, "closed pipe") { + return true + } + if strings.Contains(str, "broken pipe") { + return true + } + return websocket.IsCloseError( + err, websocket.CloseNormalClosure, websocket.CloseGoingAway, + ) +} diff --git a/lntest/itest/lnd_test.go b/lntest/itest/lnd_test.go index 701349ed..215c710d 100644 --- a/lntest/itest/lnd_test.go +++ b/lntest/itest/lnd_test.go @@ -14665,6 +14665,10 @@ var testsCases = []*testCase{ name: "send multi path payment", test: testSendMultiPathPayment, }, + { + name: "REST API", + test: testRestApi, + }, } // TestLightningNetworkDaemon performs a series of integration tests amongst a diff --git a/lntest/itest/log_error_whitelist.txt b/lntest/itest/log_error_whitelist.txt index 4267a3ae..6541cab0 100644 --- a/lntest/itest/log_error_whitelist.txt +++ b/lntest/itest/log_error_whitelist.txt @@ -130,6 +130,8 @@