Merge pull request #4511 from guggero/browser-websocket-fix

Bugfix: Allow browsers to send macaroon in WebSocket calls

docs/rest/websockets.md

@@ -0,0 +1,99 @@
+# WebSockets with `lnd`'s REST API
+
+This document describes how streaming response REST calls can be used correctly
+by making use of the WebSocket API.
+
+As an example, we are going to write a simple JavaScript program that subscribes
+to `lnd`'s
+[block notification RPC](https://api.lightning.community/#v2-chainnotifier-register-blocks).
+
+The WebSocket will be kept open as long as `lnd` runs and JavaScript program
+isn't stopped.
+
+## Browser environment
+
+When using WebSockets in a browser, there are certain security limitations of
+what header fields are allowed to be sent. Therefore, the macaroon cannot just
+be added as a `Grpc-Metadata-Macaroon` header field as it would work with normal
+REST calls. The browser will just ignore that header field and not send it.
+
+Instead we have added a workaround in `lnd`'s WebSocket proxy that allows
+sending the macaroon as a WebSocket "protocol":
+
+```javascript
+const host = 'localhost:8080'; // The default REST port of lnd, can be overwritten with --restlisten=ip:port
+const macaroon = '0201036c6e6402eb01030a10625e7e60fd00f5a6f9cd53f33fc82a...'; // The hex encoded macaroon to send
+const initialRequest = { // The initial request to send (see API docs for each RPC).
+    hash: "xlkMdV382uNPskw6eEjDGFMQHxHNnZZgL47aVDSwiRQ=", // Just some example to show that all `byte` fields always have to be base64 encoded in the REST API.
+    height: 144,
+}
+
+// The protocol is our workaround for sending the macaroon because custom header
+// fields aren't allowed to be sent by the browser when opening a WebSocket.
+const protocolString = 'Grpc-Metadata-Macaroon+' + macaroon;
+
+// Let's now connect the web socket. Notice that all WebSocket open calls are
+// always GET requests. If the RPC expects a call to be POST or DELETE (see API
+// docs to find out), the query parameter "method" can be set to overwrite.
+const wsUrl = 'wss://' + host + '/v2/chainnotifier/register/blocks?method=POST';
+let ws = new WebSocket(wsUrl, protocolString);
+ws.onopen = function (event) {
+    // After the WS connection is establishes, lnd expects the client to send the
+    // initial message. If an RPC doesn't have any request parameters, an empty
+    // JSON object has to be sent as a string, for example: ws.send('{}')
+    ws.send(JSON.stringify(initialRequest));
+}
+ws.onmessage = function (event) {
+    // We received a new message.
+    console.log(event);
+
+    // The data we're really interested in is in data and is always a string
+    // that needs to be parsed as JSON and always contains a "result" field:
+    console.log("Payload: ");
+    console.log(JSON.parse(event.data).result);
+}
+ws.onerror = function (event) {
+    // An error occured, let's log it to the console.
+    console.log(event);
+}
+```
+
+## Node.js environment
+
+With Node.js it is a bit easier to use the streaming response APIs because we
+can set the macaroon header field directly. This is the example from the API
+docs:
+
+```javascript
+// --------------------------
+// Example with websockets:
+// --------------------------
+const WebSocket = require('ws');
+const fs = require('fs');
+const macaroon = fs.readFileSync('LND_DIR/data/chain/bitcoin/simnet/admin.macaroon').toString('hex');
+let ws = new WebSocket('wss://localhost:8080/v2/chainnotifier/register/blocks?method=POST', {
+  // Work-around for self-signed certificates.
+  rejectUnauthorized: false,
+  headers: {
+    'Grpc-Metadata-Macaroon': macaroon,
+  },
+});
+let requestBody = { 
+  hash: "<byte>",
+  height: "<int64>",
+}
+ws.on('open', function() {
+    ws.send(JSON.stringify(requestBody));
+});
+ws.on('error', function(err) {
+    console.log('Error: ' + err);
+});
+ws.on('message', function(body) {
+    console.log(body);
+});
+// Console output (repeated for every message in the stream):
+//  { 
+//      "hash": <byte>, 
+//      "height": <int64>, 
+//  }
+```

lnrpc/websocket_proxy.go

@@ -21,6 +21,16 @@ const (
 	// This is necessary because the WebSocket API specifies that a
 	// handshake request must always be done through a GET request.
 	MethodOverrideParam = "method"
+
+	// HeaderWebSocketProtocol is the name of the WebSocket protocol
+	// exchange header field that we use to transport additional header
+	// fields.
+	HeaderWebSocketProtocol = "Sec-Websocket-Protocol"
+
+	// WebSocketProtocolDelimiter is the delimiter we use between the
+	// additional header field and its value. We use the plus symbol because
+	// the default delimiters aren't allowed in the protocol names.
+	WebSocketProtocolDelimiter = "+"
 )
 
 var (
@@ -32,6 +42,13 @@ var (
 		"Referer":                true,
 		"Grpc-Metadata-Macaroon": true,
 	}
+
+	// defaultProtocolsToAllow are additional header fields that we allow
+	// to be transported inside of the Sec-Websocket-Protocol field to be
+	// forwarded to the backend.
+	defaultProtocolsToAllow = map[string]bool{
+		"Grpc-Metadata-Macaroon": true,
+	}
 )
 
 // NewWebSocketProxy attempts to expose the underlying handler as a response-
@@ -101,13 +118,13 @@ func (p *WebsocketProxy) upgradeToWebSocketProxy(w http.ResponseWriter,
 		p.logger.Errorf("WS: error preparing request:", err)
 		return
 	}
-	for header := range r.Header {
+
-		headerName := textproto.CanonicalMIMEHeaderKey(header)
+	// Allow certain headers to be forwarded, either from source headers
-		forward, ok := defaultHeadersToForward[headerName]
+	// or the special Sec-Websocket-Protocol header field.
-		if ok && forward {
+	forwardHeaders(r.Header, request.Header)
-			request.Header.Set(headerName, r.Header.Get(header))
+
-		}
+	// Also allow the target request method to be overwritten, as all
-	}
+	// WebSocket establishment calls MUST be GET requests.
 	if m := r.URL.Query().Get(MethodOverrideParam); m != "" {
 		request.Method = m
 	}
@@ -182,6 +199,38 @@ func (p *WebsocketProxy) upgradeToWebSocketProxy(w http.ResponseWriter,
 	}
 }
 
+// forwardHeaders forwards certain allowed header fields from the source request
+// to the target request. Because browsers are limited in what header fields
+// they can send on the WebSocket setup call, we also allow additional fields to
+// be transported in the special Sec-Websocket-Protocol field.
+func forwardHeaders(source, target http.Header) {
+	// Forward allowed header fields directly.
+	for header := range source {
+		headerName := textproto.CanonicalMIMEHeaderKey(header)
+		forward, ok := defaultHeadersToForward[headerName]
+		if ok && forward {
+			target.Set(headerName, source.Get(header))
+		}
+	}
+
+	// Browser aren't allowed to set custom header fields on WebSocket
+	// requests. We need to allow them to submit the macaroon as a WS
+	// protocol, which is the only allowed header. Set any "protocols" we
+	// declare valid as header fields on the forwarded request.
+	protocol := source.Get(HeaderWebSocketProtocol)
+	for key := range defaultProtocolsToAllow {
+		if strings.HasPrefix(protocol, key) {
+			// The format is "<protocol name>+<value>". We know the
+			// protocol string starts with the name so we only need
+			// to set the value.
+			values := strings.Split(
+				protocol, WebSocketProtocolDelimiter,
+			)
+			target.Set(key, values[1])
+		}
+	}
+}
+
 // newRequestForwardingReader creates a new request forwarding pipe.
 func newRequestForwardingReader() *requestForwardingReader {
 	r, w := io.Pipe()

lntest/itest/rest_api.go

@@ -201,7 +201,116 @@ func testRestApi(net *lntest.NetworkHarness, ht *harnessTest) {
 				Height: uint32(height),
 			}
 			url := "/v2/chainnotifier/register/blocks"
-			c, err := openWebSocket(a, url, "POST", req)
+			c, err := openWebSocket(a, url, "POST", req, nil)
+			require.Nil(t, err, "websocket")
+			defer func() {
+				_ = c.WriteMessage(
+					websocket.CloseMessage,
+					websocket.FormatCloseMessage(
+						websocket.CloseNormalClosure,
+						"done",
+					),
+				)
+				_ = c.Close()
+			}()
+
+			msgChan := make(chan *chainrpc.BlockEpoch)
+			errChan := make(chan error)
+			timeout := time.After(defaultTimeout)
+
+			// We want to read exactly one message.
+			go func() {
+				defer close(msgChan)
+
+				_, msg, err := c.ReadMessage()
+				if err != nil {
+					errChan <- err
+					return
+				}
+
+				// The chunked/streamed responses come wrapped
+				// in either a {"result":{}} or {"error":{}}
+				// wrapper which we'll get rid of here.
+				msgStr := string(msg)
+				if !strings.Contains(msgStr, "\"result\":") {
+					errChan <- fmt.Errorf("invalid msg: %s",
+						msgStr)
+					return
+				}
+				msgStr = resultPattern.ReplaceAllString(
+					msgStr, "${1}",
+				)
+
+				// Make sure we can parse the unwrapped message
+				// into the expected proto message.
+				protoMsg := &chainrpc.BlockEpoch{}
+				err = jsonpb.UnmarshalString(
+					msgStr, protoMsg,
+				)
+				if err != nil {
+					errChan <- err
+					return
+				}
+
+				select {
+				case msgChan <- protoMsg:
+				case <-timeout:
+				}
+			}()
+
+			// Mine a block and make sure we get a message for it.
+			blockHashes, err := net.Miner.Node.Generate(1)
+			require.Nil(t, err, "generate blocks")
+			assert.Equal(t, 1, len(blockHashes), "num blocks")
+			select {
+			case msg := <-msgChan:
+				assert.Equal(
+					t, blockHashes[0].CloneBytes(),
+					msg.Hash, "block hash",
+				)
+
+			case err := <-errChan:
+				t.Fatalf("Received error from WS: %v", err)
+
+			case <-timeout:
+				t.Fatalf("Timeout before message was received")
+			}
+		},
+	}, {
+		name: "websocket subscription with macaroon in protocol",
+		run: func(t *testing.T, a, b *lntest.HarnessNode) {
+			// Find out the current best block so we can subscribe
+			// to the next one.
+			hash, height, err := net.Miner.Node.GetBestBlock()
+			require.Nil(t, err, "get best block")
+
+			// Create a new subscription to get block epoch events.
+			req := &chainrpc.BlockEpoch{
+				Hash:   hash.CloneBytes(),
+				Height: uint32(height),
+			}
+			url := "/v2/chainnotifier/register/blocks"
+
+			// This time we send the macaroon in the special header
+			// Sec-Websocket-Protocol which is the only header field
+			// available to browsers when opening a WebSocket.
+			mac, err := a.ReadMacaroon(
+				a.AdminMacPath(), defaultTimeout,
+			)
+			require.NoError(t, err, "read admin mac")
+			macBytes, err := mac.MarshalBinary()
+			require.NoError(t, err, "marshal admin mac")
+
+			customHeader := make(http.Header)
+			customHeader.Set(
+				lnrpc.HeaderWebSocketProtocol, fmt.Sprintf(
+					"Grpc-Metadata-Macaroon+%s",
+					hex.EncodeToString(macBytes),
+				),
+			)
+			c, err := openWebSocket(
+				a, url, "POST", req, customHeader,
+			)
 			require.Nil(t, err, "websocket")
 			defer func() {
 				_ = c.WriteMessage(
@@ -364,14 +473,17 @@ func makeRequest(node *lntest.HarnessNode, url, method string,
 // openWebSocket opens a new WebSocket connection to the given URL with the
 // appropriate macaroon headers and sends the request message over the socket.
 func openWebSocket(node *lntest.HarnessNode, url, method string,
-	req proto.Message) (*websocket.Conn, error) {
+	req proto.Message, customHeader http.Header) (*websocket.Conn, error) {
 
 	// Prepare our macaroon headers and assemble the full URL from the
 	// node's listening address. WebSockets always work over GET so we need
 	// to append the target request method as a query parameter.
-	header := make(http.Header)
+	header := customHeader
-	if err := addAdminMacaroon(node, header); err != nil {
+	if header == nil {
-		return nil, err
+		header = make(http.Header)
+		if err := addAdminMacaroon(node, header); err != nil {
+			return nil, err
+		}
 	}
 	fullURL := fmt.Sprintf(
 		"wss://%s%s?method=%s", node.Cfg.RESTAddr(), url, method,