lnd.xprv/lntest/itest/lnd_rest_api_test.go

package itest

import (
	"bytes"
	"context"
	"crypto/tls"
	"encoding/base64"
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"regexp"
	"strings"
	"testing"
	"time"

	"github.com/golang/protobuf/jsonpb"
	"github.com/golang/protobuf/proto"
	"github.com/gorilla/websocket"
	"github.com/lightningnetwork/lnd/lnrpc"
	"github.com/lightningnetwork/lnd/lnrpc/autopilotrpc"
	"github.com/lightningnetwork/lnd/lnrpc/chainrpc"
	"github.com/lightningnetwork/lnd/lnrpc/routerrpc"
	"github.com/lightningnetwork/lnd/lnrpc/verrpc"
	"github.com/lightningnetwork/lnd/lnrpc/walletrpc"
	"github.com/lightningnetwork/lnd/lntest"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

var (
	insecureTransport = &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}
	restClient = &http.Client{
		Transport: insecureTransport,
	}
	jsonMarshaler = &jsonpb.Marshaler{
		EmitDefaults: true,
		OrigName:     true,
		Indent:       "    ",
	}
	urlEnc          = base64.URLEncoding
	webSocketDialer = &websocket.Dialer{
		HandshakeTimeout: 45 * time.Second,
		TLSClientConfig:  insecureTransport.TLSClientConfig,
	}
	resultPattern = regexp.MustCompile("{\"result\":(.*)}")
	closeMsg      = websocket.FormatCloseMessage(
		websocket.CloseNormalClosure, "done",
	)
)

// testRestAPI tests that the most important features of the REST API work
// correctly.
func testRestAPI(net *lntest.NetworkHarness, ht *harnessTest) {
	testCases := []struct {
		name string
		run  func(*testing.T, *lntest.HarnessNode, *lntest.HarnessNode)
	}{{
		name: "simple GET",
		run: func(t *testing.T, a, b *lntest.HarnessNode) {
			// Check that the parsing into the response proto
			// message works.
			resp := &lnrpc.GetInfoResponse{}
			err := invokeGET(a, "/v1/getinfo", resp)
			require.Nil(t, err, "getinfo")
			assert.Equal(t, "#3399ff", resp.Color, "node color")

			// Make sure we get the correct field names (snake
			// case).
			_, resp2, err := makeRequest(
				a, "/v1/getinfo", "GET", nil, nil,
			)
			require.Nil(t, err, "getinfo")
			assert.Contains(
				t, string(resp2), "best_header_timestamp",
				"getinfo",
			)
		},
	}, {
		name: "simple POST and GET with query param",
		run: func(t *testing.T, a, b *lntest.HarnessNode) {
			// Add an invoice, testing POST in the process.
			req := &lnrpc.Invoice{Value: 1234}
			resp := &lnrpc.AddInvoiceResponse{}
			err := invokePOST(a, "/v1/invoices", req, resp)
			require.Nil(t, err, "add invoice")
			assert.Equal(t, 32, len(resp.RHash), "invoice rhash")

			// Make sure we can call a GET endpoint with a hex
			// encoded URL part.
			url := fmt.Sprintf("/v1/invoice/%x", resp.RHash)
			resp2 := &lnrpc.Invoice{}
			err = invokeGET(a, url, resp2)
			require.Nil(t, err, "query invoice")
			assert.Equal(t, int64(1234), resp2.Value, "invoice amt")
		},
	}, {
		name: "GET with base64 encoded byte slice in path",
		run: func(t *testing.T, a, b *lntest.HarnessNode) {
			url := "/v2/router/mc/probability/%s/%s/%d"
			url = fmt.Sprintf(
				url, urlEnc.EncodeToString(a.PubKey[:]),
				urlEnc.EncodeToString(b.PubKey[:]), 1234,
			)
			resp := &routerrpc.QueryProbabilityResponse{}
			err := invokeGET(a, url, resp)
			require.Nil(t, err, "query probability")
			assert.Greater(t, resp.Probability, 0.5, "probability")
		},
	}, {
		name: "GET with map type query param",
		run: func(t *testing.T, a, b *lntest.HarnessNode) {
			// Get a new wallet address from Alice.
			ctxb := context.Background()
			newAddrReq := &lnrpc.NewAddressRequest{
				Type: lnrpc.AddressType_WITNESS_PUBKEY_HASH,
			}
			addrRes, err := a.NewAddress(ctxb, newAddrReq)
			require.Nil(t, err, "get address")

			// Create the full URL with the map query param.
			url := "/v1/transactions/fee?target_conf=%d&" +
				"AddrToAmount[%s]=%d"
			url = fmt.Sprintf(url, 2, addrRes.Address, 50000)
			resp := &lnrpc.EstimateFeeResponse{}
			err = invokeGET(a, url, resp)
			require.Nil(t, err, "estimate fee")
			assert.Greater(t, resp.FeeSat, int64(253), "fee")
		},
	}, {
		name: "sub RPC servers REST support",
		run: func(t *testing.T, a, b *lntest.HarnessNode) {
			// Query autopilot status.
			res1 := &autopilotrpc.StatusResponse{}
			err := invokeGET(a, "/v2/autopilot/status", res1)
			require.Nil(t, err, "autopilot status")
			assert.Equal(t, false, res1.Active, "autopilot status")

			// Query the version RPC.
			res2 := &verrpc.Version{}
			err = invokeGET(a, "/v2/versioner/version", res2)
			require.Nil(t, err, "version")
			assert.Greater(
				t, res2.AppMinor, uint32(0), "lnd minor version",
			)

			// Request a new external address from the wallet kit.
			req1 := &walletrpc.AddrRequest{}
			res3 := &walletrpc.AddrResponse{}
			err = invokePOST(
				a, "/v2/wallet/address/next", req1, res3,
			)
			require.Nil(t, err, "address")
			assert.NotEmpty(t, res3.Addr, "address")
		},
	}, {
		name: "CORS headers",
		run: func(t *testing.T, a, b *lntest.HarnessNode) {
			// Alice allows all origins. Make sure we get the same
			// value back in the CORS header that we send in the
			// Origin header.
			reqHeaders := make(http.Header)
			reqHeaders.Add("Origin", "https://foo.bar:9999")
			resHeaders, body, err := makeRequest(
				a, "/v1/getinfo", "OPTIONS", nil, reqHeaders,
			)
			require.Nil(t, err, "getinfo")
			assert.Equal(
				t, "https://foo.bar:9999",
				resHeaders.Get("Access-Control-Allow-Origin"),
				"CORS header",
			)
			assert.Equal(t, 0, len(body))

			// Make sure that we don't get a value set for Bob which
			// doesn't allow any CORS origin.
			resHeaders, body, err = makeRequest(
				b, "/v1/getinfo", "OPTIONS", nil, reqHeaders,
			)
			require.Nil(t, err, "getinfo")
			assert.Equal(
				t, "",
				resHeaders.Get("Access-Control-Allow-Origin"),
				"CORS header",
			)
			assert.Equal(t, 0, len(body))
		},
	}}
	wsTestCases := []struct {
		name string
		run  func(ht *harnessTest, net *lntest.NetworkHarness)
	}{{
		name: "websocket subscription",
		run:  wsTestCaseSubscription,
	}, {
		name: "websocket subscription with macaroon in protocol",
		run:  wsTestCaseSubscriptionMacaroon,
	}, {
		name: "websocket bi-directional subscription",
		run:  wsTestCaseBiDirectionalSubscription,
	}}

	// Make sure Alice allows all CORS origins. Bob will keep the default.
	net.Alice.Cfg.ExtraArgs = append(
		net.Alice.Cfg.ExtraArgs, "--restcors=\"*\"",
	)
	err := net.RestartNode(net.Alice, nil)
	if err != nil {
		ht.t.Fatalf("Could not restart Alice to set CORS config: %v",
			err)
	}

	for _, tc := range testCases {
		tc := tc
		ht.t.Run(tc.name, func(t *testing.T) {
			tc.run(t, net.Alice, net.Bob)
		})
	}

	for _, tc := range wsTestCases {
		tc := tc
		ht.t.Run(tc.name, func(t *testing.T) {
			ht := &harnessTest{
				t: t, testCase: ht.testCase, lndHarness: net,
			}
			tc.run(ht, net)
		})
	}
}

func wsTestCaseSubscription(ht *harnessTest, net *lntest.NetworkHarness) {
	// Find out the current best block so we can subscribe to the next one.
	hash, height, err := net.Miner.Client.GetBestBlock()
	require.Nil(ht.t, err, "get best block")

	// Create a new subscription to get block epoch events.
	req := &chainrpc.BlockEpoch{
		Hash:   hash.CloneBytes(),
		Height: uint32(height),
	}
	url := "/v2/chainnotifier/register/blocks"
	c, err := openWebSocket(net.Alice, url, "POST", req, nil)
	require.Nil(ht.t, err, "websocket")
	defer func() {
		err := c.WriteMessage(websocket.CloseMessage, closeMsg)
		require.NoError(ht.t, err)
		_ = c.Close()
	}()

	msgChan := make(chan *chainrpc.BlockEpoch)
	errChan := make(chan error)
	timeout := time.After(defaultTimeout)

	// We want to read exactly one message.
	go func() {
		defer close(msgChan)

		_, msg, err := c.ReadMessage()
		if err != nil {
			errChan <- err
			return
		}

		// The chunked/streamed responses come wrapped in either a
		// {"result":{}} or {"error":{}} wrapper which we'll get rid of
		// here.
		msgStr := string(msg)
		if !strings.Contains(msgStr, "\"result\":") {
			errChan <- fmt.Errorf("invalid msg: %s", msgStr)
			return
		}
		msgStr = resultPattern.ReplaceAllString(msgStr, "${1}")

		// Make sure we can parse the unwrapped message into the
		// expected proto message.
		protoMsg := &chainrpc.BlockEpoch{}
		err = jsonpb.UnmarshalString(msgStr, protoMsg)
		if err != nil {
			errChan <- err
			return
		}

		select {
		case msgChan <- protoMsg:
		case <-timeout:
		}
	}()

	// Mine a block and make sure we get a message for it.
	blockHashes, err := net.Miner.Client.Generate(1)
	require.Nil(ht.t, err, "generate blocks")
	assert.Equal(ht.t, 1, len(blockHashes), "num blocks")
	select {
	case msg := <-msgChan:
		assert.Equal(
			ht.t, blockHashes[0].CloneBytes(), msg.Hash,
			"block hash",
		)

	case err := <-errChan:
		ht.t.Fatalf("Received error from WS: %v", err)

	case <-timeout:
		ht.t.Fatalf("Timeout before message was received")
	}
}

func wsTestCaseSubscriptionMacaroon(ht *harnessTest,
	net *lntest.NetworkHarness) {

	// Find out the current best block so we can subscribe to the next one.
	hash, height, err := net.Miner.Client.GetBestBlock()
	require.Nil(ht.t, err, "get best block")

	// Create a new subscription to get block epoch events.
	req := &chainrpc.BlockEpoch{
		Hash:   hash.CloneBytes(),
		Height: uint32(height),
	}
	url := "/v2/chainnotifier/register/blocks"

	// This time we send the macaroon in the special header
	// Sec-Websocket-Protocol which is the only header field available to
	// browsers when opening a WebSocket.
	mac, err := net.Alice.ReadMacaroon(
		net.Alice.AdminMacPath(), defaultTimeout,
	)
	require.NoError(ht.t, err, "read admin mac")
	macBytes, err := mac.MarshalBinary()
	require.NoError(ht.t, err, "marshal admin mac")

	customHeader := make(http.Header)
	customHeader.Set(lnrpc.HeaderWebSocketProtocol, fmt.Sprintf(
		"Grpc-Metadata-Macaroon+%s", hex.EncodeToString(macBytes),
	))
	c, err := openWebSocket(net.Alice, url, "POST", req, customHeader)
	require.Nil(ht.t, err, "websocket")
	defer func() {
		err := c.WriteMessage(websocket.CloseMessage, closeMsg)
		require.NoError(ht.t, err)
		_ = c.Close()
	}()

	msgChan := make(chan *chainrpc.BlockEpoch)
	errChan := make(chan error)
	timeout := time.After(defaultTimeout)

	// We want to read exactly one message.
	go func() {
		defer close(msgChan)

		_, msg, err := c.ReadMessage()
		if err != nil {
			errChan <- err
			return
		}

		// The chunked/streamed responses come wrapped in either a
		// {"result":{}} or {"error":{}} wrapper which we'll get rid of
		// here.
		msgStr := string(msg)
		if !strings.Contains(msgStr, "\"result\":") {
			errChan <- fmt.Errorf("invalid msg: %s", msgStr)
			return
		}
		msgStr = resultPattern.ReplaceAllString(msgStr, "${1}")

		// Make sure we can parse the unwrapped message into the
		// expected proto message.
		protoMsg := &chainrpc.BlockEpoch{}
		err = jsonpb.UnmarshalString(msgStr, protoMsg)
		if err != nil {
			errChan <- err
			return
		}

		select {
		case msgChan <- protoMsg:
		case <-timeout:
		}
	}()

	// Mine a block and make sure we get a message for it.
	blockHashes, err := net.Miner.Client.Generate(1)
	require.Nil(ht.t, err, "generate blocks")
	assert.Equal(ht.t, 1, len(blockHashes), "num blocks")
	select {
	case msg := <-msgChan:
		assert.Equal(
			ht.t, blockHashes[0].CloneBytes(), msg.Hash,
			"block hash",
		)

	case err := <-errChan:
		ht.t.Fatalf("Received error from WS: %v", err)

	case <-timeout:
		ht.t.Fatalf("Timeout before message was received")
	}
}

func wsTestCaseBiDirectionalSubscription(ht *harnessTest,
	net *lntest.NetworkHarness) {

	initialRequest := &lnrpc.ChannelAcceptResponse{}
	url := "/v1/channels/acceptor"

	// This time we send the macaroon in the special header
	// Sec-Websocket-Protocol which is the only header field available to
	// browsers when opening a WebSocket.
	mac, err := net.Alice.ReadMacaroon(
		net.Alice.AdminMacPath(), defaultTimeout,
	)
	require.NoError(ht.t, err, "read admin mac")
	macBytes, err := mac.MarshalBinary()
	require.NoError(ht.t, err, "marshal admin mac")

	customHeader := make(http.Header)
	customHeader.Set(lnrpc.HeaderWebSocketProtocol, fmt.Sprintf(
		"Grpc-Metadata-Macaroon+%s", hex.EncodeToString(macBytes),
	))
	conn, err := openWebSocket(
		net.Alice, url, "POST", initialRequest, customHeader,
	)
	require.Nil(ht.t, err, "websocket")
	defer func() {
		err := conn.WriteMessage(websocket.CloseMessage, closeMsg)
		require.NoError(ht.t, err)
		_ = conn.Close()
	}()

	msgChan := make(chan *lnrpc.ChannelAcceptResponse)
	errChan := make(chan error)
	done := make(chan struct{})
	timeout := time.After(defaultTimeout)

	// We want to read messages over and over again. We just accept any
	// channels that are opened.
	go func() {
		for {
			_, msg, err := conn.ReadMessage()
			if err != nil {
				errChan <- err
				return
			}

			// The chunked/streamed responses come wrapped in either
			// a {"result":{}} or {"error":{}} wrapper which we'll
			// get rid of here.
			msgStr := string(msg)
			if !strings.Contains(msgStr, "\"result\":") {
				errChan <- fmt.Errorf("invalid msg: %s", msgStr)
				return
			}
			msgStr = resultPattern.ReplaceAllString(msgStr, "${1}")

			// Make sure we can parse the unwrapped message into the
			// expected proto message.
			protoMsg := &lnrpc.ChannelAcceptRequest{}
			err = jsonpb.UnmarshalString(msgStr, protoMsg)
			if err != nil {
				errChan <- err
				return
			}

			// Send the response that we accept the channel.
			res := &lnrpc.ChannelAcceptResponse{
				Accept:        true,
				PendingChanId: protoMsg.PendingChanId,
			}
			resMsg, err := jsonMarshaler.MarshalToString(res)
			if err != nil {
				errChan <- err
				return
			}
			err = conn.WriteMessage(
				websocket.TextMessage, []byte(resMsg),
			)
			if err != nil {
				errChan <- err
				return
			}

			// Also send the message on our message channel to make
			// sure we count it as successful.
			msgChan <- res

			// Are we done or should there be more messages?
			select {
			case <-done:
				return
			default:
			}
		}
	}()

	// Before we start opening channels, make sure the two nodes are
	// connected.
	err = net.EnsureConnected(context.Background(), net.Alice, net.Bob)
	require.NoError(ht.t, err)

	// Open 3 channels to make sure multiple requests and responses can be
	// sent over the web socket.
	const numChannels = 3
	for i := 0; i < numChannels; i++ {
		openChannelAndAssert(
			context.Background(), ht, net, net.Bob, net.Alice,
			lntest.OpenChannelParams{
				Amt: 500000,
			},
		)

		select {
		case <-msgChan:
		case err := <-errChan:
			ht.t.Fatalf("Received error from WS: %v", err)

		case <-timeout:
			ht.t.Fatalf("Timeout before message was received")
		}
	}
	close(done)
}

// invokeGET calls the given URL with the GET method and appropriate macaroon
// header fields then tries to unmarshal the response into the given response
// proto message.
func invokeGET(node *lntest.HarnessNode, url string, resp proto.Message) error {
	_, rawResp, err := makeRequest(node, url, "GET", nil, nil)
	if err != nil {
		return err
	}

	return jsonpb.Unmarshal(bytes.NewReader(rawResp), resp)
}

// invokePOST calls the given URL with the POST method, request body and
// appropriate macaroon header fields then tries to unmarshal the response into
// the given response proto message.
func invokePOST(node *lntest.HarnessNode, url string, req,
	resp proto.Message) error {

	// Marshal the request to JSON using the jsonpb marshaler to get correct
	// field names.
	var buf bytes.Buffer
	if err := jsonMarshaler.Marshal(&buf, req); err != nil {
		return err
	}

	_, rawResp, err := makeRequest(node, url, "POST", &buf, nil)
	if err != nil {
		return err
	}

	return jsonpb.Unmarshal(bytes.NewReader(rawResp), resp)
}

// makeRequest calls the given URL with the given method, request body and
// appropriate macaroon header fields and returns the raw response body.
func makeRequest(node *lntest.HarnessNode, url, method string,
	request io.Reader, additionalHeaders http.Header) (http.Header, []byte,
	error) {

	// Assemble the full URL from the node's listening address then create
	// the request so we can set the macaroon on it.
	fullURL := fmt.Sprintf("https://%s%s", node.Cfg.RESTAddr(), url)
	req, err := http.NewRequest(method, fullURL, request)
	if err != nil {
		return nil, nil, err
	}
	if err := addAdminMacaroon(node, req.Header); err != nil {
		return nil, nil, err
	}
	for key, values := range additionalHeaders {
		for _, value := range values {
			req.Header.Add(key, value)
		}
	}

	// Do the actual call with the completed request object now.
	resp, err := restClient.Do(req)
	if err != nil {
		return nil, nil, err
	}
	defer func() { _ = resp.Body.Close() }()

	data, err := ioutil.ReadAll(resp.Body)
	return resp.Header, data, err
}

// openWebSocket opens a new WebSocket connection to the given URL with the
// appropriate macaroon headers and sends the request message over the socket.
func openWebSocket(node *lntest.HarnessNode, url, method string,
	req proto.Message, customHeader http.Header) (*websocket.Conn, error) {

	// Prepare our macaroon headers and assemble the full URL from the
	// node's listening address. WebSockets always work over GET so we need
	// to append the target request method as a query parameter.
	header := customHeader
	if header == nil {
		header = make(http.Header)
		if err := addAdminMacaroon(node, header); err != nil {
			return nil, err
		}
	}
	fullURL := fmt.Sprintf(
		"wss://%s%s?method=%s", node.Cfg.RESTAddr(), url, method,
	)
	conn, resp, err := webSocketDialer.Dial(fullURL, header)
	if err != nil {
		return nil, err
	}
	defer func() { _ = resp.Body.Close() }()

	// Send the given request message as the first message on the socket.
	reqMsg, err := jsonMarshaler.MarshalToString(req)
	if err != nil {
		return nil, err
	}
	err = conn.WriteMessage(websocket.TextMessage, []byte(reqMsg))
	if err != nil {
		return nil, err
	}

	return conn, nil
}

// addAdminMacaroon reads the admin macaroon from the node and appends it to
// the HTTP header fields.
func addAdminMacaroon(node *lntest.HarnessNode, header http.Header) error {
	mac, err := node.ReadMacaroon(node.AdminMacPath(), defaultTimeout)
	if err != nil {
		return err
	}
	macBytes, err := mac.MarshalBinary()
	if err != nil {
		return err
	}

	header.Set("Grpc-Metadata-Macaroon", hex.EncodeToString(macBytes))

	return nil
}