Merge pull request #1825 from Roasbeef/extra-gossip-message-data

channeldb+discovery: ensure we store, validate and propagate announcements with opaque data

chan_series.go

@@ -159,13 +159,14 @@ func makeNodeAnn(n *channeldb.LightningNode) (*lnwire.NodeAnnouncement, error) {
 		return nil, err
 	}
 	return &lnwire.NodeAnnouncement{
-		Signature: wireSig,
+		Signature:       wireSig,
-		Timestamp: uint32(n.LastUpdate.Unix()),
+		Timestamp:       uint32(n.LastUpdate.Unix()),
-		Addresses: n.Addresses,
+		Addresses:       n.Addresses,
-		NodeID:    n.PubKeyBytes,
+		NodeID:          n.PubKeyBytes,
-		Features:  n.Features.RawFeatureVector,
+		Features:        n.Features.RawFeatureVector,
-		RGBColor:  n.Color,
+		RGBColor:        n.Color,
-		Alias:     alias,
+		Alias:           alias,
+		ExtraOpaqueData: n.ExtraOpaqueData,
 	}, nil
 }
 
@@ -277,6 +278,7 @@ func (c *chanSeries) FetchChanUpdates(chain chainhash.Hash,
 			HtlcMinimumMsat: e1.MinHTLC,
 			BaseFee:         uint32(e1.FeeBaseMSat),
 			FeeRate:         uint32(e1.FeeProportionalMillionths),
+			ExtraOpaqueData: e1.ExtraOpaqueData,
 		}
 		chanUpdate.Signature, err = lnwire.NewSigFromRawSignature(e1.SigBytes)
 		if err != nil {
@@ -295,6 +297,7 @@ func (c *chanSeries) FetchChanUpdates(chain chainhash.Hash,
 			HtlcMinimumMsat: e2.MinHTLC,
 			BaseFee:         uint32(e2.FeeBaseMSat),
 			FeeRate:         uint32(e2.FeeProportionalMillionths),
+			ExtraOpaqueData: e1.ExtraOpaqueData,
 		}
 		chanUpdate.Signature, err = lnwire.NewSigFromRawSignature(e2.SigBytes)
 		if err != nil {

channeldb/error.go

@@ -94,3 +94,13 @@ var (
 	// to the log not having any recorded events.
 	ErrNoForwardingEvents = fmt.Errorf("no recorded forwarding events")
 )
+
+// ErrTooManyExtraOpaqueBytes creates an error which should be returned if the
+// caller attempts to write an announcement message which bares too many extra
+// opaque bytes. We limit this value in order to ensure that we don't waste
+// disk space due to nodes unnecessarily padding out their announcements with
+// garbage data.
+func ErrTooManyExtraOpaqueBytes(numBytes int) error {
+	return fmt.Errorf("max allowed number of opaque bytes is %v, received "+
+		"%v bytes", MaxAllowedExtraOpaqueBytes, numBytes)
+}

channeldb/graph.go

@@ -127,6 +127,14 @@ var (
 	nodeBloomKey = []byte("node-bloom")
 )
 
+const (
+	// MaxAllowedExtraOpaqueBytes is the largest amount of opaque bytes that
+	// we'll permit to be written to disk. We limit this as otherwise, it
+	// would be possible for a node to create a ton of updates and slowly
+	// fill our disk, and also waste bandwidth due to relaying.
+	MaxAllowedExtraOpaqueBytes = 10000
+)
+
 // ChannelGraph is a persistent, on-disk graph representation of the Lightning
 // Network. This struct can be used to implement path finding algorithms on top
 // of, and also to update a node's view based on information received from the
@@ -1707,6 +1715,14 @@ type LightningNode struct {
 	// Features is the list of protocol features supported by this node.
 	Features *lnwire.FeatureVector
 
+	// ExtraOpaqueData is the set of data that was appended to this
+	// message, some of which we may not actually know how to iterate or
+	// parse. By holding onto this data, we ensure that we're able to
+	// properly validate the set of signatures that cover these new fields,
+	// and ensure we're able to make upgrades to the network in a forwards
+	// compatible manner.
+	ExtraOpaqueData []byte
+
 	db *DB
 
 	// TODO(roasbeef): discovery will need storage to keep it's last IP
@@ -1991,6 +2007,14 @@ type ChannelEdgeInfo struct {
 	// the value output in the outpoint that created this channel.
 	Capacity btcutil.Amount
 
+	// ExtraOpaqueData is the set of data that was appended to this
+	// message, some of which we may not actually know how to iterate or
+	// parse. By holding onto this data, we ensure that we're able to
+	// properly validate the set of signatures that cover these new fields,
+	// and ensure we're able to make upgrades to the network in a forwards
+	// compatible manner.
+	ExtraOpaqueData []byte
+
 	db *DB
 }
 
@@ -2335,6 +2359,14 @@ type ChannelEdgePolicy struct {
 	// this pointer the channel graph can further be traversed.
 	Node *LightningNode
 
+	// ExtraOpaqueData is the set of data that was appended to this
+	// message, some of which we may not actually know how to iterate or
+	// parse. By holding onto this data, we ensure that we're able to
+	// properly validate the set of signatures that cover these new fields,
+	// and ensure we're able to make upgrades to the network in a forwards
+	// compatible manner.
+	ExtraOpaqueData []byte
+
 	db *DB
 }
 
@@ -2704,6 +2736,14 @@ func putLightningNode(nodeBucket *bolt.Bucket, aliasBucket *bolt.Bucket,
 		return err
 	}
 
+	if len(node.ExtraOpaqueData) > MaxAllowedExtraOpaqueBytes {
+		return ErrTooManyExtraOpaqueBytes(len(node.ExtraOpaqueData))
+	}
+	err = wire.WriteVarBytes(&b, 0, node.ExtraOpaqueData)
+	if err != nil {
+		return err
+	}
+
 	if err := aliasBucket.Put(nodePub, []byte(node.Alias)); err != nil {
 		return err
 	}
@@ -2828,6 +2868,18 @@ func deserializeLightningNode(r io.Reader) (LightningNode, error) {
 		return LightningNode{}, err
 	}
 
+	// We'll try and see if there are any opaque bytes left, if not, then
+	// we'll ignore the EOF error and return the node as is.
+	node.ExtraOpaqueData, err = wire.ReadVarBytes(
+		r, 0, MaxAllowedExtraOpaqueBytes, "blob",
+	)
+	switch {
+	case err == io.ErrUnexpectedEOF:
+	case err == io.EOF:
+	case err != nil:
+		return LightningNode{}, err
+	}
+
 	return node, nil
 }
 
@@ -2886,6 +2938,14 @@ func putChanEdgeInfo(edgeIndex *bolt.Bucket, edgeInfo *ChannelEdgeInfo, chanID [
 		return err
 	}
 
+	if len(edgeInfo.ExtraOpaqueData) > MaxAllowedExtraOpaqueBytes {
+		return ErrTooManyExtraOpaqueBytes(len(edgeInfo.ExtraOpaqueData))
+	}
+	err := wire.WriteVarBytes(&b, 0, edgeInfo.ExtraOpaqueData)
+	if err != nil {
+		return err
+	}
+
 	return edgeIndex.Put(chanID[:], b.Bytes())
 }
 
@@ -2963,6 +3023,18 @@ func deserializeChanEdgeInfo(r io.Reader) (ChannelEdgeInfo, error) {
 		return ChannelEdgeInfo{}, err
 	}
 
+	// We'll try and see if there are any opaque bytes left, if not, then
+	// we'll ignore the EOF error and return the edge as is.
+	edgeInfo.ExtraOpaqueData, err = wire.ReadVarBytes(
+		r, 0, MaxAllowedExtraOpaqueBytes, "blob",
+	)
+	switch {
+	case err == io.ErrUnexpectedEOF:
+	case err == io.EOF:
+	case err != nil:
+		return ChannelEdgeInfo{}, err
+	}
+
 	return edgeInfo, nil
 }
 
@@ -3011,6 +3083,13 @@ func putChanEdgePolicy(edges, nodes *bolt.Bucket, edge *ChannelEdgePolicy,
 		return err
 	}
 
+	if len(edge.ExtraOpaqueData) > MaxAllowedExtraOpaqueBytes {
+		return ErrTooManyExtraOpaqueBytes(len(edge.ExtraOpaqueData))
+	}
+	if err := wire.WriteVarBytes(&b, 0, edge.ExtraOpaqueData); err != nil {
+		return err
+	}
+
 	// Before we write out the new edge, we'll create a new entry in the
 	// update index in order to keep it fresh.
 	var indexKey [8 + 8]byte
@@ -3195,6 +3274,18 @@ func deserializeChanEdgePolicy(r io.Reader,
 			pub[:], err)
 	}
 
+	// We'll try and see if there are any opaque bytes left, if not, then
+	// we'll ignore the EOF error and return the edge as is.
+	edge.ExtraOpaqueData, err = wire.ReadVarBytes(
+		r, 0, MaxAllowedExtraOpaqueBytes, "blob",
+	)
+	switch {
+	case err == io.ErrUnexpectedEOF:
+	case err == io.EOF:
+	case err != nil:
+		return nil, err
+	}
+
 	edge.Node = &node
 	return edge, nil
 }

channeldb/graph_test.go

@@ -87,6 +87,7 @@ func TestNodeInsertionAndDeletion(t *testing.T) {
 		Alias:                "kek",
 		Features:             testFeatures,
 		Addresses:            testAddrs,
+		ExtraOpaqueData:      []byte("extra new data"),
 		db:                   db,
 	}
 	copy(node.PubKeyBytes[:], testPub.SerializeCompressed())
@@ -614,6 +615,11 @@ func assertEdgeInfoEqual(t *testing.T, e1 *ChannelEdgeInfo,
 		t.Fatalf("capacity doesn't match: %v vs %v", e1.Capacity,
 			e2.Capacity)
 	}
+
+	if !bytes.Equal(e1.ExtraOpaqueData, e2.ExtraOpaqueData) {
+		t.Fatalf("extra data doesn't match: %v vs %v",
+			e2.ExtraOpaqueData, e2.ExtraOpaqueData)
+	}
 }
 
 func TestEdgeInfoUpdates(t *testing.T) {
@@ -675,8 +681,9 @@ func TestEdgeInfoUpdates(t *testing.T) {
 			BitcoinSig1Bytes: testSig.Serialize(),
 			BitcoinSig2Bytes: testSig.Serialize(),
 		},
-		ChannelPoint: outpoint,
+		ChannelPoint:    outpoint,
-		Capacity:     1000,
+		Capacity:        1000,
+		ExtraOpaqueData: []byte("new unknown feature"),
 	}
 	copy(edgeInfo.NodeKey1Bytes[:], firstNode.PubKeyBytes[:])
 	copy(edgeInfo.NodeKey2Bytes[:], secondNode.PubKeyBytes[:])
@@ -697,8 +704,9 @@ func TestEdgeInfoUpdates(t *testing.T) {
 		MinHTLC:                   2342135,
 		FeeBaseMSat:               4352345,
 		FeeProportionalMillionths: 3452352,
-		Node: secondNode,
+		Node:            secondNode,
-		db:   db,
+		ExtraOpaqueData: []byte("new unknown feature2"),
+		db:              db,
 	}
 	edge2 := &ChannelEdgePolicy{
 		SigBytes:                  testSig.Serialize(),
@@ -709,8 +717,9 @@ func TestEdgeInfoUpdates(t *testing.T) {
 		MinHTLC:                   2342135,
 		FeeBaseMSat:               4352345,
 		FeeProportionalMillionths: 90392423,
-		Node: firstNode,
+		Node:            firstNode,
-		db:   db,
+		ExtraOpaqueData: []byte("new unknown feature1"),
+		db:              db,
 	}
 
 	// Next, insert both nodes into the database, they should both be
@@ -2461,6 +2470,10 @@ func compareNodes(a, b *LightningNode) error {
 		return fmt.Errorf("HaveNodeAnnouncement doesn't match: expected %#v, \n "+
 			"got %#v", a.HaveNodeAnnouncement, b.HaveNodeAnnouncement)
 	}
+	if !bytes.Equal(a.ExtraOpaqueData, b.ExtraOpaqueData) {
+		return fmt.Errorf("extra data doesn't match: %v vs %v",
+			a.ExtraOpaqueData, b.ExtraOpaqueData)
+	}
 
 	return nil
 }
@@ -2497,6 +2510,10 @@ func compareEdgePolicies(a, b *ChannelEdgePolicy) error {
 			"expected %v, got %v", a.FeeProportionalMillionths,
 			b.FeeProportionalMillionths)
 	}
+	if !bytes.Equal(a.ExtraOpaqueData, b.ExtraOpaqueData) {
+		return fmt.Errorf("extra data doesn't match: %v vs %v",
+			a.ExtraOpaqueData, b.ExtraOpaqueData)
+	}
 	if err := compareNodes(a.Node, b.Node); err != nil {
 		return err
 	}

discovery/gossiper.go

@@ -280,13 +280,14 @@ func (d *AuthenticatedGossiper) SynchronizeNode(syncPeer lnpeer.Peer) error {
 			return nil, err
 		}
 		return &lnwire.NodeAnnouncement{
-			Signature: wireSig,
+			Signature:       wireSig,
-			Timestamp: uint32(n.LastUpdate.Unix()),
+			Timestamp:       uint32(n.LastUpdate.Unix()),
-			Addresses: n.Addresses,
+			Addresses:       n.Addresses,
-			NodeID:    n.PubKeyBytes,
+			NodeID:          n.PubKeyBytes,
-			Features:  n.Features.RawFeatureVector,
+			Features:        n.Features.RawFeatureVector,
-			RGBColor:  n.Color,
+			RGBColor:        n.Color,
-			Alias:     alias,
+			Alias:           alias,
+			ExtraOpaqueData: n.ExtraOpaqueData,
 		}, nil
 	}
 
@@ -1618,6 +1619,7 @@ func (d *AuthenticatedGossiper) processNetworkAnnouncement(
 			AuthSigBytes:         msg.Signature.ToSignatureBytes(),
 			Features:             features,
 			Color:                msg.RGBColor,
+			ExtraOpaqueData:      msg.ExtraOpaqueData,
 		}
 
 		if err := d.cfg.Router.AddNode(node); err != nil {
@@ -1740,6 +1742,7 @@ func (d *AuthenticatedGossiper) processNetworkAnnouncement(
 			BitcoinKey2Bytes: msg.BitcoinKey2,
 			AuthProof:        proof,
 			Features:         featureBuf.Bytes(),
+			ExtraOpaqueData:  msg.ExtraOpaqueData,
 		}
 
 		// We will add the edge to the channel router. If the nodes
@@ -2009,6 +2012,7 @@ func (d *AuthenticatedGossiper) processNetworkAnnouncement(
 			MinHTLC:                   msg.HtlcMinimumMsat,
 			FeeBaseMSat:               lnwire.MilliSatoshi(msg.BaseFee),
 			FeeProportionalMillionths: lnwire.MilliSatoshi(msg.FeeRate),
+			ExtraOpaqueData:           msg.ExtraOpaqueData,
 		}
 
 		if err := d.cfg.Router.UpdateEdge(update); err != nil {
@@ -2477,6 +2481,7 @@ func (d *AuthenticatedGossiper) updateChannel(info *channeldb.ChannelEdgeInfo,
 		HtlcMinimumMsat: edge.MinHTLC,
 		BaseFee:         uint32(edge.FeeBaseMSat),
 		FeeRate:         uint32(edge.FeeProportionalMillionths),
+		ExtraOpaqueData: edge.ExtraOpaqueData,
 	}
 	chanUpdate.Signature, err = lnwire.NewSigFromRawSignature(edge.SigBytes)
 	if err != nil {
@@ -2518,13 +2523,14 @@ func (d *AuthenticatedGossiper) updateChannel(info *channeldb.ChannelEdgeInfo,
 	if info.AuthProof != nil {
 		chanID := lnwire.NewShortChanIDFromInt(info.ChannelID)
 		chanAnn = &lnwire.ChannelAnnouncement{
-			ShortChannelID: chanID,
+			ShortChannelID:  chanID,
-			NodeID1:        info.NodeKey1Bytes,
+			NodeID1:         info.NodeKey1Bytes,
-			NodeID2:        info.NodeKey2Bytes,
+			NodeID2:         info.NodeKey2Bytes,
-			ChainHash:      info.ChainHash,
+			ChainHash:       info.ChainHash,
-			BitcoinKey1:    info.BitcoinKey1Bytes,
+			BitcoinKey1:     info.BitcoinKey1Bytes,
-			Features:       lnwire.NewRawFeatureVector(),
+			Features:        lnwire.NewRawFeatureVector(),
-			BitcoinKey2:    info.BitcoinKey2Bytes,
+			BitcoinKey2:     info.BitcoinKey2Bytes,
+			ExtraOpaqueData: edge.ExtraOpaqueData,
 		}
 		chanAnn.NodeSig1, err = lnwire.NewSigFromRawSignature(
 			info.AuthProof.NodeSig1Bytes,

discovery/gossiper_test.go

@@ -372,10 +372,9 @@ func createAnnouncements(blockHeight uint32) (*annBatch, error) {
 }
 
 func createNodeAnnouncement(priv *btcec.PrivateKey,
-	timestamp uint32) (*lnwire.NodeAnnouncement,
+	timestamp uint32, extraBytes ...[]byte) (*lnwire.NodeAnnouncement, error) {
-	error) {
-	var err error
 
+	var err error
 	k := hex.EncodeToString(priv.Serialize())
 	alias, err := lnwire.NewNodeAlias("kek" + k[:10])
 	if err != nil {
@@ -389,6 +388,9 @@ func createNodeAnnouncement(priv *btcec.PrivateKey,
 		Features:  testFeatures,
 	}
 	copy(a.NodeID[:], priv.PubKey().SerializeCompressed())
+	if len(extraBytes) == 1 {
+		a.ExtraOpaqueData = extraBytes[0]
+	}
 
 	signer := mockSigner{priv}
 	sig, err := SignAnnouncement(&signer, priv.PubKey(), a)
@@ -405,8 +407,8 @@ func createNodeAnnouncement(priv *btcec.PrivateKey,
 }
 
 func createUpdateAnnouncement(blockHeight uint32, flags lnwire.ChanUpdateFlag,
-	nodeKey *btcec.PrivateKey, timestamp uint32) (*lnwire.ChannelUpdate,
+	nodeKey *btcec.PrivateKey, timestamp uint32,
-	error) {
+	extraBytes ...[]byte) (*lnwire.ChannelUpdate, error) {
 
 	var err error
 
@@ -421,6 +423,9 @@ func createUpdateAnnouncement(blockHeight uint32, flags lnwire.ChanUpdateFlag,
 		FeeRate:         uint32(prand.Int31()),
 		BaseFee:         uint32(prand.Int31()),
 	}
+	if len(extraBytes) == 1 {
+		a.ExtraOpaqueData = extraBytes[0]
+	}
 
 	pub := nodeKey.PubKey()
 	signer := mockSigner{nodeKey}
@@ -437,7 +442,8 @@ func createUpdateAnnouncement(blockHeight uint32, flags lnwire.ChanUpdateFlag,
 	return a, nil
 }
 
-func createRemoteChannelAnnouncement(blockHeight uint32) (*lnwire.ChannelAnnouncement, error) {
+func createRemoteChannelAnnouncement(blockHeight uint32,
+	extraBytes ...[]byte) (*lnwire.ChannelAnnouncement, error) {
 
 	var err error
 	a := &lnwire.ChannelAnnouncement{
@@ -452,6 +458,9 @@ func createRemoteChannelAnnouncement(blockHeight uint32) (*lnwire.ChannelAnnounc
 	copy(a.NodeID2[:], nodeKeyPub2.SerializeCompressed())
 	copy(a.BitcoinKey1[:], bitcoinKeyPub1.SerializeCompressed())
 	copy(a.BitcoinKey2[:], bitcoinKeyPub2.SerializeCompressed())
+	if len(extraBytes) == 1 {
+		a.ExtraOpaqueData = extraBytes[0]
+	}
 
 	pub := nodeKeyPriv1.PubKey()
 	signer := mockSigner{nodeKeyPriv1}
@@ -1736,7 +1745,6 @@ func TestDeDuplicatedAnnouncements(t *testing.T) {
 	// same channel ID. Adding this shouldn't cause an increase in the
 	// number of items as they should be de-duplicated.
 	ca2, err := createRemoteChannelAnnouncement(0)
-
 	if err != nil {
 		t.Fatalf("can't create remote channel announcement: %v", err)
 	}
@@ -2146,6 +2154,143 @@ func TestReceiveRemoteChannelUpdateFirst(t *testing.T) {
 	}
 }
 
+// TestExtraDataChannelAnnouncementValidation tests that we're able to properly
+// validate a ChannelAnnouncement that includes opaque bytes that we don't
+// currently know of.
+func TestExtraDataChannelAnnouncementValidation(t *testing.T) {
+	t.Parallel()
+
+	ctx, cleanup, err := createTestCtx(0)
+	if err != nil {
+		t.Fatalf("can't create context: %v", err)
+	}
+	defer cleanup()
+
+	remotePeer := &mockPeer{nodeKeyPriv1.PubKey(), nil, nil}
+
+	// We'll now create an announcement that contains an extra set of bytes
+	// that we don't know of ourselves, but should still include in the
+	// final signature check.
+	extraBytes := []byte("gotta validate this stil!")
+	ca, err := createRemoteChannelAnnouncement(0, extraBytes)
+	if err != nil {
+		t.Fatalf("can't create channel announcement: %v", err)
+	}
+
+	// We'll now send the announcement to the main gossiper. We should be
+	// able to validate this announcement to problem.
+	select {
+	case err = <-ctx.gossiper.ProcessRemoteAnnouncement(ca, remotePeer):
+	case <-time.After(2 * time.Second):
+		t.Fatal("did not process remote announcement")
+	}
+	if err != nil {
+		t.Fatalf("unable to process :%v", err)
+	}
+}
+
+// TestExtraDataChannelUpdateValidation tests that we're able to properly
+// validate a ChannelUpdate that includes opaque bytes that we don't currently
+// know of.
+func TestExtraDataChannelUpdateValidation(t *testing.T) {
+	t.Parallel()
+
+	ctx, cleanup, err := createTestCtx(0)
+	if err != nil {
+		t.Fatalf("can't create context: %v", err)
+	}
+	defer cleanup()
+
+	remotePeer := &mockPeer{nodeKeyPriv1.PubKey(), nil, nil}
+	timestamp := uint32(123456)
+
+	// In this scenario, we'll create two announcements, one regular
+	// channel announcement, and another channel update announcement, that
+	// has additional data that we won't be interpreting.
+	chanAnn, err := createRemoteChannelAnnouncement(0)
+	if err != nil {
+		t.Fatalf("unable to create chan ann: %v", err)
+	}
+	chanUpdAnn1, err := createUpdateAnnouncement(
+		0, 0, nodeKeyPriv1, timestamp,
+		[]byte("must also validate"),
+	)
+	if err != nil {
+		t.Fatalf("unable to create chan up: %v", err)
+	}
+	chanUpdAnn2, err := createUpdateAnnouncement(
+		0, 1, nodeKeyPriv2, timestamp,
+		[]byte("must also validate"),
+	)
+	if err != nil {
+		t.Fatalf("unable to create chan up: %v", err)
+	}
+
+	// We should be able to properly validate all three messages without
+	// any issue.
+	select {
+	case err = <-ctx.gossiper.ProcessRemoteAnnouncement(chanAnn, remotePeer):
+	case <-time.After(2 * time.Second):
+		t.Fatal("did not process remote announcement")
+	}
+	if err != nil {
+		t.Fatalf("unable to process announcement: %v", err)
+	}
+
+	select {
+	case err = <-ctx.gossiper.ProcessRemoteAnnouncement(chanUpdAnn1, remotePeer):
+	case <-time.After(2 * time.Second):
+		t.Fatal("did not process remote announcement")
+	}
+	if err != nil {
+		t.Fatalf("unable to process announcement: %v", err)
+	}
+
+	select {
+	case err = <-ctx.gossiper.ProcessRemoteAnnouncement(chanUpdAnn2, remotePeer):
+	case <-time.After(2 * time.Second):
+		t.Fatal("did not process remote announcement")
+	}
+	if err != nil {
+		t.Fatalf("unable to process announcement: %v", err)
+	}
+}
+
+// TestExtraDataNodeAnnouncementValidation tests that we're able to properly
+// validate a NodeAnnouncement that includes opaque bytes that we don't
+// currently know of.
+func TestExtraDataNodeAnnouncementValidation(t *testing.T) {
+	t.Parallel()
+
+	ctx, cleanup, err := createTestCtx(0)
+	if err != nil {
+		t.Fatalf("can't create context: %v", err)
+	}
+	defer cleanup()
+
+	remotePeer := &mockPeer{nodeKeyPriv1.PubKey(), nil, nil}
+	timestamp := uint32(123456)
+
+	// We'll create a node announcement that includes a set of opaque data
+	// which we don't know of, but will store anyway in order to ensure
+	// upgrades can flow smoothly in the future.
+	nodeAnn, err := createNodeAnnouncement(
+		nodeKeyPriv1, timestamp, []byte("gotta validate"),
+	)
+	if err != nil {
+		t.Fatalf("can't create node announcement: %v", err)
+	}
+
+	select {
+	case err = <-ctx.gossiper.ProcessRemoteAnnouncement(nodeAnn, remotePeer):
+	case <-time.After(2 * time.Second):
+		t.Fatal("did not process remote announcement")
+	}
+	if err != nil {
+		t.Fatalf("unable to process announcement: %v", err)
+	}
+}
+
 // mockPeer implements the lnpeer.Peer interface and is used to test the
 // gossiper's interaction with peers.
 type mockPeer struct {

discovery/utils.go

@@ -23,13 +23,14 @@ func CreateChanAnnouncement(chanProof *channeldb.ChannelAuthProof,
 	// authenticated channel announcement.
 	chanID := lnwire.NewShortChanIDFromInt(chanInfo.ChannelID)
 	chanAnn := &lnwire.ChannelAnnouncement{
-		ShortChannelID: chanID,
+		ShortChannelID:  chanID,
-		NodeID1:        chanInfo.NodeKey1Bytes,
+		NodeID1:         chanInfo.NodeKey1Bytes,
-		NodeID2:        chanInfo.NodeKey2Bytes,
+		NodeID2:         chanInfo.NodeKey2Bytes,
-		ChainHash:      chanInfo.ChainHash,
+		ChainHash:       chanInfo.ChainHash,
-		BitcoinKey1:    chanInfo.BitcoinKey1Bytes,
+		BitcoinKey1:     chanInfo.BitcoinKey1Bytes,
-		BitcoinKey2:    chanInfo.BitcoinKey2Bytes,
+		BitcoinKey2:     chanInfo.BitcoinKey2Bytes,
-		Features:       lnwire.NewRawFeatureVector(),
+		Features:        lnwire.NewRawFeatureVector(),
+		ExtraOpaqueData: chanInfo.ExtraOpaqueData,
 	}
 
 	var err error
@@ -76,6 +77,7 @@ func CreateChanAnnouncement(chanProof *channeldb.ChannelAuthProof,
 			HtlcMinimumMsat: e1.MinHTLC,
 			BaseFee:         uint32(e1.FeeBaseMSat),
 			FeeRate:         uint32(e1.FeeProportionalMillionths),
+			ExtraOpaqueData: e1.ExtraOpaqueData,
 		}
 		edge1Ann.Signature, err = lnwire.NewSigFromRawSignature(e1.SigBytes)
 		if err != nil {
@@ -92,6 +94,7 @@ func CreateChanAnnouncement(chanProof *channeldb.ChannelAuthProof,
 			HtlcMinimumMsat: e2.MinHTLC,
 			BaseFee:         uint32(e2.FeeBaseMSat),
 			FeeRate:         uint32(e2.FeeProportionalMillionths),
+			ExtraOpaqueData: e2.ExtraOpaqueData,
 		}
 		edge2Ann.Signature, err = lnwire.NewSigFromRawSignature(e2.SigBytes)
 		if err != nil {

lnwire/announcement_signatures.go

@@ -1,6 +1,9 @@
 package lnwire
 
-import "io"
+import (
+	"io"
+	"io/ioutil"
+)
 
 // AnnounceSignatures this is a direct message between two endpoints of a
 // channel and serves as an opt-in mechanism to allow the announcement of
@@ -30,6 +33,14 @@ type AnnounceSignatures struct {
 	// bitcoin key and and creating the reverse reference bitcoin_key ->
 	// node_key.
 	BitcoinSignature Sig
+
+	// ExtraOpaqueData is the set of data that was appended to this
+	// message, some of which we may not actually know how to iterate or
+	// parse. By holding onto this data, we ensure that we're able to
+	// properly validate the set of signatures that cover these new fields,
+	// and ensure we're able to make upgrades to the network in a forwards
+	// compatible manner.
+	ExtraOpaqueData []byte
 }
 
 // A compile time check to ensure AnnounceSignatures implements the
@@ -41,12 +52,29 @@ var _ Message = (*AnnounceSignatures)(nil)
 //
 // This is part of the lnwire.Message interface.
 func (a *AnnounceSignatures) Decode(r io.Reader, pver uint32) error {
-	return readElements(r,
+	err := readElements(r,
 		&a.ChannelID,
 		&a.ShortChannelID,
 		&a.NodeSignature,
 		&a.BitcoinSignature,
 	)
+	if err != nil {
+		return err
+	}
+
+	// Now that we've read out all the fields that we explicitly know of,
+	// we'll collect the remainder into the ExtraOpaqueData field. If there
+	// aren't any bytes, then we'll snip off the slice to avoid carrying
+	// around excess capacity.
+	a.ExtraOpaqueData, err = ioutil.ReadAll(r)
+	if err != nil {
+		return err
+	}
+	if len(a.ExtraOpaqueData) == 0 {
+		a.ExtraOpaqueData = nil
+	}
+
+	return nil
 }
 
 // Encode serializes the target AnnounceSignatures into the passed io.Writer
@@ -59,6 +87,7 @@ func (a *AnnounceSignatures) Encode(w io.Writer, pver uint32) error {
 		a.ShortChannelID,
 		a.NodeSignature,
 		a.BitcoinSignature,
+		a.ExtraOpaqueData,
 	)
 }
 
@@ -75,19 +104,5 @@ func (a *AnnounceSignatures) MsgType() MessageType {
 //
 // This is part of the lnwire.Message interface.
 func (a *AnnounceSignatures) MaxPayloadLength(pver uint32) uint32 {
-	var length uint32
+	return 65533
-
-	// ChannelID - 36 bytes
-	length += 36
-
-	// ShortChannelID - 8 bytes
-	length += 8
-
-	// NodeSignatures - 64 bytes
-	length += 64
-
-	// BitcoinSignatures - 64 bytes
-	length += 64
-
-	return length
 }

lnwire/channel_announcement.go

@@ -3,6 +3,7 @@ package lnwire
 import (
 	"bytes"
 	"io"
+	"io/ioutil"
 
 	"github.com/btcsuite/btcd/chaincfg/chainhash"
 )
@@ -48,6 +49,14 @@ type ChannelAnnouncement struct {
 	// multisig funding transaction output.
 	BitcoinKey1 [33]byte
 	BitcoinKey2 [33]byte
+
+	// ExtraOpaqueData is the set of data that was appended to this
+	// message, some of which we may not actually know how to iterate or
+	// parse. By holding onto this data, we ensure that we're able to
+	// properly validate the set of signatures that cover these new fields,
+	// and ensure we're able to make upgrades to the network in a forwards
+	// compatible manner.
+	ExtraOpaqueData []byte
 }
 
 // A compile time check to ensure ChannelAnnouncement implements the
@@ -59,7 +68,7 @@ var _ Message = (*ChannelAnnouncement)(nil)
 //
 // This is part of the lnwire.Message interface.
 func (a *ChannelAnnouncement) Decode(r io.Reader, pver uint32) error {
-	return readElements(r,
+	err := readElements(r,
 		&a.NodeSig1,
 		&a.NodeSig2,
 		&a.BitcoinSig1,
@@ -72,6 +81,23 @@ func (a *ChannelAnnouncement) Decode(r io.Reader, pver uint32) error {
 		&a.BitcoinKey1,
 		&a.BitcoinKey2,
 	)
+	if err != nil {
+		return err
+	}
+
+	// Now that we've read out all the fields that we explicitly know of,
+	// we'll collect the remainder into the ExtraOpaqueData field. If there
+	// aren't any bytes, then we'll snip off the slice to avoid carrying
+	// around excess capacity.
+	a.ExtraOpaqueData, err = ioutil.ReadAll(r)
+	if err != nil {
+		return err
+	}
+	if len(a.ExtraOpaqueData) == 0 {
+		a.ExtraOpaqueData = nil
+	}
+
+	return nil
 }
 
 // Encode serializes the target ChannelAnnouncement into the passed io.Writer
@@ -91,6 +117,7 @@ func (a *ChannelAnnouncement) Encode(w io.Writer, pver uint32) error {
 		a.NodeID2,
 		a.BitcoinKey1,
 		a.BitcoinKey2,
+		a.ExtraOpaqueData,
 	)
 }
 
@@ -107,42 +134,7 @@ func (a *ChannelAnnouncement) MsgType() MessageType {
 //
 // This is part of the lnwire.Message interface.
 func (a *ChannelAnnouncement) MaxPayloadLength(pver uint32) uint32 {
-	var length uint32
+	return 65533
-
-	// NodeSig1 - 64 bytes
-	length += 64
-
-	// NodeSig2 - 64 bytes
-	length += 64
-
-	// BitcoinSig1 - 64 bytes
-	length += 64
-
-	// BitcoinSig2 - 64 bytes
-	length += 64
-
-	// Features  (max possible features)
-	length += 65096
-
-	// ChainHash - 32 bytes
-	length += 32
-
-	// ShortChannelID - 8 bytes
-	length += 8
-
-	// NodeID1 - 33 bytes
-	length += 33
-
-	// NodeID2 - 33 bytes
-	length += 33
-
-	// BitcoinKey1 - 33 bytes
-	length += 33
-
-	// BitcoinKey2 - 33 bytes
-	length += 33
-
-	return length
 }
 
 // DataToSign is used to retrieve part of the announcement message which should
@@ -158,6 +150,7 @@ func (a *ChannelAnnouncement) DataToSign() ([]byte, error) {
 		a.NodeID2,
 		a.BitcoinKey1,
 		a.BitcoinKey2,
+		a.ExtraOpaqueData,
 	)
 	if err != nil {
 		return nil, err

lnwire/channel_update.go

@@ -3,6 +3,7 @@ package lnwire
 import (
 	"bytes"
 	"io"
+	"io/ioutil"
 
 	"github.com/btcsuite/btcd/chaincfg/chainhash"
 )
@@ -73,6 +74,14 @@ type ChannelUpdate struct {
 	// FeeRate is the fee rate that will be charged per millionth of a
 	// satoshi.
 	FeeRate uint32
+
+	// ExtraOpaqueData is the set of data that was appended to this
+	// message, some of which we may not actually know how to iterate or
+	// parse. By holding onto this data, we ensure that we're able to
+	// properly validate the set of signatures that cover these new fields,
+	// and ensure we're able to make upgrades to the network in a forwards
+	// compatible manner.
+	ExtraOpaqueData []byte
 }
 
 // A compile time check to ensure ChannelUpdate implements the lnwire.Message
@@ -84,7 +93,7 @@ var _ Message = (*ChannelUpdate)(nil)
 //
 // This is part of the lnwire.Message interface.
 func (a *ChannelUpdate) Decode(r io.Reader, pver uint32) error {
-	return readElements(r,
+	err := readElements(r,
 		&a.Signature,
 		a.ChainHash[:],
 		&a.ShortChannelID,
@@ -95,6 +104,23 @@ func (a *ChannelUpdate) Decode(r io.Reader, pver uint32) error {
 		&a.BaseFee,
 		&a.FeeRate,
 	)
+	if err != nil {
+		return err
+	}
+
+	// Now that we've read out all the fields that we explicitly know of,
+	// we'll collect the remainder into the ExtraOpaqueData field. If there
+	// aren't any bytes, then we'll snip off the slice to avoid carrying
+	// around excess capacity.
+	a.ExtraOpaqueData, err = ioutil.ReadAll(r)
+	if err != nil {
+		return err
+	}
+	if len(a.ExtraOpaqueData) == 0 {
+		a.ExtraOpaqueData = nil
+	}
+
+	return nil
 }
 
 // Encode serializes the target ChannelUpdate into the passed io.Writer
@@ -112,6 +138,7 @@ func (a *ChannelUpdate) Encode(w io.Writer, pver uint32) error {
 		a.HtlcMinimumMsat,
 		a.BaseFee,
 		a.FeeRate,
+		a.ExtraOpaqueData,
 	)
 }
 
@@ -128,36 +155,7 @@ func (a *ChannelUpdate) MsgType() MessageType {
 //
 // This is part of the lnwire.Message interface.
 func (a *ChannelUpdate) MaxPayloadLength(pver uint32) uint32 {
-	var length uint32
+	return 65533
-
-	// Signature - 64 bytes
-	length += 64
-
-	// ChainHash - 64 bytes
-	length += 32
-
-	// ShortChannelID - 8 bytes
-	length += 8
-
-	// Timestamp - 4 bytes
-	length += 4
-
-	// Flags - 2 bytes
-	length += 2
-
-	// Expiry - 2 bytes
-	length += 2
-
-	// HtlcMinimumMstat - 8 bytes
-	length += 8
-
-	// FeeBaseMstat - 4 bytes
-	length += 4
-
-	// FeeProportionalMillionths - 4 bytes
-	length += 4
-
-	return length
 }
 
 // DataToSign is used to retrieve part of the announcement message which should
@@ -175,6 +173,7 @@ func (a *ChannelUpdate) DataToSign() ([]byte, error) {
 		a.HtlcMinimumMsat,
 		a.BaseFee,
 		a.FeeRate,
+		a.ExtraOpaqueData,
 	)
 	if err != nil {
 		return nil, err

lnwire/lnwire_test.go

@@ -537,6 +537,17 @@ func TestLightningWireProtocol(t *testing.T) {
 				return
 			}
 
+			numExtraBytes := r.Int31n(1000)
+			if numExtraBytes > 0 {
+				req.ExtraOpaqueData = make([]byte, numExtraBytes)
+				_, err := r.Read(req.ExtraOpaqueData[:])
+				if err != nil {
+					t.Fatalf("unable to generate opaque "+
+						"bytes: %v", err)
+					return
+				}
+			}
+
 			v[0] = reflect.ValueOf(req)
 		},
 		MsgNodeAnnouncement: func(v []reflect.Value, r *rand.Rand) {
@@ -574,6 +585,17 @@ func TestLightningWireProtocol(t *testing.T) {
 				t.Fatalf("unable to generate addresses: %v", err)
 			}
 
+			numExtraBytes := r.Int31n(1000)
+			if numExtraBytes > 0 {
+				req.ExtraOpaqueData = make([]byte, numExtraBytes)
+				_, err := r.Read(req.ExtraOpaqueData[:])
+				if err != nil {
+					t.Fatalf("unable to generate opaque "+
+						"bytes: %v", err)
+					return
+				}
+			}
+
 			v[0] = reflect.ValueOf(req)
 		},
 		MsgChannelUpdate: func(v []reflect.Value, r *rand.Rand) {
@@ -598,6 +620,17 @@ func TestLightningWireProtocol(t *testing.T) {
 				return
 			}
 
+			numExtraBytes := r.Int31n(1000)
+			if numExtraBytes > 0 {
+				req.ExtraOpaqueData = make([]byte, numExtraBytes)
+				_, err := r.Read(req.ExtraOpaqueData[:])
+				if err != nil {
+					t.Fatalf("unable to generate opaque "+
+						"bytes: %v", err)
+					return
+				}
+			}
+
 			v[0] = reflect.ValueOf(req)
 		},
 		MsgAnnounceSignatures: func(v []reflect.Value, r *rand.Rand) {
@@ -623,6 +656,17 @@ func TestLightningWireProtocol(t *testing.T) {
 				return
 			}
 
+			numExtraBytes := r.Int31n(1000)
+			if numExtraBytes > 0 {
+				req.ExtraOpaqueData = make([]byte, numExtraBytes)
+				_, err := r.Read(req.ExtraOpaqueData[:])
+				if err != nil {
+					t.Fatalf("unable to generate opaque "+
+						"bytes: %v", err)
+					return
+				}
+			}
+
 			v[0] = reflect.ValueOf(req)
 		},
 		MsgChannelReestablish: func(v []reflect.Value, r *rand.Rand) {

lnwire/node_announcement.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"image/color"
 	"io"
+	"io/ioutil"
 	"net"
 	"unicode/utf8"
 )
@@ -83,6 +84,14 @@ type NodeAnnouncement struct {
 	// Address includes two specification fields: 'ipv6' and 'port' on
 	// which the node is accepting incoming connections.
 	Addresses []net.Addr
+
+	// ExtraOpaqueData is the set of data that was appended to this
+	// message, some of which we may not actually know how to iterate or
+	// parse. By holding onto this data, we ensure that we're able to
+	// properly validate the set of signatures that cover these new fields,
+	// and ensure we're able to make upgrades to the network in a forwards
+	// compatible manner.
+	ExtraOpaqueData []byte
 }
 
 // UpdateNodeAnnAddrs is a functional option that allows updating the addresses
@@ -102,7 +111,7 @@ var _ Message = (*NodeAnnouncement)(nil)
 //
 // This is part of the lnwire.Message interface.
 func (a *NodeAnnouncement) Decode(r io.Reader, pver uint32) error {
-	return readElements(r,
+	err := readElements(r,
 		&a.Signature,
 		&a.Features,
 		&a.Timestamp,
@@ -111,6 +120,23 @@ func (a *NodeAnnouncement) Decode(r io.Reader, pver uint32) error {
 		a.Alias[:],
 		&a.Addresses,
 	)
+	if err != nil {
+		return err
+	}
+
+	// Now that we've read out all the fields that we explicitly know of,
+	// we'll collect the remainder into the ExtraOpaqueData field. If there
+	// aren't any bytes, then we'll snip off the slice to avoid carrying
+	// around excess capacity.
+	a.ExtraOpaqueData, err = ioutil.ReadAll(r)
+	if err != nil {
+		return err
+	}
+	if len(a.ExtraOpaqueData) == 0 {
+		a.ExtraOpaqueData = nil
+	}
+
+	return nil
 }
 
 // Encode serializes the target NodeAnnouncement into the passed io.Writer
@@ -125,6 +151,7 @@ func (a *NodeAnnouncement) Encode(w io.Writer, pver uint32) error {
 		a.RGBColor,
 		a.Alias[:],
 		a.Addresses,
+		a.ExtraOpaqueData,
 	)
 }
 
@@ -156,12 +183,11 @@ func (a *NodeAnnouncement) DataToSign() ([]byte, error) {
 		a.RGBColor,
 		a.Alias[:],
 		a.Addresses,
+		a.ExtraOpaqueData,
 	)
 	if err != nil {
 		return nil, err
 	}
 
-	// TODO(roasbeef): also capture the excess bytes in msg padded out?
-
 	return w.Bytes(), nil
 }

lnwire/onion_error_test.go

@@ -6,6 +6,8 @@ import (
 	"encoding/binary"
 	"reflect"
 	"testing"
+
+	"github.com/davecgh/go-spew/spew"
 )
 
 var (
@@ -66,8 +68,8 @@ func TestEncodeDecodeCode(t *testing.T) {
 		}
 
 		if !reflect.DeepEqual(failure1, failure2) {
-			t.Fatalf("failure message are different, failure "+
+			t.Fatalf("expected %v, got %v", spew.Sdump(failure1),
-				"code(%v)", failure1.Code())
+				spew.Sdump(failure2))
 		}
 	}
 }

server.go

@@ -2988,6 +2988,7 @@ func createChannelUpdate(info *channeldb.ChannelEdgeInfo,
 		HtlcMinimumMsat: policy.MinHTLC,
 		BaseFee:         uint32(policy.FeeBaseMSat),
 		FeeRate:         uint32(policy.FeeProportionalMillionths),
+		ExtraOpaqueData: policy.ExtraOpaqueData,
 	}
 
 	var err error