chainntnfs: rewind chain on missed disconnected blocks

If the chain backend misses telling the notifier about a series of disconnected blocks, the notifier is now able to disconnect the tip to its new best block.

chainntnfs/bitcoindnotify/bitcoind.go

@@ -327,23 +327,24 @@ out:
 
 			case chain.BlockDisconnected:
 				if item.Height != b.bestBlock.Height {
-					chainntnfs.Log.Warnf("Received blocks "+
+					chainntnfs.Log.Infof("Missed disconnected" +
-						"out of order: current height="+
+						"blocks, attempting to catch up")
-						"%d, disconnected height=%d",
-						bestHeight, item.Height)
-					continue
 				}
 
-				chainntnfs.Log.Infof("Block disconnected from "+
+				newBestBlock, err := chainntnfs.RewindChain(
-					"main chain: height=%v, sha=%v",
+					b.chainConn, b.txConfNotifier,
-					item.Height, item.Hash)
+					b.bestBlock, item.Height-1,
-
+				)
-				err := b.txConfNotifier.DisconnectTip(
-					uint32(item.Height))
 				if err != nil {
-					chainntnfs.Log.Error(err)
+					chainntnfs.Log.Errorf("Unable to rewind chain "+
+						"from height %d to height %d: %v",
+						b.bestBlock.Height, item.Height-1, err)
 				}
 
+				// Set the bestBlock here in case a chain
+				// rewind partially completed.
+				b.bestBlock = newBestBlock
+
 			case chain.RelevantTx:
 				b.handleRelevantTx(item, b.bestBlock.Height)
 			}

chainntnfs/btcdnotify/btcd.go

@@ -397,20 +397,24 @@ out:
 			}
 
 			if update.blockHeight != b.bestBlock.Height {
-				chainntnfs.Log.Warnf("Received blocks out of order: "+
+				chainntnfs.Log.Infof("Missed disconnected" +
-					"current height=%d, disconnected height=%d",
+					"blocks, attempting to catch up")
-					currentHeight, update.blockHeight)
-				continue
 			}
 
-			chainntnfs.Log.Infof("Block disconnected from main chain: "+
+			newBestBlock, err := chainntnfs.RewindChain(
-				"height=%v, sha=%v", update.blockHeight, update.blockHash)
+				b.chainConn, b.txConfNotifier, b.bestBlock,
-
+				update.blockHeight-1,
-			err := b.txConfNotifier.DisconnectTip(uint32(update.blockHeight))
+			)
 			if err != nil {
-				chainntnfs.Log.Error(err)
+				chainntnfs.Log.Errorf("Unable to rewind chain "+
+					"from height %d to height %d: %v",
+					b.bestBlock.Height, update.blockHeight-1, err)
 			}
 
+			// Set the bestBlock here in case a chain rewind
+			// partially completed.
+			b.bestBlock = newBestBlock
+
 		// NOTE: we currently only use txUpdates for mempool spends and
 		// rescan spends. It might get removed entirely in the future.
 		case item := <-b.txUpdates.ChanOut():

chainntnfs/interface.go

@@ -344,6 +344,40 @@ func GetClientMissedBlocks(chainConn ChainConn, clientBestBlock *BlockEpoch,
 	return missedBlocks, nil
 }
 
+// RewindChain handles internal state updates for the notifier's TxConfNotifier
+// It has no effect if given a height greater than or equal to our current best
+// known height. It returns the new best block for the notifier.
+func RewindChain(chainConn ChainConn, txConfNotifier *TxConfNotifier,
+	currBestBlock BlockEpoch, targetHeight int32) (BlockEpoch, error) {
+
+	newBestBlock := BlockEpoch{
+		Height: currBestBlock.Height,
+		Hash:   currBestBlock.Hash,
+	}
+
+	for height := currBestBlock.Height; height > targetHeight; height-- {
+		hash, err := chainConn.GetBlockHash(int64(height - 1))
+		if err != nil {
+			return newBestBlock, fmt.Errorf("unable to "+
+				"find blockhash for disconnected height=%d: %v",
+				height, err)
+		}
+
+		Log.Infof("Block disconnected from main chain: "+
+			"height=%v, sha=%v", height, newBestBlock.Hash)
+
+		err = txConfNotifier.DisconnectTip(uint32(height))
+		if err != nil {
+			return newBestBlock, fmt.Errorf("unable to "+
+				" disconnect tip for height=%d: %v",
+				height, err)
+		}
+		newBestBlock.Height = height - 1
+		newBestBlock.Hash = hash
+	}
+	return newBestBlock, nil
+}
+
 // getMissedBlocks returns a slice of blocks: [startingHeight, endingHeight)
 // fetched from the chain.
 func getMissedBlocks(chainConn ChainConn, startingHeight,

chainntnfs/neutrinonotify/neutrino.go

@@ -365,6 +365,7 @@ func (n *NeutrinoNotifier) notificationDispatcher() {
 							"to update rescan: %v",
 							err)
 					}
+
 				}()
 
 			case *blockEpochRegistration:
@@ -417,24 +418,37 @@ func (n *NeutrinoNotifier) notificationDispatcher() {
 			}
 
 			n.heightMtx.Lock()
-			if update.height != n.bestHeight {
+			if update.height != uint32(n.bestHeight) {
-				chainntnfs.Log.Warnf("Received blocks out of order: "+
+				chainntnfs.Log.Infof("Missed disconnected" +
-					"current height=%d, disconnected height=%d",
+					"blocks, attempting to catch up")
-					n.bestHeight, update.height)
-				n.heightMtx.Unlock()
-				continue
 			}
 
-			n.bestHeight = update.height - 1
+			header, err := n.p2pNode.BlockHeaders.FetchHeaderByHeight(
-			n.heightMtx.Unlock()
+				n.bestHeight,
-
+			)
-			chainntnfs.Log.Infof("Block disconnected from main chain: "+
-				"height=%v, sha=%v", update.height, update.hash)
-
-			err := n.txConfNotifier.DisconnectTip(update.height)
 			if err != nil {
-				chainntnfs.Log.Error(err)
+				chainntnfs.Log.Errorf("Unable to fetch header"+
+					"for height %d: %v", n.bestHeight, err)
 			}
+			hash := header.BlockHash()
+			notifierBestBlock := chainntnfs.BlockEpoch{
+				Height: int32(n.bestHeight),
+				Hash:   &hash,
+			}
+			newBestBlock, err := chainntnfs.RewindChain(
+				n.chainConn, n.txConfNotifier, notifierBestBlock,
+				int32(update.height-1),
+			)
+			if err != nil {
+				chainntnfs.Log.Errorf("Unable to rewind chain "+
+					"from height %d to height %d: %v",
+					n.bestHeight, update.height-1, err)
+			}
+
+			// Set the bestHeight here in case a chain rewind
+			// partially completed.
+			n.bestHeight = uint32(newBestBlock.Height)
+			n.heightMtx.Unlock()
 
 		case err := <-n.rescanErr:
 			chainntnfs.Log.Errorf("Error during rescan: %v", err)