multi: wrap logger to request shutdown from signal on critical error

This commit adds a shutdown logger which will send a request for
shutdown on critical errors. It uses the signal package to request safe
shutdown of the daemon. Since we init our logs in config validation,
we add a started channel to the signal package to prevent the case where
we have a critical log after the ShutdownLogger has started but before
the daemon has started listening for intercepts. In this case, we just
ignore the shutdown request.

build/log_shutdown.go

@@ -0,0 +1,53 @@
+package build
+
+import (
+	"github.com/btcsuite/btclog"
+	"github.com/lightningnetwork/lnd/signal"
+)
+
+// ShutdownLogger wraps an existing logger with a shutdown function which will
+// be called on Critical/Criticalf to prompt shutdown.
+type ShutdownLogger struct {
+	btclog.Logger
+}
+
+// NewShutdownLogger creates a shutdown logger for the log provided which will
+// use the signal package to request shutdown on critical errors.
+func NewShutdownLogger(logger btclog.Logger) *ShutdownLogger {
+	return &ShutdownLogger{
+		Logger: logger,
+	}
+}
+
+// Criticalf formats message according to format specifier and writes to
+// log with LevelCritical. It will then call the shutdown logger's shutdown
+// function to prompt safe shutdown.
+//
+// Note: it is part of the btclog.Logger interface.
+func (s *ShutdownLogger) Criticalf(format string, params ...interface{}) {
+	s.Logger.Criticalf(format, params...)
+	s.shutdown()
+}
+
+// Critical formats message using the default formats for its operands
+// and writes to log with LevelCritical. It will then call the shutdown
+// logger's shutdown function to prompt safe shutdown.
+//
+// Note: it is part of the btclog.Logger interface.
+func (s *ShutdownLogger) Critical(v ...interface{}) {
+	s.Logger.Critical(v)
+	s.shutdown()
+}
+
+// shutdown checks whether we are listening for interrupts, since a shutdown
+// request to the signal package will block if it is not running, and requests
+// shutdown if possible.
+func (s *ShutdownLogger) shutdown() {
+	if !signal.Listening() {
+		s.Logger.Info("Request for shutdown ignored")
+		return
+	}
+
+	s.Logger.Info("Sending request for shutdown")
+	signal.RequestShutdown()
+}

build/logrotator.go

@@ -39,7 +39,10 @@ func NewRotatingLogWriter() *RotatingLogWriter {
 	logWriter := &LogWriter{}
 	backendLog := btclog.NewBackend(logWriter)
 	return &RotatingLogWriter{
-		GenSubLogger:     backendLog.Logger,
+		GenSubLogger: func(tag string) btclog.Logger {
+			logger := backendLog.Logger(tag)
+			return NewShutdownLogger(logger)
+		},
 		logWriter:        logWriter,
 		backendLog:       backendLog,
 		subsystemLoggers: SubLoggers{},

cmd/lncli/cmd_open_channel.go

@@ -414,7 +414,10 @@ func openChannelPsbt(ctx *cli.Context, client lnrpc.LightningClient,
 	if err != nil {
 		return fmt.Errorf("opening stream to server failed: %v", err)
 	}
-	signal.Intercept()
+
+	if err := signal.Intercept(); err != nil {
+		return err
+	}
 
 	// We also need to spawn a goroutine that reads from the server. This
 	// will copy the messages to the channel as long as they come in or add

cmd/lnd/main.go

@@ -19,7 +19,10 @@ func main() {
 	}
 
 	// Hook interceptor for os signals.
-	signal.Intercept()
+	if err := signal.Intercept(); err != nil {
+		_, _ = fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
 
 	// Call the "real" main in a nested manner so the defers will properly
 	// be executed in the case of a graceful shutdown.

mobile/bindings.go

@@ -54,7 +54,10 @@ func Start(extraArgs string, unlockerReady, rpcReady Callback) {
 	}
 
 	// Hook interceptor for os signals.
-	signal.Intercept()
+	if err := signal.Intercept(); err != nil {
+		_, _ = fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
 
 	// Set up channels that will be notified when the RPC servers are ready
 	// to accept calls.

signal/signal.go

@@ -6,8 +6,10 @@
 package signal
 
 import (
+	"errors"
 	"os"
 	"os/signal"
+	"sync/atomic"
 	"syscall"
 )
 
@@ -19,6 +21,10 @@ var (
 	// gracefully, similar to when receiving SIGINT.
 	shutdownRequestChannel = make(chan struct{})
 
+	// started indicates whether we have started our main interrupt handler.
+	// This field should be used atomically.
+	started int32
+
 	// quit is closed when instructing the main interrupt handler to exit.
 	quit = make(chan struct{})
 
@@ -26,8 +32,13 @@ var (
 	shutdownChannel = make(chan struct{})
 )
 
-// Intercept starts the interception of interrupt signals.
+// Intercept starts the interception of interrupt signals. Note that this
-func Intercept() {
+// function can only be called once.
+func Intercept() error {
+	if !atomic.CompareAndSwapInt32(&started, 0, 1) {
+		return errors.New("intercept already started")
+	}
+
 	signalsToCatch := []os.Signal{
 		os.Interrupt,
 		os.Kill,
@@ -37,6 +48,8 @@ func Intercept() {
 	}
 	signal.Notify(interruptChannel, signalsToCatch...)
 	go mainInterruptHandler()
+
+	return nil
 }
 
 // mainInterruptHandler listens for SIGINT (Ctrl+C) signals on the
@@ -85,6 +98,20 @@ func mainInterruptHandler() {
 	}
 }
 
+// Listening returns true if the main interrupt handler has been started, and
+// has not been killed.
+func Listening() bool {
+	// If our started field is not set, we are not yet listening for
+	// interrupts.
+	if atomic.LoadInt32(&started) != 1 {
+		return false
+	}
+
+	// If we have started our main goroutine, we check whether we have
+	// stopped it yet.
+	return Alive()
+}
+
 // Alive returns true if the main interrupt handler has not been killed.
 func Alive() bool {
 	select {