chanbackup: add new MultiFile struct for storing+updating Multis on disk

In this commit, we add a new MultiFile struct. We'll use this struct in
store the latest multi-channel backup on disk, swap it out atomically,
and finally extract+unpack the contents of the multi-file. The format
that's written to disk is the same as a regular Packed multi. The
contents of this new file are meant to be used to safely implement an
always up to date multi file on disk as a way for users to easily rsync
or fsnotiy (when it changes) the backup state of their channels.

We implement an atomic update and swap in the UpdateAndSwap. The method
uses relies on the underlying file system supporting an atomic rename
syscall. We first make a temporary backup file, write the latest
contents to that, then swap the temp file with the main file using
rename(2). This way, we ensure that we always have a single up to date
file, if the protocol aborts before the rename, then we can detect this,
remove the temp file, and attempt another swap.

chanbackup/backupfile.go

@@ -0,0 +1,160 @@
+package chanbackup
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/lightningnetwork/lnd/keychain"
+)
+
+const (
+	// DefaultBackupFileName is the default name of the auto updated static
+	// channel backup fie.
+	DefaultBackupFileName = "channel.backup"
+
+	// DefaultTempBackupFileName is the default name of the temporary SCB
+	// file that we'll use to atomically update the primary back up file
+	// when new channel are detected.
+	DefaultTempBackupFileName = "temp-dont-use.backup"
+)
+
+var (
+	// ErrNoBackupFileExists is returned if caller attempts to call
+	// UpdateAndSwap with the file name not set.
+	ErrNoBackupFileExists = fmt.Errorf("back up file name not set")
+
+	// ErrNoTempBackupFile is returned if caller attempts to call
+	// UpdateAndSwap with the temp back up file name not set.
+	ErrNoTempBackupFile = fmt.Errorf("temp backup file not set")
+)
+
+// MultiFile represents a file on disk that a caller can use to read the packed
+// multi backup into an unpacked one, and also atomically update the contents
+// on disk once new channels have been opened, and old ones closed. This struct
+// relies on an atomic file rename property which most widely use file systems
+// have.
+type MultiFile struct {
+	// fileName is the file name of the main back up file.
+	fileName string
+
+	// mainFile is an open handle to the main back up file.
+	mainFile *os.File
+
+	// tempFileName is the name of the file that we'll use to stage a new
+	// packed multi-chan backup, and the rename to the main back up file.
+	tempFileName string
+
+	// tempFile is an open handle to the temp back up file.
+	tempFile *os.File
+}
+
+// NewMultiFile create a new multi-file instance at the target location on the
+// file system.
+func NewMultiFile(fileName string) *MultiFile {
+
+	// We'll our temporary backup file in the very same directory as the
+	// main backup file.
+	backupFileDir := filepath.Dir(fileName)
+	tempFileName := filepath.Join(
+		backupFileDir, DefaultTempBackupFileName,
+	)
+
+	return &MultiFile{
+		fileName:     fileName,
+		tempFileName: tempFileName,
+	}
+}
+
+// UpdateAndSwap will attempt write a new temporary backup file to disk with
+// the newBackup encoded, then atomically swap (via rename) the old file for
+// the new file by updating the name of the new file to the old.
+func (b *MultiFile) UpdateAndSwap(newBackup PackedMulti) error {
+	// If the main backup file isn't set, then we can't proceed.
+	if b.fileName == "" {
+		return ErrNoBackupFileExists
+	}
+
+	// If the old back up file still exists, then we'll delete it before
+	// proceeding.
+	if _, err := os.Stat(b.tempFileName); err == nil {
+		log.Infof("Found old temp backup @ %v, removing before swap",
+			b.tempFileName)
+
+		err = os.Remove(b.tempFileName)
+		if err != nil {
+			return fmt.Errorf("unable to remove temp "+
+				"backup file: %v", err)
+		}
+	}
+
+	// Now that we know the staging area is clear, we'll create the new
+	// temporary back up file.
+	var err error
+	b.tempFile, err = os.Create(b.tempFileName)
+	if err != nil {
+		return err
+	}
+
+	// With the file created, we'll write the new packed multi backup and
+	// remove the temporary file all together once this method exits.
+	_, err = b.tempFile.Write([]byte(newBackup))
+	if err != nil {
+		return err
+	}
+	if err := b.tempFile.Sync(); err != nil {
+		return err
+	}
+	defer os.Remove(b.tempFileName)
+
+	log.Infof("Swapping old multi backup file from %v to %v",
+		b.tempFileName, b.fileName)
+
+	// Finally, we'll attempt to atomically rename the temporary file to
+	// the main back up file. If this succeeds, then we'll only have a
+	// single file on disk once this method exits.
+	return os.Rename(b.tempFileName, b.fileName)
+}
+
+// ExtractMulti attempts to extract the packed multi backup we currently point
+// to into an unpacked version. This method will fail if no backup file
+// currently exists as the specified location.
+func (b *MultiFile) ExtractMulti(keyChain keychain.KeyRing) (*Multi, error) {
+	var err error
+
+	// If the backup file isn't already set, then we'll attempt to open it
+	// anew.
+	if b.mainFile == nil {
+		// We'll return an error if the main file isn't currently set.
+		if b.fileName == "" {
+			return nil, ErrNoBackupFileExists
+		}
+
+		// Otherwise, we'll open the file to prep for reading the
+		// contents.
+		b.mainFile, err = os.Open(b.fileName)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Before we start to read the file, we'll ensure that the next read
+	// call will start from the front of the file.
+	_, err = b.mainFile.Seek(0, 0)
+	if err != nil {
+		return nil, err
+	}
+
+	// With our seek successful, we'll now attempt to read the contents of
+	// the entire file in one swoop.
+	multiBytes, err := ioutil.ReadAll(b.mainFile)
+	if err != nil {
+		return nil, err
+	}
+
+	// Finally, we'll attempt to unpack the file and return the unpack
+	// version to the caller.
+	packedMulti := PackedMulti(multiBytes)
+	return packedMulti.Unpack(keyChain)
+}

chanbackup/backupfile_test.go

@@ -0,0 +1,289 @@
+package chanbackup
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func makeFakePackedMulti() (PackedMulti, error) {
+	newPackedMulti := make([]byte, 50)
+	if _, err := rand.Read(newPackedMulti[:]); err != nil {
+		return nil, fmt.Errorf("unable to make test backup: %v", err)
+	}
+
+	return PackedMulti(newPackedMulti), nil
+}
+
+func assertBackupMatches(t *testing.T, filePath string,
+	currentBackup PackedMulti) {
+
+	t.Helper()
+
+	packedBackup, err := ioutil.ReadFile(filePath)
+	if err != nil {
+		t.Fatalf("unable to test file: %v", err)
+	}
+
+	if !bytes.Equal(packedBackup, currentBackup) {
+		t.Fatalf("backups don't match after first swap: "+
+			"expected %x got %x", packedBackup[:],
+			currentBackup)
+	}
+}
+
+func assertFileDeleted(t *testing.T, filePath string) {
+	t.Helper()
+
+	_, err := os.Stat(filePath)
+	if err == nil {
+		t.Fatalf("file %v still exists: ", filePath)
+	}
+}
+
+// TestUpdateAndSwap test that we're able to properly swap out old backups on
+// disk with new ones. Additionally, after a swap operation succeeds, then each
+// time we should only have the main backup file on disk, as the temporary file
+// has been removed.
+func TestUpdateAndSwap(t *testing.T) {
+	t.Parallel()
+
+	tempTestDir, err := ioutil.TempDir("", "")
+	if err != nil {
+		t.Fatalf("unable to make temp dir: %v", err)
+	}
+	defer os.Remove(tempTestDir)
+
+	testCases := []struct {
+		fileName     string
+		tempFileName string
+
+		oldTempExists bool
+
+		valid bool
+	}{
+		// Main file name is blank, should fail.
+		{
+			fileName: "",
+			valid:    false,
+		},
+
+		// Old temporary file still exists, should be removed. Only one
+		// file should remain.
+		{
+			fileName: filepath.Join(
+				tempTestDir, DefaultBackupFileName,
+			),
+			tempFileName: filepath.Join(
+				tempTestDir, DefaultTempBackupFileName,
+			),
+			oldTempExists: true,
+			valid:         true,
+		},
+
+		// Old temp doesn't exist, should swap out file, only a single
+		// file remains.
+		{
+			fileName: filepath.Join(
+				tempTestDir, DefaultBackupFileName,
+			),
+			tempFileName: filepath.Join(
+				tempTestDir, DefaultTempBackupFileName,
+			),
+			valid: true,
+		},
+	}
+	for i, testCase := range testCases {
+		// Ensure that all created files are removed at the end of the
+		// test case.
+		defer os.Remove(testCase.fileName)
+		defer os.Remove(testCase.tempFileName)
+
+		backupFile := NewMultiFile(testCase.fileName)
+
+		// To start with, we'll make a random byte slice that'll pose
+		// as our packed multi backup.
+		newPackedMulti, err := makeFakePackedMulti()
+		if err != nil {
+			t.Fatalf("unable to make test backup: %v", err)
+		}
+
+		// If the old temporary file is meant to exist, then we'll
+		// create it now as an empty file.
+		if testCase.oldTempExists {
+			_, err := os.Create(testCase.tempFileName)
+			if err != nil {
+				t.Fatalf("unable to create temp file: %v", err)
+			}
+
+			// TODO(roasbeef): mock out fs calls?
+		}
+
+		// With our backup created, we'll now attempt to swap out this
+		// backup, for the old one.
+		err = backupFile.UpdateAndSwap(PackedMulti(newPackedMulti))
+		switch {
+		// If this is a valid test case, and we failed, then we'll
+		// return an error.
+		case err != nil && testCase.valid:
+			t.Fatalf("#%v, unable to swap file: %v", i, err)
+
+		// If this is an invalid test case, and we passed it, then
+		// we'll return an error.
+		case err == nil && !testCase.valid:
+			t.Fatalf("#%v file swap should have failed: %v", i, err)
+		}
+
+		if !testCase.valid {
+			continue
+		}
+
+		// If we read out the file on disk, then it should match
+		// exactly what we wrote. The temp backup file should also be
+		// gone.
+		assertBackupMatches(t, testCase.fileName, newPackedMulti)
+		assertFileDeleted(t, testCase.tempFileName)
+
+		// Now that we know this is a valid test case, we'll make a new
+		// packed multi to swap out this current one.
+		newPackedMulti2, err := makeFakePackedMulti()
+		if err != nil {
+			t.Fatalf("unable to make test backup: %v", err)
+		}
+
+		// We'll then attempt to swap the old version for this new one.
+		err = backupFile.UpdateAndSwap(PackedMulti(newPackedMulti2))
+		if err != nil {
+			t.Fatalf("unable to swap file: %v", err)
+		}
+
+		// Once again, the file written on disk should have been
+		// properly swapped out with the new instance.
+		assertBackupMatches(t, testCase.fileName, newPackedMulti2)
+
+		// Additionally, we shouldn't be able to find the temp backup
+		// file on disk, as it should be deleted each time.
+		assertFileDeleted(t, testCase.tempFileName)
+	}
+}
+
+func assertMultiEqual(t *testing.T, a, b *Multi) {
+
+	if len(a.StaticBackups) != len(b.StaticBackups) {
+		t.Fatalf("expected %v backups, got %v", len(a.StaticBackups),
+			len(b.StaticBackups))
+	}
+
+	for i := 0; i < len(a.StaticBackups); i++ {
+		assertSingleEqual(t, a.StaticBackups[i], b.StaticBackups[i])
+	}
+}
+
+// TestExtractMulti tests that given a valid packed multi file on disk, we're
+// able to read it multiple times repeatedly.
+func TestExtractMulti(t *testing.T) {
+	t.Parallel()
+
+	keyRing := &mockKeyRing{}
+
+	// First, as prep, we'll create a single chan backup, then pack that
+	// fully into a multi backup.
+	channel, err := genRandomOpenChannelShell()
+	if err != nil {
+		t.Fatalf("unable to gen chan: %v", err)
+	}
+
+	singleBackup := NewSingle(channel, nil)
+
+	var b bytes.Buffer
+	unpackedMulti := Multi{
+		StaticBackups: []Single{singleBackup},
+	}
+	err = unpackedMulti.PackToWriter(&b, keyRing)
+	if err != nil {
+		t.Fatalf("unable to pack to writer: %v", err)
+	}
+
+	packedMulti := PackedMulti(b.Bytes())
+
+	// Finally, we'll make a new temporary file, then write out the packed
+	// multi directly to to it.
+	tempFile, err := ioutil.TempFile("", "")
+	if err != nil {
+		t.Fatalf("unable to create temp file: %v", err)
+	}
+	defer os.Remove(tempFile.Name())
+
+	_, err = tempFile.Write(packedMulti)
+	if err != nil {
+		t.Fatalf("unable to write temp file: %v", err)
+	}
+	if err := tempFile.Sync(); err != nil {
+		t.Fatalf("unable to sync temp file: %v", err)
+	}
+
+	testCases := []struct {
+		fileName string
+		pass     bool
+	}{
+		// Main file not read, file name not present.
+		{
+			fileName: "",
+			pass:     false,
+		},
+
+		// Main file not read, file name is there, but file doesn't
+		// exist.
+		{
+			fileName: "kek",
+			pass:     false,
+		},
+
+		// Main file not read, should be able to read multiple times.
+		{
+			fileName: tempFile.Name(),
+			pass:     true,
+		},
+	}
+	for i, testCase := range testCases {
+		// First, we'll make our backup file with the specified name.
+		backupFile := NewMultiFile(testCase.fileName)
+
+		// With our file made, we'll now attempt to read out the
+		// multi-file.
+		freshUnpackedMulti, err := backupFile.ExtractMulti(keyRing)
+		switch {
+		// If this is a valid test case, and we failed, then we'll
+		// return an error.
+		case err != nil && testCase.pass:
+			t.Fatalf("#%v, unable to extract file: %v", i, err)
+
+			// If this is an invalid test case, and we passed it, then
+			// we'll return an error.
+		case err == nil && !testCase.pass:
+			t.Fatalf("#%v file extraction should have "+
+				"failed: %v", i, err)
+		}
+
+		if !testCase.pass {
+			continue
+		}
+
+		// We'll now ensure that the unpacked multi we read is
+		// identical to the one we wrote out above.
+		assertMultiEqual(t, &unpackedMulti, freshUnpackedMulti)
+
+		// We should also be able to read the file again, as we have an
+		// existing handle to it.
+		freshUnpackedMulti, err = backupFile.ExtractMulti(keyRing)
+		if err != nil {
+			t.Fatalf("unable to unpack multi: %v", err)
+		}
+
+		assertMultiEqual(t, &unpackedMulti, freshUnpackedMulti)
+	}
+}