kvdb+etcd: enable channeldb testing with both bdb/etcd

channeldb/channel_test.go

@@ -98,13 +98,22 @@ func makeTestDB() (*DB, func(), error) {
 	}
 
 	// Next, create channeldb for the first time.
-	cdb, err := Open(tempDirName, OptionClock(testClock))
+	backend, backendCleanup, err := kvdb.GetTestBackend(tempDirName, "cdb")
 	if err != nil {
+		backendCleanup()
+		return nil, nil, err
+	}
+
+	cdb, err := CreateWithBackend(backend, OptionClock(testClock))
+	if err != nil {
+		backendCleanup()
+		os.RemoveAll(tempDirName)
 		return nil, nil, err
 	}
 
 	cleanUp := func() {
 		cdb.Close()
+		backendCleanup()
 		os.RemoveAll(tempDirName)
 	}
 

channeldb/db.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"net"
 	"os"
-	"path/filepath"
 	"time"
 
 	"github.com/btcsuite/btcd/btcec"
@@ -154,15 +153,32 @@ type DB struct {
 	dryRun bool
 }
 
-// Open opens an existing channeldb. Any necessary schemas migrations due to
-// updates will take place as necessary.
+// Open opens or creates channeldb. Any necessary schemas migrations due
+// to updates will take place as necessary.
+// TODO(bhandras): deprecate this function.
 func Open(dbPath string, modifiers ...OptionModifier) (*DB, error) {
-	path := filepath.Join(dbPath, dbName)
+	opts := DefaultOptions()
+	for _, modifier := range modifiers {
+		modifier(&opts)
+	}
 
-	if !fileExists(path) {
-		if err := createChannelDB(dbPath); err != nil {
-			return nil, err
-		}
+	backend, err := kvdb.GetBoltBackend(dbPath, dbName, opts.NoFreelistSync)
+	if err != nil {
+		return nil, err
+	}
+
+	db, err := CreateWithBackend(backend, modifiers...)
+	if err == nil {
+		db.dbPath = dbPath
+	}
+	return db, err
+}
+
+// CreateWithBackend creates channeldb instance using the passed kvdb.Backend.
+// Any necessary schemas migrations due to updates will take place as necessary.
+func CreateWithBackend(backend kvdb.Backend, modifiers ...OptionModifier) (*DB, error) {
+	if err := initChannelDB(backend); err != nil {
+		return nil, err
 	}
 
 	opts := DefaultOptions()
@@ -170,16 +186,8 @@ func Open(dbPath string, modifiers ...OptionModifier) (*DB, error) {
 		modifier(&opts)
 	}
 
-	// Specify bbolt freelist options to reduce heap pressure in case the
-	// freelist grows to be very large.
-	bdb, err := kvdb.Open(kvdb.BoltBackendName, path, opts.NoFreelistSync)
-	if err != nil {
-		return nil, err
-	}
-
 	chanDB := &DB{
-		Backend: bdb,
-		dbPath:  dbPath,
+		Backend: backend,
 		clock:   opts.clock,
 		dryRun:  opts.dryRun,
 	}
@@ -189,7 +197,7 @@ func Open(dbPath string, modifiers ...OptionModifier) (*DB, error) {
 
 	// Synchronize the version of database and apply migrations if needed.
 	if err := chanDB.syncVersions(dbVersions); err != nil {
-		bdb.Close()
+		backend.Close()
 		return nil, err
 	}
 
@@ -251,20 +259,15 @@ func (d *DB) Wipe() error {
 // the case that the target path has not yet been created or doesn't yet exist,
 // then the path is created. Additionally, all required top-level buckets used
 // within the database are created.
-func createChannelDB(dbPath string) error {
-	if !fileExists(dbPath) {
-		if err := os.MkdirAll(dbPath, 0700); err != nil {
-			return err
+func initChannelDB(db kvdb.Backend) error {
+	err := kvdb.Update(db, func(tx kvdb.RwTx) error {
+		meta := &Meta{}
+		// Check if DB is already initialized.
+		err := fetchMeta(meta, tx)
+		if err == nil {
+			return nil
 		}
-	}
 
-	path := filepath.Join(dbPath, dbName)
-	bdb, err := kvdb.Create(kvdb.BoltBackendName, path, true)
-	if err != nil {
-		return err
-	}
-
-	err = kvdb.Update(bdb, func(tx kvdb.RwTx) error {
 		if _, err := tx.CreateTopLevelBucket(openChannelBucket); err != nil {
 			return err
 		}
@@ -331,16 +334,14 @@ func createChannelDB(dbPath string) error {
 			return err
 		}
 
-		meta := &Meta{
-			DbVersionNumber: getLatestDBVersion(dbVersions),
-		}
+		meta.DbVersionNumber = getLatestDBVersion(dbVersions)
 		return putMeta(meta, tx)
 	})
 	if err != nil {
-		return fmt.Errorf("unable to create new channeldb")
+		return fmt.Errorf("unable to create new channeldb: %v", err)
 	}
 
-	return bdb.Close()
+	return nil
 }
 
 // fileExists returns true if the file exists, and false otherwise.

channeldb/db_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/btcsuite/btcd/wire"
 	"github.com/btcsuite/btcutil"
 	"github.com/davecgh/go-spew/spew"
+	"github.com/lightningnetwork/lnd/channeldb/kvdb"
 	"github.com/lightningnetwork/lnd/keychain"
 	"github.com/lightningnetwork/lnd/lnwire"
 	"github.com/lightningnetwork/lnd/shachain"
@@ -33,7 +34,13 @@ func TestOpenWithCreate(t *testing.T) {
 
 	// Next, open thereby creating channeldb for the first time.
 	dbPath := filepath.Join(tempDirName, "cdb")
-	cdb, err := Open(dbPath)
+	backend, cleanup, err := kvdb.GetTestBackend(dbPath, "cdb")
+	if err != nil {
+		t.Fatalf("unable to get test db backend: %v", err)
+	}
+	defer cleanup()
+
+	cdb, err := CreateWithBackend(backend)
 	if err != nil {
 		t.Fatalf("unable to create channeldb: %v", err)
 	}
@@ -73,7 +80,13 @@ func TestWipe(t *testing.T) {
 
 	// Next, open thereby creating channeldb for the first time.
 	dbPath := filepath.Join(tempDirName, "cdb")
-	cdb, err := Open(dbPath)
+	backend, cleanup, err := kvdb.GetTestBackend(dbPath, "cdb")
+	if err != nil {
+		t.Fatalf("unable to get test db backend: %v", err)
+	}
+	defer cleanup()
+
+	cdb, err := CreateWithBackend(backend)
 	if err != nil {
 		t.Fatalf("unable to create channeldb: %v", err)
 	}

channeldb/kvdb/backend.go

@@ -0,0 +1,79 @@
+package kvdb
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/lightningnetwork/lnd/channeldb/kvdb/etcd"
+)
+
+// fileExists returns true if the file exists, and false otherwise.
+func fileExists(path string) bool {
+	if _, err := os.Stat(path); err != nil {
+		if os.IsNotExist(err) {
+			return false
+		}
+	}
+
+	return true
+}
+
+// GetBoltBackend opens (or creates if doesn't exits) a bbolt
+// backed database and returns a kvdb.Backend wrapping it.
+func GetBoltBackend(path, name string, noFreeListSync bool) (Backend, error) {
+	dbFilePath := filepath.Join(path, name)
+	var (
+		db  Backend
+		err error
+	)
+
+	if !fileExists(dbFilePath) {
+		if !fileExists(path) {
+			if err := os.MkdirAll(path, 0700); err != nil {
+				return nil, err
+			}
+		}
+
+		db, err = Create(BoltBackendName, dbFilePath, noFreeListSync)
+	} else {
+		db, err = Open(BoltBackendName, dbFilePath, noFreeListSync)
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	return db, nil
+}
+
+// GetTestBackend opens (or creates if doesn't exist) a bbolt or etcd
+// backed database (for testing), and returns a kvdb.Backend and a cleanup
+// func. Whether to create/open bbolt or embedded etcd database is based
+// on the TestBackend constant which is conditionally compiled with build tag.
+// The passed path is used to hold all db files, while the name is only used
+// for bbolt.
+func GetTestBackend(path, name string) (Backend, func(), error) {
+	empty := func() {}
+
+	if TestBackend == BoltBackendName {
+		db, err := GetBoltBackend(path, name, true)
+		if err != nil {
+			return nil, nil, err
+		}
+		return db, empty, nil
+	} else if TestBackend == EtcdBackendName {
+		config, cleanup, err := etcd.NewEmbeddedEtcdInstance(path)
+		if err != nil {
+			return nil, nil, err
+		}
+		backend, err := Open(EtcdBackendName, *config)
+		if err != nil {
+			cleanup()
+			return nil, nil, err
+		}
+		return backend, cleanup, nil
+	}
+
+	return nil, nil, fmt.Errorf("unknown backend")
+}

channeldb/kvdb/kvdb_test_bdb.go

@@ -0,0 +1,5 @@
+// +build !kvdb_etcd
+
+package kvdb
+
+const TestBackend = "bdb"

channeldb/kvdb/kvdb_test_etcd.go

@@ -0,0 +1,5 @@
+// +build kvdb_etcd
+
+package kvdb
+
+const TestBackend = "etcd"

channeldb/meta_test.go

@@ -3,6 +3,7 @@ package channeldb
 import (
 	"bytes"
 	"io/ioutil"
+	"os"
 	"testing"
 
 	"github.com/go-errors/errors"
@@ -421,12 +422,21 @@ func TestMigrationReversion(t *testing.T) {
 	t.Parallel()
 
 	tempDirName, err := ioutil.TempDir("", "channeldb")
+	defer func() {
+		os.RemoveAll(tempDirName)
+	}()
 	if err != nil {
 		t.Fatalf("unable to create temp dir: %v", err)
 	}
 
-	cdb, err := Open(tempDirName)
+	backend, cleanup, err := kvdb.GetTestBackend(tempDirName, "cdb")
 	if err != nil {
+		t.Fatalf("unable to get test db backend: %v", err)
+	}
+
+	cdb, err := CreateWithBackend(backend)
+	if err != nil {
+		cleanup()
 		t.Fatalf("unable to open channeldb: %v", err)
 	}
 
@@ -442,12 +452,19 @@ func TestMigrationReversion(t *testing.T) {
 
 	// Close the database. Even if we succeeded, our next step is to reopen.
 	cdb.Close()
+	cleanup()
 
 	if err != nil {
 		t.Fatalf("unable to increase db version: %v", err)
 	}
 
-	_, err = Open(tempDirName)
+	backend, cleanup, err = kvdb.GetTestBackend(tempDirName, "cdb")
+	if err != nil {
+		t.Fatalf("unable to get test db backend: %v", err)
+	}
+	defer cleanup()
+
+	_, err = CreateWithBackend(backend)
 	if err != ErrDBReversion {
 		t.Fatalf("unexpected error when opening channeldb, "+
 			"want: %v, got: %v", ErrDBReversion, err)