diff --git a/channeldb/kvdb/etcd/db.go b/channeldb/kvdb/etcd/db.go new file mode 100644 index 00000000..4d7c9d0d --- /dev/null +++ b/channeldb/kvdb/etcd/db.go @@ -0,0 +1,55 @@ +package etcd + +import ( + "time" + + "github.com/coreos/etcd/clientv3" +) + +const ( + // etcdConnectionTimeout is the timeout until successful connection to the + // etcd instance. + etcdConnectionTimeout = 10 * time.Second +) + +// db holds a reference to the etcd client connection. +type db struct { + cli *clientv3.Client +} + +// BackendConfig holds and etcd backend config and connection parameters. +type BackendConfig struct { + // Host holds the peer url of the etcd instance. + Host string + + // User is the username for the etcd peer. + User string + + // Pass is the password for the etcd peer. + Pass string +} + +// newEtcdBackend returns a db object initialized with the passed backend +// config. If etcd connection cannot be estabished, then returns error. +func newEtcdBackend(config BackendConfig) (*db, error) { + cli, err := clientv3.New(clientv3.Config{ + Endpoints: []string{config.Host}, + DialTimeout: etcdConnectionTimeout, + Username: config.User, + Password: config.Pass, + }) + if err != nil { + return nil, err + } + + backend := &db{ + cli: cli, + } + + return backend, nil +} + +// Close closes the db, but closing the underlying etcd client connection. +func (db *db) Close() error { + return db.cli.Close() +} diff --git a/channeldb/kvdb/etcd/embed.go b/channeldb/kvdb/etcd/embed.go new file mode 100644 index 00000000..b996293c --- /dev/null +++ b/channeldb/kvdb/etcd/embed.go @@ -0,0 +1,72 @@ +package etcd + +import ( + "fmt" + "net" + "net/url" + "time" + + "github.com/coreos/etcd/embed" +) + +const ( + // readyTimeout is the time until the embedded etcd instance should start. + readyTimeout = 10 * time.Second +) + +// getFreePort returns a random open TCP port. +func getFreePort() int { + ln, err := net.Listen("tcp", "[::]:0") + if err != nil { + panic(err) + } + + port := ln.Addr().(*net.TCPAddr).Port + + err = ln.Close() + if err != nil { + panic(err) + } + + return port +} + +// NewEmbeddedEtcdInstance creates an embedded etcd instance for testing, +// listening on random open ports. Returns the backend config and a cleanup +// func that will stop the etcd instance. +func NewEmbeddedEtcdInstance(path string) (*BackendConfig, func(), error) { + cfg := embed.NewConfig() + cfg.Dir = path + + // To ensure that we can submit large transactions. + cfg.MaxTxnOps = 1000 + + // Listen on random free ports. + clientURL := fmt.Sprintf("127.0.0.1:%d", getFreePort()) + peerURL := fmt.Sprintf("127.0.0.1:%d", getFreePort()) + cfg.LCUrls = []url.URL{{Host: clientURL}} + cfg.LPUrls = []url.URL{{Host: peerURL}} + + etcd, err := embed.StartEtcd(cfg) + if err != nil { + return nil, nil, err + } + + select { + case <-etcd.Server.ReadyNotify(): + case <-time.After(readyTimeout): + etcd.Close() + return nil, nil, + fmt.Errorf("etcd failed to start after: %v", readyTimeout) + } + + connConfig := &BackendConfig{ + Host: "http://" + peerURL, + User: "user", + Pass: "pass", + } + + return connConfig, func() { + etcd.Close() + }, nil +} diff --git a/channeldb/kvdb/etcd/fixture_test.go b/channeldb/kvdb/etcd/fixture_test.go new file mode 100644 index 00000000..7082f186 --- /dev/null +++ b/channeldb/kvdb/etcd/fixture_test.go @@ -0,0 +1,127 @@ +package etcd + +import ( + "context" + "io/ioutil" + "os" + "testing" + "time" + + "github.com/coreos/etcd/clientv3" +) + +const ( + // testEtcdTimeout is used for all RPC calls initiated by the test fixture. + testEtcdTimeout = 5 * time.Second +) + +// EtcdTestFixture holds internal state of the etcd test fixture. +type EtcdTestFixture struct { + t *testing.T + cli *clientv3.Client + config *BackendConfig + cleanup func() +} + +// NewTestEtcdInstance creates an embedded etcd instance for testing, listening +// on random open ports. Returns the connection config and a cleanup func that +// will stop the etcd instance. +func NewTestEtcdInstance(t *testing.T, path string) (*BackendConfig, func()) { + t.Helper() + + config, cleanup, err := NewEmbeddedEtcdInstance(path) + if err != nil { + t.Fatalf("error while staring embedded etcd instance: %v", err) + } + + return config, cleanup +} + +// NewTestEtcdTestFixture creates a new etcd-test fixture. This is helper +// object to facilitate etcd tests and ensure pre and post conditions. +func NewEtcdTestFixture(t *testing.T) *EtcdTestFixture { + tmpDir, err := ioutil.TempDir("", "etcd") + if err != nil { + t.Fatalf("unable to create temp dir: %v", err) + } + + config, etcdCleanup := NewTestEtcdInstance(t, tmpDir) + + cli, err := clientv3.New(clientv3.Config{ + Endpoints: []string{config.Host}, + Username: config.User, + Password: config.Pass, + }) + if err != nil { + os.RemoveAll(tmpDir) + t.Fatalf("unable to create etcd test fixture: %v", err) + } + + return &EtcdTestFixture{ + t: t, + cli: cli, + config: config, + cleanup: func() { + etcdCleanup() + os.RemoveAll(tmpDir) + }, + } +} + +// Put puts a string key/value into the test etcd database. +func (f *EtcdTestFixture) Put(key, value string) { + ctx, cancel := context.WithTimeout(context.TODO(), testEtcdTimeout) + defer cancel() + + _, err := f.cli.Put(ctx, key, value) + if err != nil { + f.t.Fatalf("etcd test fixture failed to put: %v", err) + } +} + +// Get queries a key and returns the stored value from the test etcd database. +func (f *EtcdTestFixture) Get(key string) string { + ctx, cancel := context.WithTimeout(context.TODO(), testEtcdTimeout) + defer cancel() + + resp, err := f.cli.Get(ctx, key) + if err != nil { + f.t.Fatalf("etcd test fixture failed to put: %v", err) + } + + if len(resp.Kvs) > 0 { + return string(resp.Kvs[0].Value) + } + + return "" +} + +// Dump scans and returns all key/values from the test etcd database. +func (f *EtcdTestFixture) Dump() map[string]string { + ctx, cancel := context.WithTimeout(context.TODO(), testEtcdTimeout) + defer cancel() + + resp, err := f.cli.Get(ctx, "", clientv3.WithPrefix()) + if err != nil { + f.t.Fatalf("etcd test fixture failed to put: %v", err) + } + + result := make(map[string]string) + for _, kv := range resp.Kvs { + result[string(kv.Key)] = string(kv.Value) + } + + return result +} + +// BackendConfig returns the backend config for connecting to theembedded +// etcd instance. +func (f *EtcdTestFixture) BackendConfig() BackendConfig { + return *f.config +} + +// Cleanup should be called at test fixture teardown to stop the embedded +// etcd instance and remove all temp db files form the filesystem. +func (f *EtcdTestFixture) Cleanup() { + f.cleanup() +} diff --git a/channeldb/kvdb/etcd/stm.go b/channeldb/kvdb/etcd/stm.go new file mode 100644 index 00000000..8d8c96e0 --- /dev/null +++ b/channeldb/kvdb/etcd/stm.go @@ -0,0 +1,728 @@ +package etcd + +import ( + "context" + "fmt" + "math" + "strings" + + v3 "github.com/coreos/etcd/clientv3" +) + +// KV stores a key/value pair. +type KV struct { + key string + val string +} + +// STM is an interface for software transactional memory. +// All calls that return error will do so only if STM is manually handled and +// abort the apply closure otherwise. In both case the returned error is a +// DatabaseError. +type STM interface { + // Get returns the value for a key and inserts the key in the txn's read + // set. Returns nil if there's no matching key, or the key is empty. + Get(key string) ([]byte, error) + + // Put adds a value for a key to the txn's write set. + Put(key, val string) + + // Del adds a delete operation for the key to the txn's write set. + Del(key string) + + // First returns the first k/v that begins with prefix or nil if there's + // no such k/v pair. If the key is found it is inserted to the txn's + // read set. Returns nil if there's no match. + First(prefix string) (*KV, error) + + // Last returns the last k/v that begins with prefix or nil if there's + // no such k/v pair. If the key is found it is inserted to the txn's + // read set. Returns nil if there's no match. + Last(prefix string) (*KV, error) + + // Prev returns the previous k/v before key that begins with prefix or + // nil if there's no such k/v. If the key is found it is inserted to the + // read set. Returns nil if there's no match. + Prev(prefix, key string) (*KV, error) + + // Next returns the next k/v after key that begins with prefix or nil + // if there's no such k/v. If the key is found it is inserted to the + // txn's read set. Returns nil if there's no match. + Next(prefix, key string) (*KV, error) + + // Seek will return k/v at key beginning with prefix. If the key doesn't + // exists Seek will return the next k/v after key beginning with prefix. + // If a matching k/v is found it is inserted to the txn's read set. Returns + // nil if there's no match. + Seek(prefix, key string) (*KV, error) + + // OnCommit calls the passed callback func upon commit. + OnCommit(func()) + + // Commit attempts to apply the txn's changes to the server. + // Commit may return CommitError if transaction is outdated and needs retry. + Commit() error + + // Rollback emties the read and write sets such that a subsequent commit + // won't alter the database. + Rollback() +} + +// CommitError is used to check if there was an error +// due to stale data in the transaction. +type CommitError struct{} + +// Error returns a static string for CommitError for +// debugging/logging purposes. +func (e CommitError) Error() string { + return "commit failed" +} + +// DatabaseError is used to wrap errors that are not +// related to stale data in the transaction. +type DatabaseError struct { + msg string + err error +} + +// Unwrap returns the wrapped error in a DatabaseError. +func (e *DatabaseError) Unwrap() error { + return e.err +} + +// Error simply converts DatabaseError to a string that +// includes both the message and the wrapped error. +func (e DatabaseError) Error() string { + return fmt.Sprintf("etcd error: %v - %v", e.msg, e.err) +} + +// stmGet is the result of a read operation, +// a value and the mod revision of the key/value. +type stmGet struct { + val string + rev int64 +} + +// readSet stores all reads done in an STM. +type readSet map[string]stmGet + +// stmPut stores a value and an operation (put/delete). +type stmPut struct { + val string + op v3.Op +} + +// writeSet stroes all writes done in an STM. +type writeSet map[string]stmPut + +// stm implements repeatable-read software transactional memory +// over etcd. +type stm struct { + // client is an etcd client handling all RPC communications + // to the etcd instance/cluster. + client *v3.Client + + // manual is set to true for manual transactions which don't + // execute in the STM run loop. + manual bool + + // options stores optional settings passed by the user. + options *STMOptions + + // prefetch hold prefetched key values and revisions. + prefetch readSet + + // rset holds read key values and revisions. + rset readSet + + // wset holds overwritten keys and their values. + wset writeSet + + // getOpts are the opts used for gets. + getOpts []v3.OpOption + + // revision stores the snapshot revision after first read. + revision int64 + + // onCommit gets called upon commit. + onCommit func() +} + +// STMOptions can be used to pass optional settings +// when an STM is created. +type STMOptions struct { + // ctx holds an externally provided abort context. + ctx context.Context +} + +// STMOptionFunc is a function that updates the passed STMOptions. +type STMOptionFunc func(*STMOptions) + +// WithAbortContext specifies the context for permanently +// aborting the transaction. +func WithAbortContext(ctx context.Context) STMOptionFunc { + return func(so *STMOptions) { + so.ctx = ctx + } +} + +// RunSTM runs the apply function by creating an STM using serializable snapshot +// isolation, passing it to the apply and handling commit errors and retries. +func RunSTM(cli *v3.Client, apply func(STM) error, so ...STMOptionFunc) error { + return runSTM(makeSTM(cli, false, so...), apply) +} + +// NewSTM creates a new STM instance, using serializable snapshot isolation. +func NewSTM(cli *v3.Client, so ...STMOptionFunc) STM { + return makeSTM(cli, true, so...) +} + +// makeSTM is the actual constructor of the stm. It first apply all passed +// options then creates the stm object and resets it before returning. +func makeSTM(cli *v3.Client, manual bool, so ...STMOptionFunc) *stm { + opts := &STMOptions{ + ctx: cli.Ctx(), + } + + // Apply all functional options. + for _, fo := range so { + fo(opts) + } + + s := &stm{ + client: cli, + manual: manual, + options: opts, + prefetch: make(map[string]stmGet), + } + + // Reset read and write set. + s.Rollback() + + return s +} + +// runSTM implements the run loop of the STM, running the apply func, catching +// errors and handling commit. The loop will quit on every error except +// CommitError which is used to indicate a necessary retry. +func runSTM(s *stm, apply func(STM) error) error { + out := make(chan error, 1) + + go func() { + defer func() { + // Recover DatabaseError panics so + // we can return them. + if r := recover(); r != nil { + e, ok := r.(DatabaseError) + if !ok { + // Unknown panic. + panic(r) + } + + // Return the error. + out <- e.Unwrap() + } + }() + + var err error + + // In a loop try to apply and commit and roll back + // if the database has changed (CommitError). + for { + // Abort STM if there was an application error. + if err = apply(s); err != nil { + break + } + + err = s.Commit() + + // Re-apply only upon commit error + // (meaning the database was changed). + if _, ok := err.(CommitError); !ok { + // Anything that's not a CommitError + // aborts the STM run loop. + break + } + + // Rollback before trying to re-apply. + s.Rollback() + } + + // Return the error to the caller. + out <- err + }() + + return <-out +} + +// add inserts a txn response to the read set. This is useful when the txn +// fails due to conflict where the txn response can be used to prefetch +// key/values. +func (rs readSet) add(txnResp *v3.TxnResponse) { + for _, resp := range txnResp.Responses { + getResp := (*v3.GetResponse)(resp.GetResponseRange()) + for _, kv := range getResp.Kvs { + rs[string(kv.Key)] = stmGet{ + val: string(kv.Value), + rev: kv.ModRevision, + } + } + } +} + +// gets is a helper to create an op slice for transaction +// construction. +func (rs readSet) gets() []v3.Op { + ops := make([]v3.Op, 0, len(rs)) + + for k := range rs { + ops = append(ops, v3.OpGet(k)) + } + + return ops +} + +// cmps returns a cmp list testing values in read set didn't change. +func (rs readSet) cmps() []v3.Cmp { + cmps := make([]v3.Cmp, 0, len(rs)) + for key, getValue := range rs { + cmps = append(cmps, v3.Compare(v3.ModRevision(key), "=", getValue.rev)) + } + + return cmps +} + +// cmps returns a cmp list testing no writes have happened past rev. +func (ws writeSet) cmps(rev int64) []v3.Cmp { + cmps := make([]v3.Cmp, 0, len(ws)) + for key := range ws { + cmps = append(cmps, v3.Compare(v3.ModRevision(key), "<", rev)) + } + + return cmps +} + +// puts is the list of ops for all pending writes. +func (ws writeSet) puts() []v3.Op { + puts := make([]v3.Op, 0, len(ws)) + for _, v := range ws { + puts = append(puts, v.op) + } + + return puts +} + +// fetch is a helper to fetch key/value given options. If a value is returned +// then fetch will try to fix the STM's snapshot revision (if not already set). +// We'll also cache the returned key/value in the read set. +func (s *stm) fetch(key string, opts ...v3.OpOption) ([]KV, error) { + resp, err := s.client.Get( + s.options.ctx, key, append(opts, s.getOpts...)..., + ) + if err != nil { + dbErr := DatabaseError{ + msg: "stm.fetch() failed", + err: err, + } + + // Do not panic when executing a manual transaction. + if s.manual { + return nil, dbErr + } + + // Panic when executing inside the STM runloop. + panic(dbErr) + } + + // Set revison and serializable options upon first fetch + // for any subsequent fetches. + if s.getOpts == nil { + s.revision = resp.Header.Revision + s.getOpts = []v3.OpOption{ + v3.WithRev(s.revision), + v3.WithSerializable(), + } + } + + var result []KV + + // Fill the read set with key/values returned. + for _, kv := range resp.Kvs { + // Remove from prefetch. + key := string(kv.Key) + val := string(kv.Value) + + delete(s.prefetch, key) + + // Add to read set. + s.rset[key] = stmGet{ + val: val, + rev: kv.ModRevision, + } + + result = append(result, KV{key, val}) + } + + return result, nil +} + +// Get returns the value for key. If there's no such +// key/value in the database or the passed key is empty +// Get will return nil. +func (s *stm) Get(key string) ([]byte, error) { + if key == "" { + return nil, nil + } + + // Return freshly written value if present. + if put, ok := s.wset[key]; ok { + if put.op.IsDelete() { + return nil, nil + } + + return []byte(put.val), nil + } + + // Populate read set if key is present in + // the prefetch set. + if getValue, ok := s.prefetch[key]; ok { + delete(s.prefetch, key) + s.rset[key] = getValue + } + + // Return value if alread in read set. + if getVal, ok := s.rset[key]; ok { + return []byte(getVal.val), nil + } + + // Fetch and return value. + kvs, err := s.fetch(key) + if err != nil { + return nil, err + } + + if len(kvs) > 0 { + return []byte(kvs[0].val), nil + } + + // Return empty result if key not in DB. + return nil, nil +} + +// First returns the first key/value matching prefix. If there's no key starting +// with prefix, Last will return nil. +func (s *stm) First(prefix string) (*KV, error) { + return s.next(prefix, prefix, true) +} + +// Last returns the last key/value with prefix. If there's no key starting with +// prefix, Last will return nil. +func (s *stm) Last(prefix string) (*KV, error) { + // As we don't know the full range, fetch the last + // key/value with this prefix first. + resp, err := s.fetch(prefix, v3.WithLastKey()...) + if err != nil { + return nil, err + } + + var ( + kv KV + found bool + ) + + if len(resp) > 0 { + kv = resp[0] + found = true + } + + // Now make sure there's nothing in the write set + // that is a better match, meaning it has the same + // prefix but is greater or equal than the current + // best candidate. Note that this is not efficient + // when the write set is large! + for k, put := range s.wset { + if put.op.IsDelete() { + continue + } + + if strings.HasPrefix(k, prefix) && k >= kv.key { + kv.key = k + kv.val = put.val + found = true + } + } + + if found { + return &kv, nil + } + + return nil, nil +} + +// Prev returns the prior key/value before key (with prefix). If there's no such +// key Next will return nil. +func (s *stm) Prev(prefix, startKey string) (*KV, error) { + var result KV + + fetchKey := startKey + matchFound := false + + for { + // Ask etcd to retrieve one key that is a + // match in descending order from the passed key. + opts := []v3.OpOption{ + v3.WithRange(fetchKey), + v3.WithSort(v3.SortByKey, v3.SortDescend), + v3.WithLimit(1), + } + + kvs, err := s.fetch(prefix, opts...) + if err != nil { + return nil, err + } + + if len(kvs) == 0 { + break + } + + kv := &kvs[0] + + // WithRange and WithPrefix can't be used + // together, so check prefix here. If the + // returned key no longer has the prefix, + // then break out. + if !strings.HasPrefix(kv.key, prefix) { + break + } + + // Fetch the prior key if this is deleted. + if put, ok := s.wset[kv.key]; ok && put.op.IsDelete() { + fetchKey = kv.key + continue + } + + result = *kv + matchFound = true + + break + } + + // Closre holding all checks to find a possibly + // better match. + matches := func(key string) bool { + if !strings.HasPrefix(key, prefix) { + return false + } + + if !matchFound { + return key < startKey + } + + // matchFound == true + return result.key <= key && key < startKey + } + + // Now go trough the write set and check + // if there's an even better match. + for k, put := range s.wset { + if !put.op.IsDelete() && matches(k) { + result.key = k + result.val = put.val + matchFound = true + } + } + + if !matchFound { + return nil, nil + } + + return &result, nil +} + +// Next returns the next key/value after key (with prefix). If there's no such +// key Next will return nil. +func (s *stm) Next(prefix string, key string) (*KV, error) { + return s.next(prefix, key, false) +} + +// Seek "seeks" to the key (with prefix). If the key doesn't exists it'll get +// the next key with the same prefix. If no key fills this criteria, Seek will +// return nil. +func (s *stm) Seek(prefix, key string) (*KV, error) { + return s.next(prefix, key, true) +} + +// next will try to retrieve the next match that has prefix and starts with the +// passed startKey. If includeStartKey is set to true, it'll return the value +// of startKey (essentially implementing seek). +func (s *stm) next(prefix, startKey string, includeStartKey bool) (*KV, error) { + var result KV + + fetchKey := startKey + firstFetch := true + matchFound := false + + for { + // Ask etcd to retrieve one key that is a + // match in ascending order from the passed key. + opts := []v3.OpOption{ + v3.WithFromKey(), + v3.WithSort(v3.SortByKey, v3.SortAscend), + v3.WithLimit(1), + } + + // By default we include the start key too + // if it is a full match. + if includeStartKey && firstFetch { + firstFetch = false + } else { + // If we'd like to retrieve the first key + // after the start key. + fetchKey += "\x00" + } + + kvs, err := s.fetch(fetchKey, opts...) + if err != nil { + return nil, err + } + + if len(kvs) == 0 { + break + } + + kv := &kvs[0] + // WithRange and WithPrefix can't be used + // together, so check prefix here. If the + // returned key no longer has the prefix, + // then break the fetch loop. + if !strings.HasPrefix(kv.key, prefix) { + break + } + + // Move on to fetch starting with the next + // key if this one is marked deleted. + if put, ok := s.wset[kv.key]; ok && put.op.IsDelete() { + fetchKey = kv.key + continue + } + + result = *kv + matchFound = true + + break + } + + // Closure holding all checks to find a possibly + // better match. + matches := func(k string) bool { + if !strings.HasPrefix(k, prefix) { + return false + } + + if includeStartKey && !matchFound { + return startKey <= k + } + + if !includeStartKey && !matchFound { + return startKey < k + } + + if includeStartKey && matchFound { + return startKey <= k && k <= result.key + } + + // !includeStartKey && matchFound. + return startKey < k && k <= result.key + } + + // Now go trough the write set and check + // if there's an even better match. + for k, put := range s.wset { + if !put.op.IsDelete() && matches(k) { + result.key = k + result.val = put.val + matchFound = true + } + } + + if !matchFound { + return nil, nil + } + + return &result, nil +} + +// Put sets the value of the passed key. The actual put will happen upon commit. +func (s *stm) Put(key, val string) { + s.wset[key] = stmPut{ + val: val, + op: v3.OpPut(key, val), + } +} + +// Del marks a key as deleted. The actual delete will happen upon commit. +func (s *stm) Del(key string) { + s.wset[key] = stmPut{ + val: "", + op: v3.OpDelete(key), + } +} + +// OnCommit sets the callback that is called upon committing the STM +// transaction. +func (s *stm) OnCommit(cb func()) { + s.onCommit = cb +} + +// Commit builds the final transaction and tries to execute it. If commit fails +// because the keys have changed return a CommitError, otherwise return a +// DatabaseError. +func (s *stm) Commit() error { + // Create the compare set. + cmps := append(s.rset.cmps(), s.wset.cmps(s.revision+1)...) + // Create a transaction with the optional abort context. + txn := s.client.Txn(s.options.ctx) + + // If the compare set holds, try executing the puts. + txn = txn.If(cmps...) + txn = txn.Then(s.wset.puts()...) + + // Prefetch keys in case of conflict to save + // a round trip to etcd. + txn = txn.Else(s.rset.gets()...) + + txnresp, err := txn.Commit() + if err != nil { + return DatabaseError{ + msg: "stm.Commit() failed", + err: err, + } + } + + // Call the commit callback if the transaction + // was successful. + if txnresp.Succeeded { + if s.onCommit != nil { + s.onCommit() + } + + return nil + } + + // Load prefetch before if commit failed. + s.rset.add(txnresp) + s.prefetch = s.rset + + // Return CommitError indicating that the transaction + // can be retried. + return CommitError{} +} + +// Rollback resets the STM. This is useful for uncommitted transaction rollback +// and also used in the STM main loop to reset state if commit fails. +func (s *stm) Rollback() { + s.rset = make(map[string]stmGet) + s.wset = make(map[string]stmPut) + s.getOpts = nil + s.revision = math.MaxInt64 - 1 +} diff --git a/channeldb/kvdb/etcd/stm_test.go b/channeldb/kvdb/etcd/stm_test.go new file mode 100644 index 00000000..80b2ef1d --- /dev/null +++ b/channeldb/kvdb/etcd/stm_test.go @@ -0,0 +1,342 @@ +package etcd + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" +) + +func reverseKVs(a []KV) []KV { + for i, j := 0, len(a)-1; i < j; i, j = i+1, j-1 { + a[i], a[j] = a[j], a[i] + } + + return a +} + +func TestPutToEmpty(t *testing.T) { + t.Parallel() + + f := NewEtcdTestFixture(t) + defer f.Cleanup() + + db, err := newEtcdBackend(f.BackendConfig()) + assert.NoError(t, err) + + apply := func(stm STM) error { + stm.Put("123", "abc") + return nil + } + + err = RunSTM(db.cli, apply) + assert.NoError(t, err) + + assert.Equal(t, "abc", f.Get("123")) +} + +func TestGetPutDel(t *testing.T) { + t.Parallel() + + f := NewEtcdTestFixture(t) + defer f.cleanup() + + testKeyValues := []KV{ + {"a", "1"}, + {"b", "2"}, + {"c", "3"}, + {"d", "4"}, + {"e", "5"}, + } + + for _, kv := range testKeyValues { + f.Put(kv.key, kv.val) + } + + db, err := newEtcdBackend(f.BackendConfig()) + assert.NoError(t, err) + + apply := func(stm STM) error { + // Get some non existing keys. + v, err := stm.Get("") + assert.NoError(t, err) + assert.Nil(t, v) + + v, err = stm.Get("x") + assert.NoError(t, err) + assert.Nil(t, v) + + // Get all existing keys. + for _, kv := range testKeyValues { + v, err = stm.Get(kv.key) + assert.NoError(t, err) + assert.Equal(t, []byte(kv.val), v) + } + + // Overwrite, then delete an existing key. + stm.Put("c", "6") + + v, err = stm.Get("c") + assert.NoError(t, err) + assert.Equal(t, []byte("6"), v) + + stm.Del("c") + + v, err = stm.Get("c") + assert.NoError(t, err) + assert.Nil(t, v) + + // Re-add the deleted key. + stm.Put("c", "7") + + v, err = stm.Get("c") + assert.NoError(t, err) + assert.Equal(t, []byte("7"), v) + + // Add a new key. + stm.Put("x", "x") + + v, err = stm.Get("x") + assert.NoError(t, err) + assert.Equal(t, []byte("x"), v) + + return nil + } + + err = RunSTM(db.cli, apply) + assert.NoError(t, err) + + assert.Equal(t, "1", f.Get("a")) + assert.Equal(t, "2", f.Get("b")) + assert.Equal(t, "7", f.Get("c")) + assert.Equal(t, "4", f.Get("d")) + assert.Equal(t, "5", f.Get("e")) + assert.Equal(t, "x", f.Get("x")) +} + +func TestFirstLastNextPrev(t *testing.T) { + t.Parallel() + + f := NewEtcdTestFixture(t) + defer f.Cleanup() + + testKeyValues := []KV{ + {"kb", "1"}, + {"kc", "2"}, + {"kda", "3"}, + {"ke", "4"}, + {"w", "w"}, + } + for _, kv := range testKeyValues { + f.Put(kv.key, kv.val) + } + + db, err := newEtcdBackend(f.BackendConfig()) + assert.NoError(t, err) + + apply := func(stm STM) error { + // First/Last on valid multi item interval. + kv, err := stm.First("k") + assert.NoError(t, err) + assert.Equal(t, &KV{"kb", "1"}, kv) + + kv, err = stm.Last("k") + assert.NoError(t, err) + assert.Equal(t, &KV{"ke", "4"}, kv) + + // First/Last on single item interval. + kv, err = stm.First("w") + assert.NoError(t, err) + assert.Equal(t, &KV{"w", "w"}, kv) + + kv, err = stm.Last("w") + assert.NoError(t, err) + assert.Equal(t, &KV{"w", "w"}, kv) + + // Next/Prev on start/end. + kv, err = stm.Next("k", "ke") + assert.NoError(t, err) + assert.Nil(t, kv) + + kv, err = stm.Prev("k", "kb") + assert.NoError(t, err) + assert.Nil(t, kv) + + // Next/Prev in the middle. + kv, err = stm.Next("k", "kc") + assert.NoError(t, err) + assert.Equal(t, &KV{"kda", "3"}, kv) + + kv, err = stm.Prev("k", "ke") + assert.NoError(t, err) + assert.Equal(t, &KV{"kda", "3"}, kv) + + // Delete first item, then add an item before the + // deleted one. Check that First/Next will "jump" + // over the deleted item and return the new first. + stm.Del("kb") + stm.Put("ka", "0") + + kv, err = stm.First("k") + assert.NoError(t, err) + assert.Equal(t, &KV{"ka", "0"}, kv) + + kv, err = stm.Prev("k", "kc") + assert.NoError(t, err) + assert.Equal(t, &KV{"ka", "0"}, kv) + + // Similarly test that a new end is returned if + // the old end is deleted first. + stm.Del("ke") + stm.Put("kf", "5") + + kv, err = stm.Last("k") + assert.NoError(t, err) + assert.Equal(t, &KV{"kf", "5"}, kv) + + kv, err = stm.Next("k", "kda") + assert.NoError(t, err) + assert.Equal(t, &KV{"kf", "5"}, kv) + + // Overwrite one in the middle. + stm.Put("kda", "6") + + kv, err = stm.Next("k", "kc") + assert.NoError(t, err) + assert.Equal(t, &KV{"kda", "6"}, kv) + + // Add three in the middle, then delete one. + stm.Put("kdb", "7") + stm.Put("kdc", "8") + stm.Put("kdd", "9") + stm.Del("kdc") + + // Check that stepping from first to last returns + // the expected sequence. + var kvs []KV + + curr, err := stm.First("k") + assert.NoError(t, err) + + for curr != nil { + kvs = append(kvs, *curr) + curr, err = stm.Next("k", curr.key) + assert.NoError(t, err) + } + + expected := []KV{ + {"ka", "0"}, + {"kc", "2"}, + {"kda", "6"}, + {"kdb", "7"}, + {"kdd", "9"}, + {"kf", "5"}, + } + assert.Equal(t, expected, kvs) + + // Similarly check that stepping from last to first + // returns the expected sequence. + kvs = []KV{} + + curr, err = stm.Last("k") + assert.NoError(t, err) + + for curr != nil { + kvs = append(kvs, *curr) + curr, err = stm.Prev("k", curr.key) + assert.NoError(t, err) + } + + expected = reverseKVs(expected) + assert.Equal(t, expected, kvs) + + return nil + } + + err = RunSTM(db.cli, apply) + assert.NoError(t, err) + + assert.Equal(t, "0", f.Get("ka")) + assert.Equal(t, "2", f.Get("kc")) + assert.Equal(t, "6", f.Get("kda")) + assert.Equal(t, "7", f.Get("kdb")) + assert.Equal(t, "9", f.Get("kdd")) + assert.Equal(t, "5", f.Get("kf")) + assert.Equal(t, "w", f.Get("w")) +} + +func TestCommitError(t *testing.T) { + t.Parallel() + + f := NewEtcdTestFixture(t) + defer f.Cleanup() + + db, err := newEtcdBackend(f.BackendConfig()) + assert.NoError(t, err) + + // Preset DB state. + f.Put("123", "xyz") + + // Count the number of applies. + cnt := 0 + + apply := func(stm STM) error { + // STM must have the key/value. + val, err := stm.Get("123") + assert.NoError(t, err) + + if cnt == 0 { + assert.Equal(t, []byte("xyz"), val) + + // Put a conflicting key/value during the first apply. + f.Put("123", "def") + } + + // We'd expect to + stm.Put("123", "abc") + + cnt++ + return nil + } + + err = RunSTM(db.cli, apply) + assert.NoError(t, err) + assert.Equal(t, 2, cnt) + + assert.Equal(t, "abc", f.Get("123")) +} + +func TestManualTxError(t *testing.T) { + t.Parallel() + + f := NewEtcdTestFixture(t) + defer f.Cleanup() + + db, err := newEtcdBackend(f.BackendConfig()) + assert.NoError(t, err) + + // Preset DB state. + f.Put("123", "xyz") + + stm := NewSTM(db.cli) + + val, err := stm.Get("123") + assert.NoError(t, err) + assert.Equal(t, []byte("xyz"), val) + + // Put a conflicting key/value. + f.Put("123", "def") + + // Should still get the original version. + val, err = stm.Get("123") + assert.NoError(t, err) + assert.Equal(t, []byte("xyz"), val) + + // Commit will fail with CommitError. + err = stm.Commit() + var e CommitError + assert.True(t, errors.As(err, &e)) + + // We expect that the transacton indeed did not commit. + assert.Equal(t, "def", f.Get("123")) +} diff --git a/go.mod b/go.mod index 1c4c1ca0..d6674b8e 100644 --- a/go.mod +++ b/go.mod @@ -14,10 +14,18 @@ require ( github.com/btcsuite/btcwallet/wallet/txrules v1.0.0 github.com/btcsuite/btcwallet/walletdb v1.3.1 github.com/btcsuite/btcwallet/wtxmgr v1.1.1-0.20200515224913-e0e62245ecbe + github.com/coreos/etcd v3.3.18+incompatible + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd v0.0.0-00010101000000-000000000000 // indirect + github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/davecgh/go-spew v1.1.1 + github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/go-errors/errors v1.0.1 github.com/go-openapi/strfmt v0.19.5 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.3.1 + github.com/google/btree v1.0.0 // indirect + github.com/gorilla/websocket v1.4.1 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/grpc-ecosystem/grpc-gateway v1.8.6 @@ -25,7 +33,9 @@ require ( github.com/jackpal/go-nat-pmp v0.0.0-20170405195558-28a68d0c24ad github.com/jedib0t/go-pretty v4.3.0+incompatible github.com/jessevdk/go-flags v1.4.0 + github.com/jonboulle/clockwork v0.1.0 // indirect github.com/jrick/logrotate v1.0.0 + github.com/json-iterator/go v1.1.9 // indirect github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c // indirect github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d // indirect github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8 // indirect @@ -43,12 +53,19 @@ require ( github.com/ltcsuite/ltcd v0.0.0-20190101042124-f37f8bf35796 github.com/mattn/go-runewidth v0.0.9 // indirect github.com/miekg/dns v0.0.0-20171125082028-79bfde677fa8 + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/prometheus/client_golang v0.9.3 github.com/rogpeppe/fastuuid v1.2.0 // indirect + github.com/soheilhy/cmux v0.1.4 // indirect + github.com/stretchr/testify v1.4.0 + github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect github.com/tv42/zbase32 v0.0.0-20160707012821-501572607d02 github.com/urfave/cli v1.18.0 + github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect + go.etcd.io/bbolt v1.3.3 + go.uber.org/zap v1.14.1 // indirect golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37 - golang.org/x/net v0.0.0-20190206173232-65e2d4e15006 + golang.org/x/net v0.0.0-20190620200207-3b0461eec859 golang.org/x/sys v0.0.0-20200116001909-b77594299b42 // indirect golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 google.golang.org/genproto v0.0.0-20190201180003-4b09977fb922 @@ -57,6 +74,7 @@ require ( gopkg.in/macaroon-bakery.v2 v2.0.1 gopkg.in/macaroon.v2 v2.0.0 gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect + sigs.k8s.io/yaml v1.1.0 // indirect ) replace github.com/lightningnetwork/lnd/ticker => ./ticker @@ -71,4 +89,6 @@ replace git.schwanenlied.me/yawning/bsaes.git => github.com/Yawning/bsaes v0.0.0 // btcsuite/btcutil package requests a newer version. replace golang.org/x/crypto => golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67 +replace github.com/coreos/go-systemd => github.com/coreos/go-systemd/v22 v22.0.0 + go 1.12 diff --git a/go.sum b/go.sum index eb42ab16..f52981d8 100644 --- a/go.sum +++ b/go.sum @@ -70,10 +70,20 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY= github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.18+incompatible h1:Zz1aXgDrFFi1nadh58tA9ktt06cmPTwNNP3dXwIq1lE= +github.com/coreos/etcd v3.3.18+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd/v22 v22.0.0 h1:XJIw/+VlJ+87J+doOxznsAWIdmWuViOVhkQamW5YV28= +github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/frankban/quicktest v1.2.2 h1:xfmOhhoH5fGPgbEAlhLpJH9p0z/0Qizio9osmvn9IUY= github.com/frankban/quicktest v1.2.2/go.mod h1:Qh/WofXFeiAFII1aEBu529AtJo6Zg2VHscnEsbBnJ20= @@ -91,20 +101,31 @@ github.com/go-openapi/strfmt v0.19.5 h1:0utjKrw+BAh8s57XE9Xz8DUBsVvPmRUB6styvl9w github.com/go-openapi/strfmt v0.19.5/go.mod h1:eftuHTlB/dI8Uq8JJOyRlieZf+WkkxUuk0dgdHXr2Qk= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42 h1:q3pnF5JFBNRz8sRD+IRj7Y6DMyYGTNqnZ9axTbSfoNI= github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= +github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 h1:Iju5GlWwrvL6UBg4zJJt3btmonfrMlCDdsejg4CZE7c= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= @@ -122,8 +143,12 @@ github.com/jedib0t/go-pretty v4.3.0+incompatible/go.mod h1:XemHduiw8R651AF9Pt4Fw github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jrick/logrotate v1.0.0 h1:lQ1bL/n9mBNeIXoTUoYRlK4dHuNJVofX9oWqBtPnSzI= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= +github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c h1:3UvYABOQRhJAApj9MdCN+Ydv841ETSoy6xLzdmmr/9A= github.com/juju/clock v0.0.0-20190205081909-9c5c9712527c/go.mod h1:nD0vlnrUjcjJhqN5WuCWZyzfd5AHZAC9/ajvbSx69xA= github.com/juju/errors v0.0.0-20190806202954-0232dcc7464d h1:hJXjZMxj0SWlMoQkzeZDLi2cmeiWKa7y1B8Rg+qaoEc= @@ -170,6 +195,11 @@ github.com/miekg/dns v0.0.0-20171125082028-79bfde677fa8 h1:PRMAcldsl4mXKJeRNB/KV github.com/miekg/dns v0.0.0-20171125082028-79bfde677fa8/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -179,6 +209,7 @@ github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5 github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -197,30 +228,50 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 h1:LnC5Kc/wtumK+WB441p7ynQJzVuNRJiqddSIE3IlSEQ= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/zbase32 v0.0.0-20160707012821-501572607d02 h1:tcJ6OjwOMvExLlzrAVZute09ocAGa7KqOON60++Gz4E= github.com/tv42/zbase32 v0.0.0-20160707012821-501572607d02/go.mod h1:tHlrkM198S068ZqfrO6S8HsoJq2bF3ETfTL+kt4tInY= github.com/urfave/cli v1.18.0 h1:m9MfmZWX7bwr9kUcs/Asr95j0IVXzGNNc+/5ku2m26Q= github.com/urfave/cli v1.18.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.mongodb.org/mongo-driver v1.0.3 h1:GKoji1ld3tw2aC+GX1wbr/J2fX13yNacEYoJ8Nhr0yU= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= +go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.14.1 h1:nYDKopTbvAPq/NrUVZwT15y2lpROBiLLyoRTbXOYWOo= +go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67 h1:ng3VDlRp5/DHpSWl02R4rM9I+8M2rhmsuLwAMmkLQWE= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529 h1:iMGN4xG0cnqj3t+zOM8wUB0BiPKHEwSxEZCvzcbZuvk= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -229,10 +280,15 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190206173232-65e2d4e15006 h1:bfLnR+k0tq5Lqt6dflRLcZiz6UaXCMt3vhYJ1l4FQ80= golang.org/x/net v0.0.0-20190206173232-65e2d4e15006/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -241,6 +297,8 @@ golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503 h1:5SvYFrOM3W8Mexn9/oA44Ji7vhXAZQ9hiP+1Q/DMrWg= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd h1:DBH9mDw0zluJT/R+nGuV3jWFWLFaHyYZWD4tOT+cjn0= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg= @@ -252,6 +310,11 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2I golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190201180003-4b09977fb922 h1:mBVYJnbrXLA/ZCBTCe7PtEgAUP+1bg92qTaFoPHdz+8= @@ -267,6 +330,7 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v1 v1.0.1 h1:oQFRXzZ7CkBGdm1XZm/EbQYaYNNEElNBOd09M6cqNso= gopkg.in/errgo.v1 v1.0.1/go.mod h1:3NjfXwocQRYAPTq4/fzX+CwUhPRcR/azYRhj8G+LqMo= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/macaroon-bakery.v2 v2.0.1 h1:0N1TlEdfLP4HXNCg7MQUMp5XwvOoxk+oe9Owr2cpvsc= @@ -284,3 +348,6 @@ gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=