From 229e34b32608c6d47e259fed60e00411b3d1c9f5 Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Mon, 18 Jan 2016 23:43:41 -0800 Subject: [PATCH 01/30] make SPVCon version agnostic --- uspv/eight333.go | 61 ++++++++++++++++++++-------------------------- uspv/msghandler.go | 4 +-- 2 files changed, 29 insertions(+), 36 deletions(-) diff --git a/uspv/eight333.go b/uspv/eight333.go index 4b6ec382..75f8c52e 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -18,12 +18,9 @@ const ( headerFileName = "headers.bin" // Except hash-160s, those aren't backwards. But anything that's 32 bytes is. // because, cmon, 32? Gotta reverse that. But 20? 20 is OK. - NETVERSION = wire.TestNet3 - VERSION = 70011 -) -var ( - params = &chaincfg.TestNet3Params + // version hardcoded for now, probably ok...? + VERSION = 70011 ) type SPVCon struct { @@ -33,7 +30,6 @@ type SPVCon struct { localVersion uint32 // version we report remoteVersion uint32 // version remote node remoteHeight int32 // block height they're on - netType wire.BitcoinNet // what's the point of the input queue? remove? leave for now... inMsgQueue chan wire.Message // Messages coming in from remote node @@ -42,50 +38,59 @@ type SPVCon struct { WBytes uint64 // total bytes written RBytes uint64 // total bytes read - TS *TxStore + TS *TxStore + param *chaincfg.Params } -func (s *SPVCon) Open(remoteNode string, hfn string, inTs *TxStore) error { +func OpenSPV(remoteNode string, hfn string, + inTs *TxStore, p *chaincfg.Params) (SPVCon, error) { + // create new SPVCon + var s SPVCon + + // assign network parameters to SPVCon + s.param = p + // open header file err := s.openHeaderFile(headerFileName) if err != nil { - return err + return s, err } // open TCP connection s.con, err = net.Dial("tcp", remoteNode) if err != nil { - return err + return s, err } + // assign version bits for local node s.localVersion = VERSION - s.netType = NETVERSION + // transaction store for this SPV connection s.TS = inTs myMsgVer, err := wire.NewMsgVersionFromConn(s.con, 0, 0) if err != nil { - return err + return s, err } err = myMsgVer.AddUserAgent("test", "zero") if err != nil { - return err + return s, err } // must set this to enable SPV stuff myMsgVer.AddService(wire.SFNodeBloom) // this actually sends - n, err := wire.WriteMessageN(s.con, myMsgVer, s.localVersion, s.netType) + n, err := wire.WriteMessageN(s.con, myMsgVer, s.localVersion, s.param.Net) if err != nil { - return err + return s, err } s.WBytes += uint64(n) log.Printf("wrote %d byte version message to %s\n", n, s.con.RemoteAddr().String()) - n, m, b, err := wire.ReadMessageN(s.con, s.localVersion, s.netType) + n, m, b, err := wire.ReadMessageN(s.con, s.localVersion, s.param.Net) if err != nil { - return err + return s, err } s.RBytes += uint64(n) log.Printf("got %d byte response %x\n command: %s\n", n, b, m.Command()) @@ -99,9 +104,9 @@ func (s *SPVCon) Open(remoteNode string, hfn string, inTs *TxStore) error { mv.ProtocolVersion, mv.ProtocolVersion) mva := wire.NewMsgVerAck() - n, err = wire.WriteMessageN(s.con, mva, s.localVersion, s.netType) + n, err = wire.WriteMessageN(s.con, mva, s.localVersion, s.param.Net) if err != nil { - return err + return s, err } s.WBytes += uint64(n) @@ -110,7 +115,7 @@ func (s *SPVCon) Open(remoteNode string, hfn string, inTs *TxStore) error { s.outMsgQueue = make(chan wire.Message) go s.outgoingMessageHandler() - return nil + return s, nil } func (s *SPVCon) openHeaderFile(hfn string) error { @@ -118,7 +123,7 @@ func (s *SPVCon) openHeaderFile(hfn string) error { if err != nil { if os.IsNotExist(err) { var b bytes.Buffer - err = params.GenesisBlock.Header.Serialize(&b) + err = s.param.GenesisBlock.Header.Serialize(&b) if err != nil { return err } @@ -252,7 +257,7 @@ func (s *SPVCon) IngestHeaders(m *wire.MsgHeaders) (bool, error) { // advance chain tip tip++ // check last header - worked := CheckHeader(s.headerFile, tip, params) + worked := CheckHeader(s.headerFile, tip, s.param) if !worked { if endPos < 8080 { // jeez I give up, back to genesis @@ -299,15 +304,3 @@ func (s *SPVCon) AskForMerkBlocks(current, last uint32) error { return nil } - -func sendMBReq(cn net.Conn, blkhash wire.ShaHash) error { - iv1 := wire.NewInvVect(wire.InvTypeFilteredBlock, &blkhash) - gdataB := wire.NewMsgGetData() - _ = gdataB.AddInvVect(iv1) - n, err := wire.WriteMessageN(cn, gdataB, VERSION, NETVERSION) - if err != nil { - return err - } - log.Printf("sent %d byte block request\n", n) - return nil -} diff --git a/uspv/msghandler.go b/uspv/msghandler.go index 76e7f1b9..b0db4b01 100644 --- a/uspv/msghandler.go +++ b/uspv/msghandler.go @@ -9,7 +9,7 @@ import ( func (s *SPVCon) incomingMessageHandler() { for { - n, xm, _, err := wire.ReadMessageN(s.con, s.localVersion, s.netType) + n, xm, _, err := wire.ReadMessageN(s.con, s.localVersion, s.param.Net) if err != nil { log.Printf("ReadMessageN error. Disconnecting: %s\n", err.Error()) return @@ -79,7 +79,7 @@ func (s *SPVCon) incomingMessageHandler() { func (s *SPVCon) outgoingMessageHandler() { for { msg := <-s.outMsgQueue - n, err := wire.WriteMessageN(s.con, msg, s.localVersion, s.netType) + n, err := wire.WriteMessageN(s.con, msg, s.localVersion, s.param.Net) if err != nil { log.Printf("Write message error: %s", err.Error()) } From 7dbf4a4a4fe2065eecfce506d8878dd5ff97b9ee Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Tue, 19 Jan 2016 01:33:58 -0800 Subject: [PATCH 02/30] update uspv, add function for height from header still a bunch of problems getting utxo sets --- uspv/eight333.go | 40 ++++++++++++++++++++++++++++++++++++++++ uspv/mblock.go | 3 +++ uspv/msghandler.go | 12 ++++++++++-- uspv/txstore.go | 34 ++++++++++++++++++---------------- 4 files changed, 71 insertions(+), 18 deletions(-) diff --git a/uspv/eight333.go b/uspv/eight333.go index 75f8c52e..b20a21ea 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -156,6 +156,45 @@ func (s *SPVCon) SendFilter(f *bloom.Filter) { return } +// HeightFromHeader gives you the block height given a 80 byte block header +// seems like looking for the merkle root is the best way to do this +func (s *SPVCon) HeightFromHeader(query wire.BlockHeader) (uint32, error) { + // start from the most recent and work back in time; even though that's + // kind of annoying it's probably a lot faster since things tend to have + // happened recently. + + // seek to last header + lastPos, err := s.headerFile.Seek(-80, os.SEEK_END) + if err != nil { + return 0, err + } + height := lastPos / 80 + + var current wire.BlockHeader + + for height > 0 { + // grab header from disk + err = current.Deserialize(s.headerFile) + if err != nil { + return 0, err + } + // check if merkle roots match + if current.MerkleRoot.IsEqual(&query.MerkleRoot) { + // if they do, great, return height + return uint32(height), nil + } + // skip back one header (2 because we just read one) + _, err = s.headerFile.Seek(-160, os.SEEK_CUR) + if err != nil { + return 0, err + } + // decrement height + height-- + } + // finished for loop without finding match + return 0, fmt.Errorf("Header not found on disk") +} + func (s *SPVCon) AskForHeaders() error { var hdr wire.BlockHeader ghdr := wire.NewMsgGetHeaders() @@ -201,6 +240,7 @@ func (s *SPVCon) AskForHeaders() error { func (s *SPVCon) IngestHeaders(m *wire.MsgHeaders) (bool, error) { var err error + // seek to last header _, err = s.headerFile.Seek(-80, os.SEEK_END) if err != nil { return false, err diff --git a/uspv/mblock.go b/uspv/mblock.go index dc71608c..6200f0c2 100644 --- a/uspv/mblock.go +++ b/uspv/mblock.go @@ -77,6 +77,9 @@ func checkMBlock(m *wire.MsgMerkleBlock) ([]*wire.ShaHash, error) { if len(m.Flags) == 0 { return nil, fmt.Errorf("No flag bits") } + if len(m.Hashes) < 2 { // nothing but the merkle root + return nil, nil // nothin. + } var s []merkleNode // the stack var r []*wire.ShaHash // slice to return; txids we care about diff --git a/uspv/msghandler.go b/uspv/msghandler.go index b0db4b01..ab431d94 100644 --- a/uspv/msghandler.go +++ b/uspv/msghandler.go @@ -39,13 +39,21 @@ func (s *SPVCon) incomingMessageHandler() { if err != nil { log.Printf("Merkle block error: %s\n", err.Error()) return - // continue } fmt.Printf(" got %d txs ", len(txids)) // fmt.Printf(" = got %d txs from block %s\n", // len(txids), m.Header.BlockSha().String()) + var height uint32 + if len(txids) > 0 { + // make sure block is in our store before adding txs + height, err = s.HeightFromHeader(m.Header) + if err != nil { + log.Printf("Merkle block height error: %s\n", err.Error()) + continue + } + } for _, txid := range txids { - err := s.TS.AddTxid(txid) + err := s.TS.AddTxid(txid, height) if err != nil { log.Printf("Txid store error: %s\n", err.Error()) } diff --git a/uspv/txstore.go b/uspv/txstore.go index 7d4f4e5a..b847479c 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -11,10 +11,11 @@ import ( ) type TxStore struct { - KnownTxids []*wire.ShaHash - Utxos []Utxo // stacks on stacks - Sum int64 // racks on racks - Adrs []MyAdr // endeavouring to acquire capital + OKTxids map[wire.ShaHash]uint32 // known good txids and their heights + + Utxos []Utxo // stacks on stacks + Sum int64 // racks on racks + Adrs []MyAdr // endeavouring to acquire capital } type Utxo struct { // cash money. @@ -30,6 +31,12 @@ type MyAdr struct { // an address I have the private key for KeyIdx uint32 // index for private key needed to sign / spend } +func NewTxStore() TxStore { + var txs TxStore + txs.OKTxids = make(map[wire.ShaHash]uint32) + return txs +} + // add addresses into the TxStore func (t *TxStore) AddAdr(a btcutil.Address, kidx uint32) { var ma MyAdr @@ -40,11 +47,11 @@ func (t *TxStore) AddAdr(a btcutil.Address, kidx uint32) { } // add txid of interest -func (t *TxStore) AddTxid(txid *wire.ShaHash) error { +func (t *TxStore) AddTxid(txid *wire.ShaHash, height uint32) error { if txid == nil { return fmt.Errorf("tried to add nil txid") } - t.KnownTxids = append(t.KnownTxids, txid) + t.OKTxids[*txid] = height return nil } @@ -62,19 +69,13 @@ func (t *TxStore) GimmeFilter() (*bloom.Filter, error) { // Ingest a tx into wallet, dealing with both gains and losses func (t *TxStore) IngestTx(tx *wire.MsgTx) error { - var match bool inTxid := tx.TxSha() - for _, ktxid := range t.KnownTxids { - if inTxid.IsEqual(ktxid) { - match = true - break // found tx match, - } - } - if !match { + height, ok := t.OKTxids[inTxid] + if !ok { return fmt.Errorf("we don't care about tx %s", inTxid.String()) } - err := t.AbsorbTx(tx) + err := t.AbsorbTx(tx, height) if err != nil { return err } @@ -87,7 +88,7 @@ func (t *TxStore) IngestTx(tx *wire.MsgTx) error { } // Absorb money into wallet from a tx -func (t *TxStore) AbsorbTx(tx *wire.MsgTx) error { +func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height uint32) error { if tx == nil { return fmt.Errorf("Tried to add nil tx") } @@ -102,6 +103,7 @@ func (t *TxStore) AbsorbTx(tx *wire.MsgTx) error { hits++ acq += out.Value var newu Utxo + newu.AtHeight = height newu.KeyIdx = a.KeyIdx newu.Txo = *out From 709b8a05cd0986860ec4b2772cb9155dcdffba44 Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Tue, 19 Jan 2016 10:59:13 -0800 Subject: [PATCH 03/30] oh right, another fun thing I forgot about When there's only the coinbase tx, the merkle root = the coinbase txid. Needless complication in bitcoin == job security? --- uspv/mblock.go | 3 --- uspv/txstore.go | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/uspv/mblock.go b/uspv/mblock.go index 6200f0c2..dc71608c 100644 --- a/uspv/mblock.go +++ b/uspv/mblock.go @@ -77,9 +77,6 @@ func checkMBlock(m *wire.MsgMerkleBlock) ([]*wire.ShaHash, error) { if len(m.Flags) == 0 { return nil, fmt.Errorf("No flag bits") } - if len(m.Hashes) < 2 { // nothing but the merkle root - return nil, nil // nothin. - } var s []merkleNode // the stack var r []*wire.ShaHash // slice to return; txids we care about diff --git a/uspv/txstore.go b/uspv/txstore.go index b847479c..02f6f73b 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -5,6 +5,8 @@ import ( "fmt" "log" + "li.lan/labs/uwallet" + "github.com/btcsuite/btcd/wire" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcutil/bloom" @@ -72,6 +74,7 @@ func (t *TxStore) IngestTx(tx *wire.MsgTx) error { inTxid := tx.TxSha() height, ok := t.OKTxids[inTxid] if !ok { + log.Printf("False postive tx? %s", uwallet.TxToString(tx)) return fmt.Errorf("we don't care about tx %s", inTxid.String()) } From ece5dc9d2da5a70a6ac280f539d378947464ef88 Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Tue, 19 Jan 2016 11:59:01 -0800 Subject: [PATCH 04/30] add txtostring --- uspv/txstore.go | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/uspv/txstore.go b/uspv/txstore.go index 02f6f73b..9b1d4179 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -5,8 +5,6 @@ import ( "fmt" "log" - "li.lan/labs/uwallet" - "github.com/btcsuite/btcd/wire" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcutil/bloom" @@ -74,7 +72,7 @@ func (t *TxStore) IngestTx(tx *wire.MsgTx) error { inTxid := tx.TxSha() height, ok := t.OKTxids[inTxid] if !ok { - log.Printf("False postive tx? %s", uwallet.TxToString(tx)) + log.Printf("False postive tx? %s", TxToString(tx)) return fmt.Errorf("we don't care about tx %s", inTxid.String()) } @@ -147,3 +145,21 @@ func (t *TxStore) ExpellTx(tx *wire.MsgTx) error { t.Sum -= loss return nil } + +// TxToString prints out some info about a transaction. for testing / debugging +func TxToString(tx *wire.MsgTx) string { + str := "\t\t\t - Tx - \n" + for i, in := range tx.TxIn { + str += fmt.Sprintf("Input %d: %s\n", i, in.PreviousOutPoint.String()) + str += fmt.Sprintf("SigScript for input %d: %x\n", i, in.SignatureScript) + } + for i, out := range tx.TxOut { + if out != nil { + str += fmt.Sprintf("\toutput %d script: %x amt: %d\n", + i, out.PkScript, out.Value) + } else { + str += fmt.Sprintf("output %d nil (WARNING)\n", i) + } + } + return str +} From 58c8d32ac5d76dc40672f7580a7289f3c0964ccc Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Tue, 19 Jan 2016 14:23:18 -0800 Subject: [PATCH 05/30] add a height channel to keep track of incoming merkleblocks --- uspv/eight333.go | 36 ++++++++++++++++++++++++++++++++---- uspv/msghandler.go | 24 ++++++++++++++---------- uspv/txstore.go | 1 + 3 files changed, 47 insertions(+), 14 deletions(-) diff --git a/uspv/eight333.go b/uspv/eight333.go index b20a21ea..c0e467e2 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -38,8 +38,11 @@ type SPVCon struct { WBytes uint64 // total bytes written RBytes uint64 // total bytes read - TS *TxStore - param *chaincfg.Params + TS *TxStore // transaction store to write to + param *chaincfg.Params // network parameters (testnet3, testnetL) + + // mBlockQueue is for keeping track of what height we've requested. + mBlockQueue chan RootAndHeight } func OpenSPV(remoteNode string, hfn string, @@ -114,7 +117,7 @@ func OpenSPV(remoteNode string, hfn string, go s.incomingMessageHandler() s.outMsgQueue = make(chan wire.Message) go s.outgoingMessageHandler() - + s.mBlockQueue = make(chan RootAndHeight, 10) // queue of 10 requests? more? return s, nil } @@ -319,27 +322,52 @@ func (s *SPVCon) IngestHeaders(m *wire.MsgHeaders) (bool, error) { return true, nil } +// RootAndHeight is needed instead of just height in case a fullnode +// responds abnormally (?) by sending out of order merkleblocks. +// we cache a merkleroot:height pair in the queue so we don't have to +// look them up from the disk. +type RootAndHeight struct { + root wire.ShaHash + height uint32 +} + +// NewRootAndHeight saves like 2 lines. +func NewRootAndHeight(r wire.ShaHash, h uint32) (rah RootAndHeight) { + rah.root = r + rah.height = h + return +} + +// AskForMerkBlocks requests blocks from current to last +// right now this asks for 1 block per getData message. +// Maybe it's faster to ask for many in a each message? func (s *SPVCon) AskForMerkBlocks(current, last uint32) error { var hdr wire.BlockHeader _, err := s.headerFile.Seek(int64(current*80), os.SEEK_SET) if err != nil { return err } + // loop through all heights where we want merkleblocks. for current < last { + // load header from file err = hdr.Deserialize(s.headerFile) if err != nil { return err } - current++ bHash := hdr.BlockSha() + // create inventory we're asking for iv1 := wire.NewInvVect(wire.InvTypeFilteredBlock, &bHash) gdataMsg := wire.NewMsgGetData() + // add inventory err = gdataMsg.AddInvVect(iv1) if err != nil { return err } + rah := NewRootAndHeight(hdr.MerkleRoot, current) s.outMsgQueue <- gdataMsg + s.mBlockQueue <- rah // push height and mroot of requested block on queue + current++ } return nil diff --git a/uspv/msghandler.go b/uspv/msghandler.go index ab431d94..2c61f9e5 100644 --- a/uspv/msghandler.go +++ b/uspv/msghandler.go @@ -43,22 +43,26 @@ func (s *SPVCon) incomingMessageHandler() { fmt.Printf(" got %d txs ", len(txids)) // fmt.Printf(" = got %d txs from block %s\n", // len(txids), m.Header.BlockSha().String()) - var height uint32 - if len(txids) > 0 { - // make sure block is in our store before adding txs - height, err = s.HeightFromHeader(m.Header) - if err != nil { - log.Printf("Merkle block height error: %s\n", err.Error()) - continue - } + // var height uint32 + // if len(txids) > 0 { + // make sure block is in our store before adding txs + // height, err = s.HeightFromHeader(m.Header) + // height = 20000 + // if err != nil { + //log.Printf("Merkle block height error: %s\n", err.Error()) + // continue + // } + // } + rah := <-s.mBlockQueue // pop height off mblock queue + if !rah.root.IsEqual(&m.Header.MerkleRoot) { + log.Printf("out of order error") } for _, txid := range txids { - err := s.TS.AddTxid(txid, height) + err := s.TS.AddTxid(txid, rah.height) if err != nil { log.Printf("Txid store error: %s\n", err.Error()) } } - // nextReq <- true case *wire.MsgHeaders: moar, err := s.IngestHeaders(m) diff --git a/uspv/txstore.go b/uspv/txstore.go index 9b1d4179..e6c0ebad 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -51,6 +51,7 @@ func (t *TxStore) AddTxid(txid *wire.ShaHash, height uint32) error { if txid == nil { return fmt.Errorf("tried to add nil txid") } + log.Printf("added %s at height %d\n", txid.String(), height) t.OKTxids[*txid] = height return nil } From 0b5daa9b2b484e5bb4b3b605830211778732e183 Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Tue, 19 Jan 2016 20:02:18 -0800 Subject: [PATCH 06/30] add utxodb for on disk storage of transactions. --- uspv/eight333.go | 29 ++++++++- uspv/msghandler.go | 12 +--- uspv/txstore.go | 16 ++--- uspv/utxodb.go | 146 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 184 insertions(+), 19 deletions(-) create mode 100644 uspv/utxodb.go diff --git a/uspv/eight333.go b/uspv/eight333.go index c0e467e2..2677c86d 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -8,6 +8,7 @@ import ( "net" "os" + "github.com/boltdb/bolt" "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/wire" "github.com/btcsuite/btcutil/bloom" @@ -69,6 +70,10 @@ func OpenSPV(remoteNode string, hfn string, s.localVersion = VERSION // transaction store for this SPV connection + err = inTs.OpenDB("utxo.db") + if err != nil { + return s, err + } s.TS = inTs myMsgVer, err := wire.NewMsgVersionFromConn(s.con, 0, 0) @@ -117,10 +122,31 @@ func OpenSPV(remoteNode string, hfn string, go s.incomingMessageHandler() s.outMsgQueue = make(chan wire.Message) go s.outgoingMessageHandler() - s.mBlockQueue = make(chan RootAndHeight, 10) // queue of 10 requests? more? + s.mBlockQueue = make(chan RootAndHeight, 32) // queue depth 32 is a thing + return s, nil } +func (ts *TxStore) OpenDB(filename string) error { + var err error + ts.StateDB, err = bolt.Open(filename, 0644, nil) + if err != nil { + return err + } + // create buckets if they're not already there + return ts.StateDB.Update(func(tx *bolt.Tx) error { + _, err = tx.CreateBucketIfNotExists(BKTUtxos) + if err != nil { + return err + } + _, err = tx.CreateBucketIfNotExists(BKTOld) + if err != nil { + return err + } + return nil + }) +} + func (s *SPVCon) openHeaderFile(hfn string) error { _, err := os.Stat(hfn) if err != nil { @@ -138,7 +164,6 @@ func (s *SPVCon) openHeaderFile(hfn string) error { hfn) } } - s.headerFile, err = os.OpenFile(hfn, os.O_RDWR, 0600) if err != nil { return err diff --git a/uspv/msghandler.go b/uspv/msghandler.go index 2c61f9e5..505dff3b 100644 --- a/uspv/msghandler.go +++ b/uspv/msghandler.go @@ -43,17 +43,9 @@ func (s *SPVCon) incomingMessageHandler() { fmt.Printf(" got %d txs ", len(txids)) // fmt.Printf(" = got %d txs from block %s\n", // len(txids), m.Header.BlockSha().String()) - // var height uint32 - // if len(txids) > 0 { - // make sure block is in our store before adding txs - // height, err = s.HeightFromHeader(m.Header) - // height = 20000 - // if err != nil { - //log.Printf("Merkle block height error: %s\n", err.Error()) - // continue - // } - // } rah := <-s.mBlockQueue // pop height off mblock queue + // this verifies order, and also that the returned header fits + // into our SPV header file if !rah.root.IsEqual(&m.Header.MerkleRoot) { log.Printf("out of order error") } diff --git a/uspv/txstore.go b/uspv/txstore.go index e6c0ebad..a71ce36c 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -5,6 +5,7 @@ import ( "fmt" "log" + "github.com/boltdb/bolt" "github.com/btcsuite/btcd/wire" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcutil/bloom" @@ -13,17 +14,18 @@ import ( type TxStore struct { OKTxids map[wire.ShaHash]uint32 // known good txids and their heights - Utxos []Utxo // stacks on stacks - Sum int64 // racks on racks - Adrs []MyAdr // endeavouring to acquire capital + Utxos []Utxo // stacks on stacks + Sum int64 // racks on racks + Adrs []MyAdr // endeavouring to acquire capital + StateDB *bolt.DB // place to write all this down } type Utxo struct { // cash money. // combo of outpoint and txout which has all the info needed to spend - Op wire.OutPoint - Txo wire.TxOut - AtHeight uint32 // block height where this tx was confirmed, 0 for unconf - KeyIdx uint32 // index for private key needed to sign / spend + AtHeight uint32 // block height where this tx was confirmed, 0 for unconf + KeyIdx uint32 // index for private key needed to sign / spend + Op wire.OutPoint // where + Txo wire.TxOut // what } type MyAdr struct { // an address I have the private key for diff --git a/uspv/utxodb.go b/uspv/utxodb.go new file mode 100644 index 00000000..ee787b80 --- /dev/null +++ b/uspv/utxodb.go @@ -0,0 +1,146 @@ +package uspv + +import ( + "bytes" + "encoding/binary" + "fmt" + + "github.com/boltdb/bolt" +) + +var ( + BKTUtxos = []byte("DuffelBag") // leave the rest to collect interest + BKTOld = []byte("SpentTxs") // for bookkeeping + KEYState = []byte("LastUpdate") // last state of DB +) + +func (u *Utxo) SaveToDB(dbx *bolt.DB) error { + return dbx.Update(func(tx *bolt.Tx) error { + duf := tx.Bucket(BKTUtxos) + b, err := u.ToBytes() + if err != nil { + return err + } + // key : val is txid:everything else + return duf.Put(b[:36], b[36:]) + }) +} + +func (ts *TxStore) LoadUtxos() error { + var kc, vc []byte + + err := ts.StateDB.View(func(tx *bolt.Tx) error { + duf := tx.Bucket(BKTUtxos) + if duf == nil { + return fmt.Errorf("no duffel bag") + } + spent := tx.Bucket(BKTOld) + if spent == nil { + return fmt.Errorf("no spenttx bucket") + } + + duf.ForEach(func(k, v []byte) error { + // have to copy these here, otherwise append will crash it. + // not quite sure why but append does weird stuff I guess. + copy(kc, k) + copy(vc, v) + if spent.Get(kc) == nil { // if it's not in the spent bucket + // create a new utxo + newU, err := UtxoFromBytes(append(kc, vc...)) + if err != nil { + return err + } + // and add it to ram + ts.Utxos = append(ts.Utxos, newU) + } + return nil + }) + return nil + }) + if err != nil { + return err + } + return nil +} + +// ToBytes turns a Utxo into some bytes. +// note that the txid is the first 36 bytes and in our use cases will be stripped +// off, but is left here for other applications +func (u *Utxo) ToBytes() ([]byte, error) { + var buf bytes.Buffer + // write 32 byte txid of the utxo + _, err := buf.Write(u.Op.Hash.Bytes()) + if err != nil { + return nil, err + } + // write 4 byte outpoint index within the tx to spend + err = binary.Write(&buf, binary.BigEndian, u.Op.Index) + if err != nil { + return nil, err + } + // write 4 byte height of utxo + err = binary.Write(&buf, binary.BigEndian, u.AtHeight) + if err != nil { + return nil, err + } + // write 4 byte key index of utxo + err = binary.Write(&buf, binary.BigEndian, u.KeyIdx) + if err != nil { + return nil, err + } + // write 8 byte amount of money at the utxo + err = binary.Write(&buf, binary.BigEndian, u.Txo.Value) + if err != nil { + return nil, err + } + + // write variable length (usually like 25 byte) pkscript + _, err = buf.Write(u.Txo.PkScript) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +// UtxoFromBytes turns bytes into a Utxo. Note it wants the txid and outindex +// in the first 36 bytes, which isn't stored that way in the boldDB, +// but can be easily appended. +func UtxoFromBytes(b []byte) (Utxo, error) { + var u Utxo + if b == nil { + return u, fmt.Errorf("nil input slice") + } + buf := bytes.NewBuffer(b) + if buf.Len() < 52 { // minimum 52 bytes with no pkscript + return u, fmt.Errorf("Got %d bytes for sender, expect > 52", buf.Len()) + } + // read 32 byte txid + err := u.Op.Hash.SetBytes(buf.Next(32)) + if err != nil { + return u, err + } + // read 4 byte outpoint index within the tx to spend + err = binary.Read(buf, binary.BigEndian, &u.Op.Index) + if err != nil { + return u, err + } + // read 4 byte height of utxo + err = binary.Read(buf, binary.BigEndian, &u.AtHeight) + if err != nil { + return u, err + } + // read 4 byte key index of utxo + err = binary.Read(buf, binary.BigEndian, &u.KeyIdx) + if err != nil { + return u, err + } + // read 8 byte amount of money at the utxo + err = binary.Read(buf, binary.BigEndian, &u.Txo.Value) + if err != nil { + return u, err + } + // read variable length (usually like 25 byte) pkscript + u.Txo.PkScript = buf.Bytes() + return u, nil +} From 29d0c0cc6f3e83bfb914d65d0cd63fd6c2617d98 Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Tue, 19 Jan 2016 23:40:04 -0800 Subject: [PATCH 07/30] fix bolt slices. db read / write works. also switch heights from uint32 to int32, which I guess is what they use. Makes this thing last 2 billion blocks shorter. If we get past uint32 time. --- uspv/eight333.go | 22 +++++++++++++++++----- uspv/txstore.go | 10 +++++----- uspv/utxodb.go | 10 +++++----- 3 files changed, 27 insertions(+), 15 deletions(-) diff --git a/uspv/eight333.go b/uspv/eight333.go index 2677c86d..41a83f69 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -111,6 +111,9 @@ func OpenSPV(remoteNode string, hfn string, log.Printf("remote reports version %x (dec %d)\n", mv.ProtocolVersion, mv.ProtocolVersion) + // set remote height + s.remoteHeight = mv.LastBlock + mva := wire.NewMsgVerAck() n, err = wire.WriteMessageN(s.con, mva, s.localVersion, s.param.Net) if err != nil { @@ -118,9 +121,9 @@ func OpenSPV(remoteNode string, hfn string, } s.WBytes += uint64(n) - s.inMsgQueue = make(chan wire.Message) + s.inMsgQueue = make(chan wire.Message, 1) go s.incomingMessageHandler() - s.outMsgQueue = make(chan wire.Message) + s.outMsgQueue = make(chan wire.Message, 1) go s.outgoingMessageHandler() s.mBlockQueue = make(chan RootAndHeight, 32) // queue depth 32 is a thing @@ -353,11 +356,11 @@ func (s *SPVCon) IngestHeaders(m *wire.MsgHeaders) (bool, error) { // look them up from the disk. type RootAndHeight struct { root wire.ShaHash - height uint32 + height int32 } // NewRootAndHeight saves like 2 lines. -func NewRootAndHeight(r wire.ShaHash, h uint32) (rah RootAndHeight) { +func NewRootAndHeight(r wire.ShaHash, h int32) (rah RootAndHeight) { rah.root = r rah.height = h return @@ -366,8 +369,17 @@ func NewRootAndHeight(r wire.ShaHash, h uint32) (rah RootAndHeight) { // AskForMerkBlocks requests blocks from current to last // right now this asks for 1 block per getData message. // Maybe it's faster to ask for many in a each message? -func (s *SPVCon) AskForMerkBlocks(current, last uint32) error { +func (s *SPVCon) AskForMerkBlocks(current, last int32) error { var hdr wire.BlockHeader + // if last is 0, that means go as far as we can + if last == 0 { + n, err := s.headerFile.Seek(-80, os.SEEK_END) + if err != nil { + return err + } + last = int32(n / 80) + } + _, err := s.headerFile.Seek(int64(current*80), os.SEEK_SET) if err != nil { return err diff --git a/uspv/txstore.go b/uspv/txstore.go index a71ce36c..a0332ee5 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -12,7 +12,7 @@ import ( ) type TxStore struct { - OKTxids map[wire.ShaHash]uint32 // known good txids and their heights + OKTxids map[wire.ShaHash]int32 // known good txids and their heights Utxos []Utxo // stacks on stacks Sum int64 // racks on racks @@ -22,7 +22,7 @@ type TxStore struct { type Utxo struct { // cash money. // combo of outpoint and txout which has all the info needed to spend - AtHeight uint32 // block height where this tx was confirmed, 0 for unconf + AtHeight int32 // block height where this tx was confirmed, 0 for unconf KeyIdx uint32 // index for private key needed to sign / spend Op wire.OutPoint // where Txo wire.TxOut // what @@ -35,7 +35,7 @@ type MyAdr struct { // an address I have the private key for func NewTxStore() TxStore { var txs TxStore - txs.OKTxids = make(map[wire.ShaHash]uint32) + txs.OKTxids = make(map[wire.ShaHash]int32) return txs } @@ -49,7 +49,7 @@ func (t *TxStore) AddAdr(a btcutil.Address, kidx uint32) { } // add txid of interest -func (t *TxStore) AddTxid(txid *wire.ShaHash, height uint32) error { +func (t *TxStore) AddTxid(txid *wire.ShaHash, height int32) error { if txid == nil { return fmt.Errorf("tried to add nil txid") } @@ -92,7 +92,7 @@ func (t *TxStore) IngestTx(tx *wire.MsgTx) error { } // Absorb money into wallet from a tx -func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height uint32) error { +func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height int32) error { if tx == nil { return fmt.Errorf("Tried to add nil tx") } diff --git a/uspv/utxodb.go b/uspv/utxodb.go index ee787b80..abfb2f94 100644 --- a/uspv/utxodb.go +++ b/uspv/utxodb.go @@ -27,7 +27,6 @@ func (u *Utxo) SaveToDB(dbx *bolt.DB) error { } func (ts *TxStore) LoadUtxos() error { - var kc, vc []byte err := ts.StateDB.View(func(tx *bolt.Tx) error { duf := tx.Bucket(BKTUtxos) @@ -42,11 +41,12 @@ func (ts *TxStore) LoadUtxos() error { duf.ForEach(func(k, v []byte) error { // have to copy these here, otherwise append will crash it. // not quite sure why but append does weird stuff I guess. - copy(kc, k) - copy(vc, v) - if spent.Get(kc) == nil { // if it's not in the spent bucket + if spent.Get(k) == nil { // if it's not in the spent bucket // create a new utxo - newU, err := UtxoFromBytes(append(kc, vc...)) + x := make([]byte, len(k)+len(v)) + copy(x, k) + copy(x[len(k):], v) + newU, err := UtxoFromBytes(x) if err != nil { return err } From 1f40c7214ef1087b4f64025956b5b18711ec7aa7 Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Wed, 20 Jan 2016 01:23:47 -0800 Subject: [PATCH 08/30] good enough storage of spent txs --- uspv/txstore.go | 15 +++++++++++---- uspv/utxodb.go | 49 +++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/uspv/txstore.go b/uspv/txstore.go index a0332ee5..7c1abee3 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -83,7 +83,7 @@ func (t *TxStore) IngestTx(tx *wire.MsgTx) error { if err != nil { return err } - err = t.ExpellTx(tx) + err = t.ExpellTx(tx, height) if err != nil { return err } @@ -115,7 +115,10 @@ func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height int32) error { newop.Hash = tx.TxSha() newop.Index = uint32(i) newu.Op = newop - + err := newu.SaveToDB(t.StateDB) + if err != nil { + return err + } t.Utxos = append(t.Utxos, newu) break } @@ -127,7 +130,7 @@ func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height int32) error { } // Expell money from wallet due to a tx -func (t *TxStore) ExpellTx(tx *wire.MsgTx) error { +func (t *TxStore) ExpellTx(tx *wire.MsgTx, height int32) error { if tx == nil { return fmt.Errorf("Tried to add nil tx") } @@ -139,7 +142,11 @@ func (t *TxStore) ExpellTx(tx *wire.MsgTx) error { if myutxo.Op == in.PreviousOutPoint { hits++ loss += myutxo.Txo.Value - // delete from my utxo set + err := t.MarkSpent(&myutxo.Op, height, tx) + if err != nil { + return err + } + // delete from my in-ram utxo set t.Utxos = append(t.Utxos[:i], t.Utxos[i+1:]...) } } diff --git a/uspv/utxodb.go b/uspv/utxodb.go index abfb2f94..32fcf06b 100644 --- a/uspv/utxodb.go +++ b/uspv/utxodb.go @@ -5,6 +5,8 @@ import ( "encoding/binary" "fmt" + "github.com/btcsuite/btcd/wire" + "github.com/boltdb/bolt" ) @@ -26,8 +28,32 @@ func (u *Utxo) SaveToDB(dbx *bolt.DB) error { }) } -func (ts *TxStore) LoadUtxos() error { +func (ts *TxStore) MarkSpent(op *wire.OutPoint, h int32, stx *wire.MsgTx) error { + // we write in key = outpoint (32 hash, 4 index) + // value = spending txid + // if we care about the spending tx we can store that in another bucket. + return ts.StateDB.Update(func(tx *bolt.Tx) error { + old := tx.Bucket(BKTOld) + opb, err := outPointToBytes(op) + if err != nil { + return err + } + var buf bytes.Buffer + err = binary.Write(&buf, binary.BigEndian, h) + if err != nil { + return err + } + sha := stx.TxSha() + err = old.Put(opb, sha.Bytes()) // write k:v outpoint:txid + if err != nil { + return err + } + return nil + }) +} + +func (ts *TxStore) LoadUtxos() error { err := ts.StateDB.View(func(tx *bolt.Tx) error { duf := tx.Bucket(BKTUtxos) if duf == nil { @@ -41,7 +67,8 @@ func (ts *TxStore) LoadUtxos() error { duf.ForEach(func(k, v []byte) error { // have to copy these here, otherwise append will crash it. // not quite sure why but append does weird stuff I guess. - if spent.Get(k) == nil { // if it's not in the spent bucket + stx := spent.Get(k) + if stx == nil { // if it's not in the spent bucket // create a new utxo x := make([]byte, len(k)+len(v)) copy(x, k) @@ -52,6 +79,9 @@ func (ts *TxStore) LoadUtxos() error { } // and add it to ram ts.Utxos = append(ts.Utxos, newU) + } else { + fmt.Printf("had utxo %x but spent by tx %x...\n", + k, stx[:8]) } return nil }) @@ -63,6 +93,21 @@ func (ts *TxStore) LoadUtxos() error { return nil } +// outPointToBytes turns an outpoint into 36 bytes. +func outPointToBytes(op *wire.OutPoint) ([]byte, error) { + var buf bytes.Buffer + _, err := buf.Write(op.Hash.Bytes()) + if err != nil { + return nil, err + } + // write 4 byte outpoint index within the tx to spend + err = binary.Write(&buf, binary.BigEndian, op.Index) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + // ToBytes turns a Utxo into some bytes. // note that the txid is the first 36 bytes and in our use cases will be stripped // off, but is left here for other applications From 73bbb29026903f0048988e8d9492097d32beb830 Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Wed, 20 Jan 2016 21:08:05 -0800 Subject: [PATCH 09/30] add keyfileio to manage private key storage on disk --- uspv/eight333.go | 25 +----- uspv/keyfileio.go | 191 ++++++++++++++++++++++++++++++++++++++++++++++ uspv/txstore.go | 9 +++ uspv/utxodb.go | 37 +++++++++ 4 files changed, 239 insertions(+), 23 deletions(-) create mode 100644 uspv/keyfileio.go diff --git a/uspv/eight333.go b/uspv/eight333.go index 41a83f69..7096bef3 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -8,7 +8,6 @@ import ( "net" "os" - "github.com/boltdb/bolt" "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/wire" "github.com/btcsuite/btcutil/bloom" @@ -46,7 +45,7 @@ type SPVCon struct { mBlockQueue chan RootAndHeight } -func OpenSPV(remoteNode string, hfn string, +func OpenSPV(remoteNode string, hfn, tsfn string, inTs *TxStore, p *chaincfg.Params) (SPVCon, error) { // create new SPVCon var s SPVCon @@ -70,7 +69,7 @@ func OpenSPV(remoteNode string, hfn string, s.localVersion = VERSION // transaction store for this SPV connection - err = inTs.OpenDB("utxo.db") + err = inTs.OpenDB(tsfn) if err != nil { return s, err } @@ -130,26 +129,6 @@ func OpenSPV(remoteNode string, hfn string, return s, nil } -func (ts *TxStore) OpenDB(filename string) error { - var err error - ts.StateDB, err = bolt.Open(filename, 0644, nil) - if err != nil { - return err - } - // create buckets if they're not already there - return ts.StateDB.Update(func(tx *bolt.Tx) error { - _, err = tx.CreateBucketIfNotExists(BKTUtxos) - if err != nil { - return err - } - _, err = tx.CreateBucketIfNotExists(BKTOld) - if err != nil { - return err - } - return nil - }) -} - func (s *SPVCon) openHeaderFile(hfn string) error { _, err := os.Stat(hfn) if err != nil { diff --git a/uspv/keyfileio.go b/uspv/keyfileio.go new file mode 100644 index 00000000..0bebe4de --- /dev/null +++ b/uspv/keyfileio.go @@ -0,0 +1,191 @@ +package uspv + +import ( + "crypto/rand" + "encoding/hex" + "fmt" + "io/ioutil" + "os" + "strings" + + "github.com/btcsuite/btcd/chaincfg" + "github.com/btcsuite/btcutil/hdkeychain" + "github.com/howeyc/gopass" + "golang.org/x/crypto/nacl/secretbox" + "golang.org/x/crypto/scrypt" +) + +// warning! look at those imports! crypto! hopefully this works! + +/* on-disk stored keys are 32bytes. This is good for ed25519 private keys, +for seeds for bip32, for individual secp256k1 priv keys, and so on. +32 bytes is enough for anyone. +If you want fewer bytes, put some zeroes at the end */ + +// LoadKeyFromFileInteractive opens the file 'filename' and presents a +// keyboard prompt for the passphrase to decrypt it. It returns the +// key if decryption works, or errors out. +func LoadKeyFromFileInteractive(filename string) (*[32]byte, error) { + a, err := os.Stat(filename) + if err != nil { + return new([32]byte), err + } + if a.Size() < 80 { // there can't be a password... + return LoadKeyFromFileArg(filename, nil) + } + fmt.Printf("passphrase: ") + pass := gopass.GetPasswd() + fmt.Printf("\n") + return LoadKeyFromFileArg(filename, pass) +} + +// LoadKeyFromFileArg opens the file and returns the key. If the key is +// unencrypted it will ignore the password argument. +func LoadKeyFromFileArg(filename string, pass []byte) (*[32]byte, error) { + priv32 := new([32]byte) + keyhex, err := ioutil.ReadFile(filename) + if err != nil { + return priv32, err + } + keyhex = []byte(strings.TrimSpace(string(keyhex))) + enckey, err := hex.DecodeString(string(keyhex)) + if err != nil { + return priv32, err + } + + if len(enckey) == 32 { // UNencrypted key, length 32 + fmt.Printf("WARNING!! Key file not encrypted!!\n") + fmt.Printf("Anyone who can read the key file can take everything!\n") + fmt.Printf("You should start over and use a good passphrase!\n") + copy(priv32[:], enckey[:]) + return priv32, nil + } + // enckey should be 72 bytes. 24 for scrypt salt/box nonce, + // 16 for box auth + if len(enckey) != 72 { + return priv32, fmt.Errorf("Key length error for %s ", filename) + } + // enckey is actually encrypted, get derived key from pass and salt + // first extract salt + salt := new([24]byte) // salt (also nonce for secretbox) + dk32 := new([32]byte) // derived key array + copy(salt[:], enckey[:24]) // first 24 bytes are scrypt salt/box nonce + + dk, err := scrypt.Key(pass, salt[:], 16384, 8, 1, 32) // derive key + if err != nil { + return priv32, err + } + copy(dk32[:], dk[:]) // copy into fixed size array + + // nonce for secretbox is the same as scrypt salt. Seems fine. Really. + priv, worked := secretbox.Open(nil, enckey[24:], salt, dk32) + if worked != true { + return priv32, fmt.Errorf("Decryption failed for %s ", filename) + } + copy(priv32[:], priv[:]) //copy decrypted private key into array + + priv = nil // this probably doesn't do anything but... eh why not + return priv32, nil +} + +// saves a 32 byte key to file, prompting for passphrase. +// if user enters empty passphrase (hits enter twice), will be saved +// in the clear. +func SaveKeyToFileInteractive(filename string, priv32 *[32]byte) error { + var match bool + var pass1, pass2 []byte + for match != true { + fmt.Printf("passphrase: ") + pass1 = gopass.GetPasswd() + fmt.Printf("repeat passphrase: ") + pass2 = gopass.GetPasswd() + if string(pass1) == string(pass2) { + match = true + } else { + fmt.Printf("user input error. Try again gl hf dd.\n") + } + } + fmt.Printf("\n") + return SaveKeyToFileArg(filename, priv32, pass1) +} + +// saves a 32 byte key to a file, encrypting with pass. +// if pass is nil or zero length, doesn't encrypt and just saves in hex. +func SaveKeyToFileArg(filename string, priv32 *[32]byte, pass []byte) error { + if len(pass) == 0 { // zero-length pass, save unencrypted + keyhex := fmt.Sprintf("%x\n", priv32[:]) + err := ioutil.WriteFile(filename, []byte(keyhex), 0600) + if err != nil { + return err + } + fmt.Printf("WARNING!! Key file not encrypted!!\n") + fmt.Printf("Anyone who can read the key file can take everything!\n") + fmt.Printf("You should start over and use a good passphrase!\n") + fmt.Printf("Saved unencrypted key at %s\n", filename) + return nil + } + + salt := new([24]byte) // salt for scrypt / nonce for secretbox + dk32 := new([32]byte) // derived key from scrypt + + //get 24 random bytes for scrypt salt (and secretbox nonce) + _, err := rand.Read(salt[:]) + if err != nil { + return err + } + // next use the pass and salt to make a 32-byte derived key + dk, err := scrypt.Key(pass, salt[:], 16384, 8, 1, 32) + if err != nil { + return err + } + copy(dk32[:], dk[:]) + + enckey := append(salt[:], secretbox.Seal(nil, priv32[:], salt, dk32)...) + // enckey = append(salt, enckey...) + keyhex := fmt.Sprintf("%x\n", enckey) + + err = ioutil.WriteFile(filename, []byte(keyhex), 0600) + if err != nil { + return err + } + fmt.Printf("Wrote encrypted key to %s\n", filename) + return nil +} + +// ReadKeyFileToECPriv returns an extendedkey from a file. +// If there's no file there, it'll make one. If there's a password needed, +// it'll prompt for one. One stop function. +func ReadKeyFileToECPriv(filename string) (*hdkeychain.ExtendedKey, error) { + key32 := new([32]byte) + _, err := os.Stat(filename) + if err != nil { + if os.IsNotExist(err) { + // no key found, generate and save one + fmt.Printf("No file %s, generating.\n", filename) + rn, err := hdkeychain.GenerateSeed(32) + if err != nil { + return nil, err + } + copy(key32[:], rn[:]) + err = SaveKeyToFileInteractive(filename, key32) + if err != nil { + return nil, err + } + } else { + // unknown error, crash + fmt.Printf("unknown\n") + return nil, err + } + } + + key, err := LoadKeyFromFileInteractive(filename) + if err != nil { + return nil, err + } + + rootpriv, err := hdkeychain.NewMaster(key[:], &chaincfg.TestNet3Params) + if err != nil { + return nil, err + } + return rootpriv, nil +} diff --git a/uspv/txstore.go b/uspv/txstore.go index 7c1abee3..e666ed6c 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -5,10 +5,13 @@ import ( "fmt" "log" + "github.com/btcsuite/btcd/chaincfg" + "github.com/boltdb/bolt" "github.com/btcsuite/btcd/wire" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcutil/bloom" + "github.com/btcsuite/btcutil/hdkeychain" ) type TxStore struct { @@ -17,7 +20,13 @@ type TxStore struct { Utxos []Utxo // stacks on stacks Sum int64 // racks on racks Adrs []MyAdr // endeavouring to acquire capital + LastIdx uint32 // should equal len(Adrs) StateDB *bolt.DB // place to write all this down + // this is redundant with the SPVCon param... ugly and should be taken out + param *chaincfg.Params // network parameters (testnet3, testnetL) + + // From here, comes everything. It's a secret to everybody. + rootPrivKey *hdkeychain.ExtendedKey } type Utxo struct { // cash money. diff --git a/uspv/utxodb.go b/uspv/utxodb.go index 32fcf06b..e27e9140 100644 --- a/uspv/utxodb.go +++ b/uspv/utxodb.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/binary" "fmt" + "log" "github.com/btcsuite/btcd/wire" @@ -16,6 +17,42 @@ var ( KEYState = []byte("LastUpdate") // last state of DB ) +func (ts *TxStore) OpenDB(filename string) error { + var err error + ts.StateDB, err = bolt.Open(filename, 0644, nil) + if err != nil { + return err + } + // create buckets if they're not already there + return ts.StateDB.Update(func(tx *bolt.Tx) error { + _, err = tx.CreateBucketIfNotExists(BKTUtxos) + if err != nil { + return err + } + _, err = tx.CreateBucketIfNotExists(BKTOld) + if err != nil { + return err + } + return nil + }) +} + +func (ts *TxStore) PopulateAdrs(lastKey uint32) error { + for k := uint32(0); k < lastKey; k++ { + + priv, err := ts.rootPrivKey.Child(k) + if err != nil { + log.Fatal(err) + } + myadr, err := priv.Address(ts.param) + if err != nil { + log.Fatal(err) + } + fmt.Printf("made adr %s\n", myadr.String()) + ts.AddAdr(myadr, k) + } + return nil +} func (u *Utxo) SaveToDB(dbx *bolt.DB) error { return dbx.Update(func(tx *bolt.Tx) error { duf := tx.Bucket(BKTUtxos) From 278971936f5bd3faf72c1d9a317e4fee06228ad2 Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Wed, 20 Jan 2016 21:27:58 -0800 Subject: [PATCH 10/30] pass network parameters as function arg --- uspv/keyfileio.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/uspv/keyfileio.go b/uspv/keyfileio.go index 0bebe4de..ec359edf 100644 --- a/uspv/keyfileio.go +++ b/uspv/keyfileio.go @@ -155,7 +155,8 @@ func SaveKeyToFileArg(filename string, priv32 *[32]byte, pass []byte) error { // ReadKeyFileToECPriv returns an extendedkey from a file. // If there's no file there, it'll make one. If there's a password needed, // it'll prompt for one. One stop function. -func ReadKeyFileToECPriv(filename string) (*hdkeychain.ExtendedKey, error) { +func ReadKeyFileToECPriv( + filename string, p *chaincfg.Params) (*hdkeychain.ExtendedKey, error) { key32 := new([32]byte) _, err := os.Stat(filename) if err != nil { @@ -183,7 +184,7 @@ func ReadKeyFileToECPriv(filename string) (*hdkeychain.ExtendedKey, error) { return nil, err } - rootpriv, err := hdkeychain.NewMaster(key[:], &chaincfg.TestNet3Params) + rootpriv, err := hdkeychain.NewMaster(key[:], p) if err != nil { return nil, err } From b7a2c46ea670d37ed3c1131160c1e2fef2806a58 Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Wed, 20 Jan 2016 22:57:05 -0800 Subject: [PATCH 11/30] remove storage of pkscript; don't need it. --- uspv/txstore.go | 15 ++++++++------- uspv/utxodb.go | 12 ++---------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/uspv/txstore.go b/uspv/txstore.go index e666ed6c..036ca4fe 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -30,11 +30,12 @@ type TxStore struct { } type Utxo struct { // cash money. - // combo of outpoint and txout which has all the info needed to spend - AtHeight int32 // block height where this tx was confirmed, 0 for unconf - KeyIdx uint32 // index for private key needed to sign / spend - Op wire.OutPoint // where - Txo wire.TxOut // what + // all the info needed to spend + AtHeight int32 // block height where this tx was confirmed, 0 for unconf + KeyIdx uint32 // index for private key needed to sign / spend + Value int64 // higher is better + + Op wire.OutPoint // where } type MyAdr struct { // an address I have the private key for @@ -118,7 +119,7 @@ func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height int32) error { var newu Utxo newu.AtHeight = height newu.KeyIdx = a.KeyIdx - newu.Txo = *out + newu.Value = out.Value var newop wire.OutPoint newop.Hash = tx.TxSha() @@ -150,7 +151,7 @@ func (t *TxStore) ExpellTx(tx *wire.MsgTx, height int32) error { for i, myutxo := range t.Utxos { if myutxo.Op == in.PreviousOutPoint { hits++ - loss += myutxo.Txo.Value + loss += myutxo.Value err := t.MarkSpent(&myutxo.Op, height, tx) if err != nil { return err diff --git a/uspv/utxodb.go b/uspv/utxodb.go index e27e9140..ffa7ee00 100644 --- a/uspv/utxodb.go +++ b/uspv/utxodb.go @@ -171,13 +171,7 @@ func (u *Utxo) ToBytes() ([]byte, error) { return nil, err } // write 8 byte amount of money at the utxo - err = binary.Write(&buf, binary.BigEndian, u.Txo.Value) - if err != nil { - return nil, err - } - - // write variable length (usually like 25 byte) pkscript - _, err = buf.Write(u.Txo.PkScript) + err = binary.Write(&buf, binary.BigEndian, u.Value) if err != nil { return nil, err } @@ -218,11 +212,9 @@ func UtxoFromBytes(b []byte) (Utxo, error) { return u, err } // read 8 byte amount of money at the utxo - err = binary.Read(buf, binary.BigEndian, &u.Txo.Value) + err = binary.Read(buf, binary.BigEndian, &u.Value) if err != nil { return u, err } - // read variable length (usually like 25 byte) pkscript - u.Txo.PkScript = buf.Bytes() return u, nil } From 9215c18113ed361548bb059689f4352f7afd0ed4 Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Thu, 21 Jan 2016 01:04:45 -0800 Subject: [PATCH 12/30] keep track of addresses, check incoming txs for full pkscript match --- uspv/eight333.go | 3 +- uspv/txstore.go | 27 ++++++++---- uspv/utxodb.go | 105 ++++++++++++++++++++++++++++++++++++++--------- 3 files changed, 106 insertions(+), 29 deletions(-) diff --git a/uspv/eight333.go b/uspv/eight333.go index 7096bef3..c3082a77 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -73,7 +73,8 @@ func OpenSPV(remoteNode string, hfn, tsfn string, if err != nil { return s, err } - s.TS = inTs + inTs.Param = p + s.TS = inTs // copy pointer of txstore into spvcon myMsgVer, err := wire.NewMsgVersionFromConn(s.con, 0, 0) if err != nil { diff --git a/uspv/txstore.go b/uspv/txstore.go index 036ca4fe..09fd09f1 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -5,6 +5,8 @@ import ( "fmt" "log" + "github.com/btcsuite/btcd/txscript" + "github.com/btcsuite/btcd/chaincfg" "github.com/boltdb/bolt" @@ -23,7 +25,7 @@ type TxStore struct { LastIdx uint32 // should equal len(Adrs) StateDB *bolt.DB // place to write all this down // this is redundant with the SPVCon param... ugly and should be taken out - param *chaincfg.Params // network parameters (testnet3, testnetL) + Param *chaincfg.Params // network parameters (testnet3, testnetL) // From here, comes everything. It's a secret to everybody. rootPrivKey *hdkeychain.ExtendedKey @@ -39,20 +41,23 @@ type Utxo struct { // cash money. } type MyAdr struct { // an address I have the private key for - btcutil.Address + PkhAdr btcutil.Address KeyIdx uint32 // index for private key needed to sign / spend + // ^^ this is kindof redundant because it'll just be their position + // inside the Adrs slice, right? leave for now } -func NewTxStore() TxStore { +func NewTxStore(rootkey *hdkeychain.ExtendedKey) TxStore { var txs TxStore + txs.rootPrivKey = rootkey txs.OKTxids = make(map[wire.ShaHash]int32) return txs } -// add addresses into the TxStore +// add addresses into the TxStore in memory func (t *TxStore) AddAdr(a btcutil.Address, kidx uint32) { var ma MyAdr - ma.Address = a + ma.PkhAdr = a ma.KeyIdx = kidx t.Adrs = append(t.Adrs, ma) return @@ -75,7 +80,7 @@ func (t *TxStore) GimmeFilter() (*bloom.Filter, error) { } f := bloom.NewFilter(uint32(len(t.Adrs)), 0, 0.001, wire.BloomUpdateNone) for _, a := range t.Adrs { - f.Add(a.ScriptAddress()) + f.Add(a.PkhAdr.ScriptAddress()) } return f, nil } @@ -111,9 +116,12 @@ func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height int32) error { // check if any of the tx's outputs match my adrs for i, out := range tx.TxOut { // in each output of tx for _, a := range t.Adrs { // compare to each adr we have - // more correct would be to check for full script - // contains could have false positive? (p2sh/p2pkh same hash ..?) - if bytes.Contains(out.PkScript, a.ScriptAddress()) { // hit + // check for full script to eliminate false positives + aPKscript, err := txscript.PayToAddrScript(a.PkhAdr) + if err != nil { + return err + } + if bytes.Equal(out.PkScript, aPKscript) { // hit hits++ acq += out.Value var newu Utxo @@ -129,6 +137,7 @@ func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height int32) error { if err != nil { return err } + t.Sum += newu.Value t.Utxos = append(t.Utxos, newu) break } diff --git a/uspv/utxodb.go b/uspv/utxodb.go index ffa7ee00..9c5704ff 100644 --- a/uspv/utxodb.go +++ b/uspv/utxodb.go @@ -4,17 +4,19 @@ import ( "bytes" "encoding/binary" "fmt" - "log" "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" "github.com/boltdb/bolt" ) var ( - BKTUtxos = []byte("DuffelBag") // leave the rest to collect interest - BKTOld = []byte("SpentTxs") // for bookkeeping - KEYState = []byte("LastUpdate") // last state of DB + BKTUtxos = []byte("DuffelBag") // leave the rest to collect interest + BKTOld = []byte("SpentTxs") // for bookkeeping + BKTState = []byte("MiscState") // last state of DB + + KEYNumKeys = []byte("NumKeys") // number of keys used ) func (ts *TxStore) OpenDB(filename string) error { @@ -33,26 +35,70 @@ func (ts *TxStore) OpenDB(filename string) error { if err != nil { return err } + _, err = tx.CreateBucketIfNotExists(BKTState) + if err != nil { + return err + } return nil }) } +// NewAdr creates a new, never before seen address, and increments the +// DB counter as well as putting it in the ram Adrs store, and returns it +func (ts *TxStore) NewAdr() (*btcutil.AddressPubKeyHash, error) { + if ts.Param == nil { + return nil, fmt.Errorf("nil param") + } + n := uint32(len(ts.Adrs)) + priv, err := ts.rootPrivKey.Child(n) // + hdkeychain.HardenedKeyStart) + if err != nil { + return nil, err + } + + newAdr, err := priv.Address(ts.Param) + if err != nil { + return nil, err + } + + // total number of keys (now +1) into 4 bytes + var buf bytes.Buffer + err = binary.Write(&buf, binary.BigEndian, n+1) + if err != nil { + return nil, err + } + + // write to db file + err = ts.StateDB.Update(func(tx *bolt.Tx) error { + stt := tx.Bucket(BKTState) + return stt.Put(KEYNumKeys, buf.Bytes()) + }) + if err != nil { + return nil, err + } + // add in to ram. + ts.AddAdr(newAdr, n) + return newAdr, nil +} + +// PopulateAdrs just puts a bunch of adrs in ram; it doesn't touch the DB func (ts *TxStore) PopulateAdrs(lastKey uint32) error { for k := uint32(0); k < lastKey; k++ { - priv, err := ts.rootPrivKey.Child(k) + priv, err := ts.rootPrivKey.Child(k) // + hdkeychain.HardenedKeyStart) if err != nil { - log.Fatal(err) + return err } - myadr, err := priv.Address(ts.param) + + newAdr, err := priv.Address(ts.Param) if err != nil { - log.Fatal(err) + return err } - fmt.Printf("made adr %s\n", myadr.String()) - ts.AddAdr(myadr, k) + + ts.AddAdr(newAdr, k) } return nil } + func (u *Utxo) SaveToDB(dbx *bolt.DB) error { return dbx.Update(func(tx *bolt.Tx) error { duf := tx.Bucket(BKTUtxos) @@ -66,7 +112,6 @@ func (u *Utxo) SaveToDB(dbx *bolt.DB) error { } func (ts *TxStore) MarkSpent(op *wire.OutPoint, h int32, stx *wire.MsgTx) error { - // we write in key = outpoint (32 hash, 4 index) // value = spending txid // if we care about the spending tx we can store that in another bucket. @@ -90,8 +135,14 @@ func (ts *TxStore) MarkSpent(op *wire.OutPoint, h int32, stx *wire.MsgTx) error }) } -func (ts *TxStore) LoadUtxos() error { - err := ts.StateDB.View(func(tx *bolt.Tx) error { +// LoadFromDB loads everything in the db file into ram, rebuilding the TxStore +// (except the rootPrivKey, that should be done before calling this -- +// this will error if ts.rootPrivKey hasn't been loaded) +func (ts *TxStore) LoadFromDB() error { + if ts.rootPrivKey == nil { + return fmt.Errorf("LoadFromDB needs rootPrivKey loaded") + } + return ts.StateDB.View(func(tx *bolt.Tx) error { duf := tx.Bucket(BKTUtxos) if duf == nil { return fmt.Errorf("no duffel bag") @@ -100,9 +151,28 @@ func (ts *TxStore) LoadUtxos() error { if spent == nil { return fmt.Errorf("no spenttx bucket") } - + state := tx.Bucket(BKTState) + if state == nil { + return fmt.Errorf("no state bucket") + } + // first populate addresses from state bucket + numKeysBytes := state.Get(KEYNumKeys) + if numKeysBytes != nil { // NumKeys exists, read into uint32 + buf := bytes.NewBuffer(numKeysBytes) + var numKeys uint32 + err := binary.Read(buf, binary.BigEndian, &numKeys) + if err != nil { + return err + } + fmt.Printf("db says %d keys\n", numKeys) + err = ts.PopulateAdrs(numKeys) + if err != nil { + return err + } + } + // next load all utxos from db into ram duf.ForEach(func(k, v []byte) error { - // have to copy these here, otherwise append will crash it. + // have to copy k and v here, otherwise append will crash it. // not quite sure why but append does weird stuff I guess. stx := spent.Get(k) if stx == nil { // if it's not in the spent bucket @@ -115,6 +185,7 @@ func (ts *TxStore) LoadUtxos() error { return err } // and add it to ram + ts.Sum += newU.Value ts.Utxos = append(ts.Utxos, newU) } else { fmt.Printf("had utxo %x but spent by tx %x...\n", @@ -124,10 +195,6 @@ func (ts *TxStore) LoadUtxos() error { }) return nil }) - if err != nil { - return err - } - return nil } // outPointToBytes turns an outpoint into 36 bytes. From fc100921e8915331e270de54e4838449319ab10f Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Thu, 21 Jan 2016 17:59:50 -0800 Subject: [PATCH 13/30] add pushtx, and outpoint comparitor --- uspv/eight333.go | 14 ++++++++++++++ uspv/msghandler.go | 2 +- uspv/txstore.go | 13 +++++++++++-- 3 files changed, 26 insertions(+), 3 deletions(-) diff --git a/uspv/eight333.go b/uspv/eight333.go index c3082a77..045ec742 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -346,6 +346,20 @@ func NewRootAndHeight(r wire.ShaHash, h int32) (rah RootAndHeight) { return } +func (s *SPVCon) PushTx(tx *wire.MsgTx) error { + txid := tx.TxSha() + err := s.TS.AddTxid(&txid, 0) + if err != nil { + return err + } + err = s.TS.AckTx(tx) + if err != nil { + return err + } + s.outMsgQueue <- tx + return nil +} + // AskForMerkBlocks requests blocks from current to last // right now this asks for 1 block per getData message. // Maybe it's faster to ask for many in a each message? diff --git a/uspv/msghandler.go b/uspv/msghandler.go index 505dff3b..3d072832 100644 --- a/uspv/msghandler.go +++ b/uspv/msghandler.go @@ -66,7 +66,7 @@ func (s *SPVCon) incomingMessageHandler() { s.AskForHeaders() } case *wire.MsgTx: - err := s.TS.IngestTx(m) + err := s.TS.AckTx(m) if err != nil { log.Printf("Incoming Tx error: %s\n", err.Error()) } diff --git a/uspv/txstore.go b/uspv/txstore.go index 09fd09f1..c7bd604a 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -86,7 +86,7 @@ func (t *TxStore) GimmeFilter() (*bloom.Filter, error) { } // Ingest a tx into wallet, dealing with both gains and losses -func (t *TxStore) IngestTx(tx *wire.MsgTx) error { +func (t *TxStore) AckTx(tx *wire.MsgTx) error { inTxid := tx.TxSha() height, ok := t.OKTxids[inTxid] if !ok { @@ -158,7 +158,7 @@ func (t *TxStore) ExpellTx(tx *wire.MsgTx, height int32) error { for _, in := range tx.TxIn { for i, myutxo := range t.Utxos { - if myutxo.Op == in.PreviousOutPoint { + if OutPointsEqual(myutxo.Op, in.PreviousOutPoint) { hits++ loss += myutxo.Value err := t.MarkSpent(&myutxo.Op, height, tx) @@ -175,6 +175,15 @@ func (t *TxStore) ExpellTx(tx *wire.MsgTx, height int32) error { return nil } +// need this because before I was comparing pointers maybe? +// so they were the same outpoint but stored in 2 places so false negative? +func OutPointsEqual(a, b wire.OutPoint) bool { + if !a.Hash.IsEqual(&b.Hash) { + return false + } + return a.Index == b.Index +} + // TxToString prints out some info about a transaction. for testing / debugging func TxToString(tx *wire.MsgTx) string { str := "\t\t\t - Tx - \n" From fbc424492d5689260b6f3e2ec426140ee072f362 Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Thu, 21 Jan 2016 21:50:42 -0800 Subject: [PATCH 14/30] update filters as txs come in; missed some before --- uspv/eight333.go | 27 +++++++++++++++++++++++++-- uspv/msghandler.go | 10 +++++++--- uspv/txstore.go | 10 ++++++++-- uspv/utxodb.go | 2 +- 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/uspv/eight333.go b/uspv/eight333.go index 045ec742..22f582e4 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -164,6 +164,7 @@ func (s *SPVCon) PongBack(nonce uint64) { func (s *SPVCon) SendFilter(f *bloom.Filter) { s.outMsgQueue <- f.MsgFilterLoad() + return } @@ -374,12 +375,35 @@ func (s *SPVCon) AskForMerkBlocks(current, last int32) error { last = int32(n / 80) } - _, err := s.headerFile.Seek(int64(current*80), os.SEEK_SET) + // track number of utxos + track := len(s.TS.Utxos) + // create initial filter + filt, err := s.TS.GimmeFilter() + if err != nil { + return err + } + // send filter + s.SendFilter(filt) + fmt.Printf("sent filter %x\n", filt.MsgFilterLoad().Filter) + + _, err = s.headerFile.Seek(int64(current*80), os.SEEK_SET) if err != nil { return err } // loop through all heights where we want merkleblocks. for current < last { + // check if we need to update filter... every 5 new inputs...? + if track+4 < len(s.TS.Utxos) { + track = len(s.TS.Utxos) + filt, err := s.TS.GimmeFilter() + if err != nil { + return err + } + s.SendFilter(filt) + + fmt.Printf("sent filter %x\n", filt.MsgFilterLoad().Filter) + } + // load header from file err = hdr.Deserialize(s.headerFile) if err != nil { @@ -400,6 +424,5 @@ func (s *SPVCon) AskForMerkBlocks(current, last int32) error { s.mBlockQueue <- rah // push height and mroot of requested block on queue current++ } - return nil } diff --git a/uspv/msghandler.go b/uspv/msghandler.go index 3d072832..b3ee6822 100644 --- a/uspv/msghandler.go +++ b/uspv/msghandler.go @@ -1,7 +1,6 @@ package uspv import ( - "fmt" "log" "github.com/btcsuite/btcd/wire" @@ -40,7 +39,7 @@ func (s *SPVCon) incomingMessageHandler() { log.Printf("Merkle block error: %s\n", err.Error()) return } - fmt.Printf(" got %d txs ", len(txids)) + // fmt.Printf(" got %d txs ", len(txids)) // fmt.Printf(" = got %d txs from block %s\n", // len(txids), m.Header.BlockSha().String()) rah := <-s.mBlockQueue // pop height off mblock queue @@ -70,7 +69,12 @@ func (s *SPVCon) incomingMessageHandler() { if err != nil { log.Printf("Incoming Tx error: %s\n", err.Error()) } - // log.Printf("Got tx %s\n", m.TxSha().String()) + // log.Printf("Got tx %s\n", m.TxSha().String()) + + case *wire.MsgReject: + log.Printf("Rejected! cmd: %s code: %s tx: %s reason: %s", + m.Cmd, m.Code.String(), m.Hash.String(), m.Reason) + default: log.Printf("Got unknown message type %s\n", m.Command()) } diff --git a/uspv/txstore.go b/uspv/txstore.go index c7bd604a..23c4b35c 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -78,10 +78,17 @@ func (t *TxStore) GimmeFilter() (*bloom.Filter, error) { if len(t.Adrs) == 0 { return nil, fmt.Errorf("no addresses to filter for") } - f := bloom.NewFilter(uint32(len(t.Adrs)), 0, 0.001, wire.BloomUpdateNone) + // add addresses to look for incoming + elem := uint32(len(t.Adrs) + len(t.Utxos)) + f := bloom.NewFilter(elem, 0, 0.001, wire.BloomUpdateAll) for _, a := range t.Adrs { f.Add(a.PkhAdr.ScriptAddress()) } + // add txids of utxos to look for outgoing + for _, u := range t.Utxos { + f.AddOutPoint(&u.Op) + } + return f, nil } @@ -137,7 +144,6 @@ func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height int32) error { if err != nil { return err } - t.Sum += newu.Value t.Utxos = append(t.Utxos, newu) break } diff --git a/uspv/utxodb.go b/uspv/utxodb.go index 9c5704ff..68da11b3 100644 --- a/uspv/utxodb.go +++ b/uspv/utxodb.go @@ -185,8 +185,8 @@ func (ts *TxStore) LoadFromDB() error { return err } // and add it to ram - ts.Sum += newU.Value ts.Utxos = append(ts.Utxos, newU) + ts.Sum += newU.Value } else { fmt.Printf("had utxo %x but spent by tx %x...\n", k, stx[:8]) From 01c35e62ab0a06ea30ad44545e0ded2c906db69d Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Fri, 22 Jan 2016 01:41:08 -0800 Subject: [PATCH 15/30] check for already known txs (due to reorg?) --- uspv/eight333.go | 6 +++--- uspv/txstore.go | 8 ++++++-- uspv/utxodb.go | 19 +++++++++++++++---- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/uspv/eight333.go b/uspv/eight333.go index 22f582e4..5c2e459d 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -368,7 +368,7 @@ func (s *SPVCon) AskForMerkBlocks(current, last int32) error { var hdr wire.BlockHeader // if last is 0, that means go as far as we can if last == 0 { - n, err := s.headerFile.Seek(-80, os.SEEK_END) + n, err := s.headerFile.Seek(0, os.SEEK_END) if err != nil { return err } @@ -392,8 +392,8 @@ func (s *SPVCon) AskForMerkBlocks(current, last int32) error { } // loop through all heights where we want merkleblocks. for current < last { - // check if we need to update filter... every 5 new inputs...? - if track+4 < len(s.TS.Utxos) { + // check if we need to update filter... diff of 5 utxos...? + if track < len(s.TS.Utxos)-4 || track > len(s.TS.Utxos)+4 { track = len(s.TS.Utxos) filt, err := s.TS.GimmeFilter() if err != nil { diff --git a/uspv/txstore.go b/uspv/txstore.go index 23c4b35c..c9aaa656 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -140,11 +140,15 @@ func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height int32) error { newop.Hash = tx.TxSha() newop.Index = uint32(i) newu.Op = newop - err := newu.SaveToDB(t.StateDB) + dupe, err := newu.SaveToDB(t.StateDB) if err != nil { return err } - t.Utxos = append(t.Utxos, newu) + if !dupe { // only save to DB if new txid + t.Utxos = append(t.Utxos, newu) + } else { + fmt.Printf("...dupe ") + } break } } diff --git a/uspv/utxodb.go b/uspv/utxodb.go index 68da11b3..4e701cfc 100644 --- a/uspv/utxodb.go +++ b/uspv/utxodb.go @@ -7,6 +7,7 @@ import ( "github.com/btcsuite/btcd/wire" "github.com/btcsuite/btcutil" + "github.com/btcsuite/btcutil/hdkeychain" "github.com/boltdb/bolt" ) @@ -50,7 +51,7 @@ func (ts *TxStore) NewAdr() (*btcutil.AddressPubKeyHash, error) { return nil, fmt.Errorf("nil param") } n := uint32(len(ts.Adrs)) - priv, err := ts.rootPrivKey.Child(n) // + hdkeychain.HardenedKeyStart) + priv, err := ts.rootPrivKey.Child(n + hdkeychain.HardenedKeyStart) if err != nil { return nil, err } @@ -84,7 +85,7 @@ func (ts *TxStore) NewAdr() (*btcutil.AddressPubKeyHash, error) { func (ts *TxStore) PopulateAdrs(lastKey uint32) error { for k := uint32(0); k < lastKey; k++ { - priv, err := ts.rootPrivKey.Child(k) // + hdkeychain.HardenedKeyStart) + priv, err := ts.rootPrivKey.Child(k + hdkeychain.HardenedKeyStart) if err != nil { return err } @@ -99,16 +100,26 @@ func (ts *TxStore) PopulateAdrs(lastKey uint32) error { return nil } -func (u *Utxo) SaveToDB(dbx *bolt.DB) error { - return dbx.Update(func(tx *bolt.Tx) error { +// SaveToDB write a utxo to disk, and returns true if it's a dupe +func (u *Utxo) SaveToDB(dbx *bolt.DB) (bool, error) { + var dupe bool + err := dbx.Update(func(tx *bolt.Tx) error { duf := tx.Bucket(BKTUtxos) b, err := u.ToBytes() if err != nil { return err } + if duf.Get(b[:36]) != nil { // already have tx + dupe = true + return nil + } // key : val is txid:everything else return duf.Put(b[:36], b[36:]) }) + if err != nil { + return false, err + } + return dupe, nil } func (ts *TxStore) MarkSpent(op *wire.OutPoint, h int32, stx *wire.MsgTx) error { From 5a7c04bdb5693f166aff58bc2e280a69396717ae Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Fri, 22 Jan 2016 16:04:27 -0800 Subject: [PATCH 16/30] incoming invs and tx signing when we get inv messages, txs are easy but blocks are a little tricky block heigh synchronization is done via os' file system signing is a bit inelegant (searches through inputs for pkscript matches) but fast enough in practice --- uspv/eight333.go | 115 +++++++++++++++++++++++++++++++++---------- uspv/header.go | 12 ++--- uspv/mblock.go | 5 +- uspv/msghandler.go | 39 ++++++--------- uspv/sortsignsend.go | 56 +++++++++++++++++++++ uspv/txstore.go | 5 +- 6 files changed, 170 insertions(+), 62 deletions(-) create mode 100644 uspv/sortsignsend.go diff --git a/uspv/eight333.go b/uspv/eight333.go index 5c2e459d..548f450a 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -27,9 +27,11 @@ type SPVCon struct { con net.Conn // the (probably tcp) connection to the node headerFile *os.File // file for SPV headers + //[doesn't work without fancy mutexes, nevermind, just use header file] + // localHeight int32 // block height we're on + remoteHeight int32 // block height they're on localVersion uint32 // version we report remoteVersion uint32 // version remote node - remoteHeight int32 // block height they're on // what's the point of the input queue? remove? leave for now... inMsgQueue chan wire.Message // Messages coming in from remote node @@ -42,7 +44,7 @@ type SPVCon struct { param *chaincfg.Params // network parameters (testnet3, testnetL) // mBlockQueue is for keeping track of what height we've requested. - mBlockQueue chan RootAndHeight + mBlockQueue chan HashAndHeight } func OpenSPV(remoteNode string, hfn, tsfn string, @@ -125,7 +127,7 @@ func OpenSPV(remoteNode string, hfn, tsfn string, go s.incomingMessageHandler() s.outMsgQueue = make(chan wire.Message, 1) go s.outgoingMessageHandler() - s.mBlockQueue = make(chan RootAndHeight, 32) // queue depth 32 is a thing + s.mBlockQueue = make(chan HashAndHeight, 32) // queue depth 32 is a thing return s, nil } @@ -207,6 +209,36 @@ func (s *SPVCon) HeightFromHeader(query wire.BlockHeader) (uint32, error) { return 0, fmt.Errorf("Header not found on disk") } +// AskForTx requests a tx we heard about from an inv message. +// It's one at a time but should be fast enough. +func (s *SPVCon) AskForTx(txid wire.ShaHash) { + gdata := wire.NewMsgGetData() + inv := wire.NewInvVect(wire.InvTypeTx, &txid) + gdata.AddInvVect(inv) + s.outMsgQueue <- gdata +} + +// AskForBlock requests a merkle block we heard about from an inv message. +// We don't have it in our header file so when we get it we do both operations: +// appending and checking the header, and checking spv proofs +func (s *SPVCon) AskForBlock(hsh wire.ShaHash) { + gdata := wire.NewMsgGetData() + inv := wire.NewInvVect(wire.InvTypeFilteredBlock, &hsh) + gdata.AddInvVect(inv) + + info, err := s.headerFile.Stat() // get + if err != nil { + log.Fatal(err) // crash if header file disappears + } + nextHeight := int32(info.Size() / 80) + + hah := NewRootAndHeight(hsh, nextHeight) + + s.mBlockQueue <- hah // push height and mroot of requested block on queue + s.outMsgQueue <- gdata // push request to outbox + +} + func (s *SPVCon) AskForHeaders() error { var hdr wire.BlockHeader ghdr := wire.NewMsgGetHeaders() @@ -229,6 +261,7 @@ func (s *SPVCon) AskForHeaders() error { } log.Printf("suk to offset %d (should be near the end\n", ns) + // get header from last 80 bytes of file err = hdr.Deserialize(s.headerFile) if err != nil { @@ -250,6 +283,31 @@ func (s *SPVCon) AskForHeaders() error { return nil } +func (s *SPVCon) IngestMerkleBlock(m *wire.MsgMerkleBlock) error { + txids, err := checkMBlock(m) // check self-consistency + if err != nil { + return err + } + hah := <-s.mBlockQueue // pop height off mblock queue + // this verifies order, and also that the returned header fits + // into our SPV header file + newMerkBlockSha := m.Header.BlockSha() + if !hah.blockhash.IsEqual(&newMerkBlockSha) { + return fmt.Errorf("merkle block out of order error") + } + for _, txid := range txids { + err := s.TS.AddTxid(txid, hah.height) + if err != nil { + return fmt.Errorf("Txid store error: %s\n", err.Error()) + } + } + return nil +} + +// IngestHeaders takes in a bunch of headers and appends them to the +// local header file, checking that they fit. If there's no headers, +// it assumes we're done and returns false. If it worked it assumes there's +// more to request and returns true.9 func (s *SPVCon) IngestHeaders(m *wire.MsgHeaders) (bool, error) { var err error // seek to last header @@ -264,6 +322,12 @@ func (s *SPVCon) IngestHeaders(m *wire.MsgHeaders) (bool, error) { } prevHash := last.BlockSha() + endPos, err := s.headerFile.Seek(0, os.SEEK_END) + if err != nil { + return false, err + } + tip := int32(endPos/80) - 1 // move back 1 header length to read + gotNum := int64(len(m.Headers)) if gotNum > 0 { fmt.Printf("got %d headers. Range:\n%s - %s\n", @@ -273,17 +337,11 @@ func (s *SPVCon) IngestHeaders(m *wire.MsgHeaders) (bool, error) { log.Printf("got 0 headers, we're probably synced up") return false, nil } - - endPos, err := s.headerFile.Seek(0, os.SEEK_END) - if err != nil { - return false, err - } - // check first header returned to make sure it fits on the end // of our header file if !m.Headers[0].PrevBlock.IsEqual(&prevHash) { // delete 100 headers if this happens! Dumb reorg. - log.Printf("possible reorg; header msg doesn't fit. points to %s, expect %s", + log.Printf("reorg? header msg doesn't fit. points to %s, expect %s", m.Headers[0].PrevBlock.String(), prevHash.String()) if endPos < 8080 { // jeez I give up, back to genesis @@ -297,8 +355,6 @@ func (s *SPVCon) IngestHeaders(m *wire.MsgHeaders) (bool, error) { return false, fmt.Errorf("Truncated header file to try again") } - tip := endPos / 80 - tip-- // move back header length so it can read last header for _, resphdr := range m.Headers { // write to end of file err = resphdr.Serialize(s.headerFile) @@ -328,22 +384,25 @@ func (s *SPVCon) IngestHeaders(m *wire.MsgHeaders) (bool, error) { } } log.Printf("Headers to height %d OK.", tip) + return true, nil } -// RootAndHeight is needed instead of just height in case a fullnode +// HashAndHeight is needed instead of just height in case a fullnode // responds abnormally (?) by sending out of order merkleblocks. // we cache a merkleroot:height pair in the queue so we don't have to // look them up from the disk. -type RootAndHeight struct { - root wire.ShaHash - height int32 +// Also used when inv messages indicate blocks so we can add the header +// and parse the txs in one request instead of requesting headers first. +type HashAndHeight struct { + blockhash wire.ShaHash + height int32 } // NewRootAndHeight saves like 2 lines. -func NewRootAndHeight(r wire.ShaHash, h int32) (rah RootAndHeight) { - rah.root = r - rah.height = h +func NewRootAndHeight(b wire.ShaHash, h int32) (hah HashAndHeight) { + hah.blockhash = b + hah.height = h return } @@ -366,15 +425,17 @@ func (s *SPVCon) PushTx(tx *wire.MsgTx) error { // Maybe it's faster to ask for many in a each message? func (s *SPVCon) AskForMerkBlocks(current, last int32) error { var hdr wire.BlockHeader + info, err := s.headerFile.Stat() // get + if err != nil { + return err // crash if header file disappears + } + nextHeight := int32(info.Size() / 80) + fmt.Printf("have headers up to height %d\n", nextHeight-1) // if last is 0, that means go as far as we can if last == 0 { - n, err := s.headerFile.Seek(0, os.SEEK_END) - if err != nil { - return err - } - last = int32(n / 80) + last = nextHeight - 1 } - + fmt.Printf("will request merkleblocks %d to %d\n", current, last) // track number of utxos track := len(s.TS.Utxos) // create initial filter @@ -419,9 +480,9 @@ func (s *SPVCon) AskForMerkBlocks(current, last int32) error { if err != nil { return err } - rah := NewRootAndHeight(hdr.MerkleRoot, current) + hah := NewRootAndHeight(hdr.BlockSha(), current) s.outMsgQueue <- gdataMsg - s.mBlockQueue <- rah // push height and mroot of requested block on queue + s.mBlockQueue <- hah // push height and mroot of requested block on queue current++ } return nil diff --git a/uspv/header.go b/uspv/header.go index 6f9c18f9..40983e36 100644 --- a/uspv/header.go +++ b/uspv/header.go @@ -23,7 +23,7 @@ import ( const ( targetTimespan = time.Hour * 24 * 14 targetSpacing = time.Minute * 10 - epochLength = int64(targetTimespan / targetSpacing) + epochLength = int32(targetTimespan / targetSpacing) // 2016 maxDiffAdjust = 4 minRetargetTimespan = int64(targetTimespan / maxDiffAdjust) maxRetargetTimespan = int64(targetTimespan * maxDiffAdjust) @@ -90,7 +90,7 @@ func calcDiffAdjust(start, end wire.BlockHeader, p *chaincfg.Params) uint32 { return blockchain.BigToCompact(newTarget) } -func CheckHeader(r io.ReadSeeker, height int64, p *chaincfg.Params) bool { +func CheckHeader(r io.ReadSeeker, height int32, p *chaincfg.Params) bool { var err error var cur, prev, epochStart wire.BlockHeader // don't try to verfy the genesis block. That way madness lies. @@ -100,7 +100,7 @@ func CheckHeader(r io.ReadSeeker, height int64, p *chaincfg.Params) bool { // initial load of headers // load epochstart, previous and current. // get the header from the epoch start, up to 2016 blocks ago - _, err = r.Seek(80*(height-(height%epochLength)), os.SEEK_SET) + _, err = r.Seek(int64(80*(height-(height%epochLength))), os.SEEK_SET) if err != nil { log.Printf(err.Error()) return false @@ -113,7 +113,7 @@ func CheckHeader(r io.ReadSeeker, height int64, p *chaincfg.Params) bool { // log.Printf("start epoch at height %d ", height-(height%epochLength)) // seek to n-1 header - _, err = r.Seek(80*(height-1), os.SEEK_SET) + _, err = r.Seek(int64(80*(height-1)), os.SEEK_SET) if err != nil { log.Printf(err.Error()) return false @@ -125,7 +125,7 @@ func CheckHeader(r io.ReadSeeker, height int64, p *chaincfg.Params) bool { return false } // seek to curHeight header and read in - _, err = r.Seek(80*(height), os.SEEK_SET) + _, err = r.Seek(int64(80*(height)), os.SEEK_SET) if err != nil { log.Printf(err.Error()) return false @@ -184,7 +184,7 @@ difficulty adjustments, and that they all link in to each other properly. This is the only blockchain technology in the whole code base. Returns false if anything bad happens. Returns true if the range checks out with no errors. */ -func CheckRange(r io.ReadSeeker, first, last int64, p *chaincfg.Params) bool { +func CheckRange(r io.ReadSeeker, first, last int32, p *chaincfg.Params) bool { for i := first; i <= last; i++ { if !CheckHeader(r, i, p) { return false diff --git a/uspv/mblock.go b/uspv/mblock.go index dc71608c..823ec805 100644 --- a/uspv/mblock.go +++ b/uspv/mblock.go @@ -65,9 +65,8 @@ func inDeadZone(pos, size uint32) bool { return pos > last } -// take in a merkle block, parse through it, and return the -// txids that they're trying to tell us about. If there's any problem -// return an error. +// take in a merkle block, parse through it, and return txids indicated +// If there's any problem return an error. Checks self-consistency only. // doing it with a stack instead of recursion. Because... // OK I don't know why I'm just not in to recursion OK? func checkMBlock(m *wire.MsgMerkleBlock) ([]*wire.ShaHash, error) { diff --git a/uspv/msghandler.go b/uspv/msghandler.go index b3ee6822..58d15093 100644 --- a/uspv/msghandler.go +++ b/uspv/msghandler.go @@ -30,31 +30,11 @@ func (s *SPVCon) incomingMessageHandler() { case *wire.MsgPong: log.Printf("Got a pong response. OK.\n") case *wire.MsgMerkleBlock: - - // log.Printf("Got merkle block message. Will verify.\n") - // fmt.Printf("%d flag bytes, %d txs, %d hashes", - // len(m.Flags), m.Transactions, len(m.Hashes)) - txids, err := checkMBlock(m) + err = s.IngestMerkleBlock(m) if err != nil { log.Printf("Merkle block error: %s\n", err.Error()) - return + continue } - // fmt.Printf(" got %d txs ", len(txids)) - // fmt.Printf(" = got %d txs from block %s\n", - // len(txids), m.Header.BlockSha().String()) - rah := <-s.mBlockQueue // pop height off mblock queue - // this verifies order, and also that the returned header fits - // into our SPV header file - if !rah.root.IsEqual(&m.Header.MerkleRoot) { - log.Printf("out of order error") - } - for _, txid := range txids { - err := s.TS.AddTxid(txid, rah.height) - if err != nil { - log.Printf("Txid store error: %s\n", err.Error()) - } - } - case *wire.MsgHeaders: moar, err := s.IngestHeaders(m) if err != nil { @@ -70,11 +50,22 @@ func (s *SPVCon) incomingMessageHandler() { log.Printf("Incoming Tx error: %s\n", err.Error()) } // log.Printf("Got tx %s\n", m.TxSha().String()) - case *wire.MsgReject: log.Printf("Rejected! cmd: %s code: %s tx: %s reason: %s", m.Cmd, m.Code.String(), m.Hash.String(), m.Reason) - + case *wire.MsgInv: + log.Printf("got inv. Contains:\n") + for i, thing := range m.InvList { + log.Printf("\t%d)%s : %s", + i, thing.Type.String(), thing.Hash.String()) + if thing.Type == wire.InvTypeTx { // new tx, ingest + s.TS.OKTxids[thing.Hash] = 0 // unconfirmed + s.AskForTx(thing.Hash) + } + if thing.Type == wire.InvTypeBlock { // new block, ingest + s.AskForBlock(thing.Hash) + } + } default: log.Printf("Got unknown message type %s\n", m.Command()) } diff --git a/uspv/sortsignsend.go b/uspv/sortsignsend.go new file mode 100644 index 00000000..4aa9bc26 --- /dev/null +++ b/uspv/sortsignsend.go @@ -0,0 +1,56 @@ +package uspv + +import ( + "bytes" + "fmt" + + "github.com/btcsuite/btcd/txscript" + "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil/hdkeychain" + "github.com/btcsuite/btcutil/txsort" +) + +func (t *TxStore) SignThis(tx *wire.MsgTx) error { + fmt.Printf("-= SignThis =-\n") + + // sort tx before signing. + txsort.InPlaceSort(tx) + + sigs := make([][]byte, len(tx.TxIn)) + // first iterate over each input + for j, in := range tx.TxIn { + for k := uint32(0); k < uint32(len(t.Adrs)); k++ { + child, err := t.rootPrivKey.Child(k + hdkeychain.HardenedKeyStart) + if err != nil { + return err + } + myadr, err := child.Address(t.Param) + if err != nil { + return err + } + adrScript, err := txscript.PayToAddrScript(myadr) + if err != nil { + return err + } + if bytes.Equal(adrScript, in.SignatureScript) { + fmt.Printf("Hit; key %d matches input %d. Signing.\n", k, j) + priv, err := child.ECPrivKey() + if err != nil { + return err + } + sigs[j], err = txscript.SignatureScript( + tx, j, in.SignatureScript, txscript.SigHashAll, priv, true) + if err != nil { + return err + } + break + } + } + } + for i, s := range sigs { + if s != nil { + tx.TxIn[i].SignatureScript = s + } + } + return nil +} diff --git a/uspv/txstore.go b/uspv/txstore.go index c9aaa656..d1aeb45f 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -129,8 +129,7 @@ func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height int32) error { return err } if bytes.Equal(out.PkScript, aPKscript) { // hit - hits++ - acq += out.Value + var newu Utxo newu.AtHeight = height newu.KeyIdx = a.KeyIdx @@ -146,6 +145,8 @@ func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height int32) error { } if !dupe { // only save to DB if new txid t.Utxos = append(t.Utxos, newu) + acq += out.Value + hits++ } else { fmt.Printf("...dupe ") } From 851d3533e587d49fdc7a6dc5a81b656a2971f55d Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Fri, 22 Jan 2016 17:15:56 -0800 Subject: [PATCH 17/30] deals with incoming blocks and txs gets stuck on reorgs though, until program is restarted --- uspv/txstore.go | 43 ++++++++++++++++++++++++++++++++----------- uspv/utxodb.go | 22 ++++++++++++---------- 2 files changed, 44 insertions(+), 21 deletions(-) diff --git a/uspv/txstore.go b/uspv/txstore.go index d1aeb45f..afe3561e 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -19,7 +19,7 @@ import ( type TxStore struct { OKTxids map[wire.ShaHash]int32 // known good txids and their heights - Utxos []Utxo // stacks on stacks + Utxos []*Utxo // stacks on stacks Sum int64 // racks on racks Adrs []MyAdr // endeavouring to acquire capital LastIdx uint32 // should equal len(Adrs) @@ -118,16 +118,40 @@ func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height int32) error { if tx == nil { return fmt.Errorf("Tried to add nil tx") } - var hits uint32 - var acq int64 + newTxid := tx.TxSha() + var hits uint32 // how many outputs of this tx are ours + var acq int64 // total acquirement from this tx // check if any of the tx's outputs match my adrs for i, out := range tx.TxOut { // in each output of tx + dup := false // start by assuming its new until found duplicate + newOp := wire.NewOutPoint(&newTxid, uint32(i)) + // first look for dupes -- already known outpoints. + // if we find a dupe here overwrite it to the DB. + for _, u := range t.Utxos { + dup = OutPointsEqual(*newOp, u.Op) // is this outpoint known? + if dup { // found dupe + fmt.Printf(" %s is dupe\t", newOp.String()) + u.AtHeight = height // ONLY difference is height + // save modified utxo to db, overwriting old one + err := u.SaveToDB(t.StateDB) + if err != nil { + return err + } + break // out of the t.Utxo range loop + } + } + if dup { + // if we found the outpoint to be a dup above, don't add it again + // when it matches an address, just go to the next outpoint + continue + } for _, a := range t.Adrs { // compare to each adr we have // check for full script to eliminate false positives aPKscript, err := txscript.PayToAddrScript(a.PkhAdr) if err != nil { return err } + // already checked for dupes, this must be a new outpoint if bytes.Equal(out.PkScript, aPKscript) { // hit var newu Utxo @@ -139,17 +163,14 @@ func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height int32) error { newop.Hash = tx.TxSha() newop.Index = uint32(i) newu.Op = newop - dupe, err := newu.SaveToDB(t.StateDB) + err = newu.SaveToDB(t.StateDB) if err != nil { return err } - if !dupe { // only save to DB if new txid - t.Utxos = append(t.Utxos, newu) - acq += out.Value - hits++ - } else { - fmt.Printf("...dupe ") - } + + acq += out.Value + hits++ + t.Utxos = append(t.Utxos, &newu) // always add new utxo break } } diff --git a/uspv/utxodb.go b/uspv/utxodb.go index 4e701cfc..aee9d9ff 100644 --- a/uspv/utxodb.go +++ b/uspv/utxodb.go @@ -100,26 +100,28 @@ func (ts *TxStore) PopulateAdrs(lastKey uint32) error { return nil } -// SaveToDB write a utxo to disk, and returns true if it's a dupe -func (u *Utxo) SaveToDB(dbx *bolt.DB) (bool, error) { - var dupe bool +// SaveToDB write a utxo to disk, overwriting an old utxo of the same outpoint +func (u *Utxo) SaveToDB(dbx *bolt.DB) error { + err := dbx.Update(func(tx *bolt.Tx) error { duf := tx.Bucket(BKTUtxos) b, err := u.ToBytes() if err != nil { return err } - if duf.Get(b[:36]) != nil { // already have tx - dupe = true - return nil - } + // don't check for dupes here, check in AbsorbTx(). here overwrite. + // if duf.Get(b[:36]) != nil { // already have tx + // dupe = true + // return nil + // } + // key : val is txid:everything else return duf.Put(b[:36], b[36:]) }) if err != nil { - return false, err + return err } - return dupe, nil + return nil } func (ts *TxStore) MarkSpent(op *wire.OutPoint, h int32, stx *wire.MsgTx) error { @@ -196,7 +198,7 @@ func (ts *TxStore) LoadFromDB() error { return err } // and add it to ram - ts.Utxos = append(ts.Utxos, newU) + ts.Utxos = append(ts.Utxos, &newU) ts.Sum += newU.Value } else { fmt.Printf("had utxo %x but spent by tx %x...\n", From d9afd623eb746c1dedcbf80c20bd7be7e1514ff4 Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Wed, 27 Jan 2016 01:24:16 -0800 Subject: [PATCH 18/30] new stxo struct and more db methods I'm getting away from having both in-ram and on-disk stores for the transaction store data. it should all be on disk, it's safer that way. It might be slower but this will not process many txs / second anyway. --- uspv/eight333.go | 26 ++++++- uspv/msghandler.go | 31 +++++--- uspv/txstore.go | 30 ++++++-- uspv/utxodb.go | 187 ++++++++++++++++++++++++++++++++++++++------- 4 files changed, 225 insertions(+), 49 deletions(-) diff --git a/uspv/eight333.go b/uspv/eight333.go index 548f450a..a8e592d5 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -222,10 +222,19 @@ func (s *SPVCon) AskForTx(txid wire.ShaHash) { // We don't have it in our header file so when we get it we do both operations: // appending and checking the header, and checking spv proofs func (s *SPVCon) AskForBlock(hsh wire.ShaHash) { + + fmt.Printf("mBlockQueue len %d\n", len(s.mBlockQueue)) + // wait until all mblocks are done before adding + for len(s.mBlockQueue) != 0 { + // fmt.Printf("mBlockQueue len %d\n", len(s.mBlockQueue)) + } + gdata := wire.NewMsgGetData() inv := wire.NewInvVect(wire.InvTypeFilteredBlock, &hsh) gdata.AddInvVect(inv) + // TODO - wait until headers are sync'd before checking height + info, err := s.headerFile.Stat() // get if err != nil { log.Fatal(err) // crash if header file disappears @@ -437,7 +446,11 @@ func (s *SPVCon) AskForMerkBlocks(current, last int32) error { } fmt.Printf("will request merkleblocks %d to %d\n", current, last) // track number of utxos - track := len(s.TS.Utxos) + track, err := s.TS.NumUtxos() + if err != nil { + return err + } + // create initial filter filt, err := s.TS.GimmeFilter() if err != nil { @@ -454,15 +467,20 @@ func (s *SPVCon) AskForMerkBlocks(current, last int32) error { // loop through all heights where we want merkleblocks. for current < last { // check if we need to update filter... diff of 5 utxos...? - if track < len(s.TS.Utxos)-4 || track > len(s.TS.Utxos)+4 { - track = len(s.TS.Utxos) + nTrack, err := s.TS.NumUtxos() + if err != nil { + return err + } + + if track < nTrack-4 || track > nTrack+4 { + track = nTrack filt, err := s.TS.GimmeFilter() if err != nil { return err } s.SendFilter(filt) - fmt.Printf("sent filter %x\n", filt.MsgFilterLoad().Filter) + fmt.Printf("sent %d byte filter\n", len(filt.MsgFilterLoad().Filter)) } // load header from file diff --git a/uspv/msghandler.go b/uspv/msghandler.go index 58d15093..a73e872b 100644 --- a/uspv/msghandler.go +++ b/uspv/msghandler.go @@ -54,18 +54,14 @@ func (s *SPVCon) incomingMessageHandler() { log.Printf("Rejected! cmd: %s code: %s tx: %s reason: %s", m.Cmd, m.Code.String(), m.Hash.String(), m.Reason) case *wire.MsgInv: - log.Printf("got inv. Contains:\n") + go s.InvHandler(m) + + case *wire.MsgNotFound: + log.Printf("Got not found response from remote:") for i, thing := range m.InvList { - log.Printf("\t%d)%s : %s", - i, thing.Type.String(), thing.Hash.String()) - if thing.Type == wire.InvTypeTx { // new tx, ingest - s.TS.OKTxids[thing.Hash] = 0 // unconfirmed - s.AskForTx(thing.Hash) - } - if thing.Type == wire.InvTypeBlock { // new block, ingest - s.AskForBlock(thing.Hash) - } + log.Printf("\t$d) %s: %s", i, thing.Type, thing.Hash) } + default: log.Printf("Got unknown message type %s\n", m.Command()) } @@ -86,3 +82,18 @@ func (s *SPVCon) outgoingMessageHandler() { } return } + +func (s *SPVCon) InvHandler(m *wire.MsgInv) { + log.Printf("got inv. Contains:\n") + for i, thing := range m.InvList { + log.Printf("\t%d)%s : %s", + i, thing.Type.String(), thing.Hash.String()) + if thing.Type == wire.InvTypeTx { // new tx, ingest + s.TS.OKTxids[thing.Hash] = 0 // unconfirmed + s.AskForTx(thing.Hash) + } + if thing.Type == wire.InvTypeBlock { // new block, ingest + s.AskForBlock(thing.Hash) + } + } +} diff --git a/uspv/txstore.go b/uspv/txstore.go index afe3561e..c768567a 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -32,12 +32,21 @@ type TxStore struct { } type Utxo struct { // cash money. + Op wire.OutPoint // where + // all the info needed to spend AtHeight int32 // block height where this tx was confirmed, 0 for unconf KeyIdx uint32 // index for private key needed to sign / spend Value int64 // higher is better - Op wire.OutPoint // where + // IsCoinbase bool // can't spend for a while +} + +// Stxo is a utxo that has moved on. +type Stxo struct { + Utxo // when it used to be a utxo + SpendHeight int32 // height at which it met its demise + SpendTxid wire.ShaHash // the tx that consumed it } type MyAdr struct { // an address I have the private key for @@ -79,7 +88,12 @@ func (t *TxStore) GimmeFilter() (*bloom.Filter, error) { return nil, fmt.Errorf("no addresses to filter for") } // add addresses to look for incoming - elem := uint32(len(t.Adrs) + len(t.Utxos)) + nutxo, err := t.NumUtxos() + if err != nil { + return nil, err + } + + elem := uint32(len(t.Adrs)) + nutxo f := bloom.NewFilter(elem, 0, 0.001, wire.BloomUpdateAll) for _, a := range t.Adrs { f.Add(a.PkhAdr.ScriptAddress()) @@ -121,7 +135,7 @@ func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height int32) error { newTxid := tx.TxSha() var hits uint32 // how many outputs of this tx are ours var acq int64 // total acquirement from this tx - // check if any of the tx's outputs match my adrs + // check if any of the tx's outputs match my known outpoints for i, out := range tx.TxOut { // in each output of tx dup := false // start by assuming its new until found duplicate newOp := wire.NewOutPoint(&newTxid, uint32(i)) @@ -133,7 +147,7 @@ func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height int32) error { fmt.Printf(" %s is dupe\t", newOp.String()) u.AtHeight = height // ONLY difference is height // save modified utxo to db, overwriting old one - err := u.SaveToDB(t.StateDB) + err := t.SaveUtxo(u) if err != nil { return err } @@ -145,15 +159,15 @@ func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height int32) error { // when it matches an address, just go to the next outpoint continue } + // check if this is a new txout matching one of my addresses for _, a := range t.Adrs { // compare to each adr we have // check for full script to eliminate false positives aPKscript, err := txscript.PayToAddrScript(a.PkhAdr) if err != nil { return err } - // already checked for dupes, this must be a new outpoint if bytes.Equal(out.PkScript, aPKscript) { // hit - + // already checked for dupes, so this must be a new outpoint var newu Utxo newu.AtHeight = height newu.KeyIdx = a.KeyIdx @@ -163,7 +177,7 @@ func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height int32) error { newop.Hash = tx.TxSha() newop.Index = uint32(i) newu.Op = newop - err = newu.SaveToDB(t.StateDB) + err = t.SaveUtxo(&newu) if err != nil { return err } @@ -193,7 +207,7 @@ func (t *TxStore) ExpellTx(tx *wire.MsgTx, height int32) error { if OutPointsEqual(myutxo.Op, in.PreviousOutPoint) { hits++ loss += myutxo.Value - err := t.MarkSpent(&myutxo.Op, height, tx) + err := t.MarkSpent(*myutxo, height, tx) if err != nil { return err } diff --git a/uspv/utxodb.go b/uspv/utxodb.go index aee9d9ff..d6d5763b 100644 --- a/uspv/utxodb.go +++ b/uspv/utxodb.go @@ -14,7 +14,8 @@ import ( var ( BKTUtxos = []byte("DuffelBag") // leave the rest to collect interest - BKTOld = []byte("SpentTxs") // for bookkeeping + BKTStxos = []byte("SpentTxs") // for bookkeeping + BKTTxns = []byte("Txns") // all txs we care about, for replays BKTState = []byte("MiscState") // last state of DB KEYNumKeys = []byte("NumKeys") // number of keys used @@ -27,16 +28,20 @@ func (ts *TxStore) OpenDB(filename string) error { return err } // create buckets if they're not already there - return ts.StateDB.Update(func(tx *bolt.Tx) error { - _, err = tx.CreateBucketIfNotExists(BKTUtxos) + return ts.StateDB.Update(func(btx *bolt.Tx) error { + _, err = btx.CreateBucketIfNotExists(BKTUtxos) if err != nil { return err } - _, err = tx.CreateBucketIfNotExists(BKTOld) + _, err = btx.CreateBucketIfNotExists(BKTStxos) if err != nil { return err } - _, err = tx.CreateBucketIfNotExists(BKTState) + _, err = btx.CreateBucketIfNotExists(BKTTxns) + if err != nil { + return err + } + _, err = btx.CreateBucketIfNotExists(BKTState) if err != nil { return err } @@ -69,8 +74,8 @@ func (ts *TxStore) NewAdr() (*btcutil.AddressPubKeyHash, error) { } // write to db file - err = ts.StateDB.Update(func(tx *bolt.Tx) error { - stt := tx.Bucket(BKTState) + err = ts.StateDB.Update(func(btx *bolt.Tx) error { + stt := btx.Bucket(BKTState) return stt.Put(KEYNumKeys, buf.Bytes()) }) if err != nil { @@ -81,6 +86,24 @@ func (ts *TxStore) NewAdr() (*btcutil.AddressPubKeyHash, error) { return newAdr, nil } +// NumUtxos returns the number of utxos in the DB. +func (ts *TxStore) NumUtxos() (uint32, error) { + var n uint32 + err := ts.StateDB.View(func(btx *bolt.Tx) error { + duf := btx.Bucket(BKTUtxos) + if duf == nil { + return fmt.Errorf("no duffel bag") + } + stats := duf.Stats() + n = uint32(stats.KeyN) + return nil + }) + if err != nil { + return 0, err + } + return n, nil +} + // PopulateAdrs just puts a bunch of adrs in ram; it doesn't touch the DB func (ts *TxStore) PopulateAdrs(lastKey uint32) error { for k := uint32(0); k < lastKey; k++ { @@ -101,10 +124,9 @@ func (ts *TxStore) PopulateAdrs(lastKey uint32) error { } // SaveToDB write a utxo to disk, overwriting an old utxo of the same outpoint -func (u *Utxo) SaveToDB(dbx *bolt.DB) error { - - err := dbx.Update(func(tx *bolt.Tx) error { - duf := tx.Bucket(BKTUtxos) +func (ts *TxStore) SaveUtxo(u *Utxo) error { + err := ts.StateDB.Update(func(btx *bolt.Tx) error { + duf := btx.Bucket(BKTUtxos) b, err := u.ToBytes() if err != nil { return err @@ -124,26 +146,47 @@ func (u *Utxo) SaveToDB(dbx *bolt.DB) error { return nil } -func (ts *TxStore) MarkSpent(op *wire.OutPoint, h int32, stx *wire.MsgTx) error { +func (ts *TxStore) MarkSpent(ut Utxo, h int32, stx *wire.MsgTx) error { // we write in key = outpoint (32 hash, 4 index) // value = spending txid // if we care about the spending tx we can store that in another bucket. - return ts.StateDB.Update(func(tx *bolt.Tx) error { - old := tx.Bucket(BKTOld) - opb, err := outPointToBytes(op) + + var st Stxo + st.Utxo = ut + st.SpendHeight = h + st.SpendTxid = stx.TxSha() + + return ts.StateDB.Update(func(btx *bolt.Tx) error { + duf := btx.Bucket(BKTUtxos) + old := btx.Bucket(BKTStxos) + txns := btx.Bucket(BKTTxns) + + opb, err := outPointToBytes(&st.Op) if err != nil { return err } - var buf bytes.Buffer - err = binary.Write(&buf, binary.BigEndian, h) + + err = duf.Delete(opb) // not utxo anymore if err != nil { return err } + + stxb, err := st.ToBytes() + if err != nil { + return err + } + + err = old.Put(opb, stxb) // write k:v outpoint:stxo bytes + if err != nil { + return err + } + + // store spending tx sha := stx.TxSha() - err = old.Put(opb, sha.Bytes()) // write k:v outpoint:txid - if err != nil { - return err - } + var buf bytes.Buffer + stx.Serialize(&buf) + txns.Put(sha.Bytes(), buf.Bytes()) + return nil }) } @@ -155,16 +198,16 @@ func (ts *TxStore) LoadFromDB() error { if ts.rootPrivKey == nil { return fmt.Errorf("LoadFromDB needs rootPrivKey loaded") } - return ts.StateDB.View(func(tx *bolt.Tx) error { - duf := tx.Bucket(BKTUtxos) + return ts.StateDB.View(func(btx *bolt.Tx) error { + duf := btx.Bucket(BKTUtxos) if duf == nil { return fmt.Errorf("no duffel bag") } - spent := tx.Bucket(BKTOld) + spent := btx.Bucket(BKTStxos) if spent == nil { return fmt.Errorf("no spenttx bucket") } - state := tx.Bucket(BKTState) + state := btx.Bucket(BKTState) if state == nil { return fmt.Errorf("no state bucket") } @@ -268,8 +311,8 @@ func UtxoFromBytes(b []byte) (Utxo, error) { return u, fmt.Errorf("nil input slice") } buf := bytes.NewBuffer(b) - if buf.Len() < 52 { // minimum 52 bytes with no pkscript - return u, fmt.Errorf("Got %d bytes for sender, expect > 52", buf.Len()) + if buf.Len() < 52 { // utxos are 52 bytes + return u, fmt.Errorf("Got %d bytes for utxo, expect 52", buf.Len()) } // read 32 byte txid err := u.Op.Hash.SetBytes(buf.Next(32)) @@ -298,3 +341,93 @@ func UtxoFromBytes(b []byte) (Utxo, error) { } return u, nil } + +// ToBytes turns an Stxo into some bytes. +// outpoint txid, outpoint idx, height, key idx, amt, spendheight, spendtxid +func (s *Stxo) ToBytes() ([]byte, error) { + var buf bytes.Buffer + // write 32 byte txid of the utxo + _, err := buf.Write(s.Op.Hash.Bytes()) + if err != nil { + return nil, err + } + // write 4 byte outpoint index within the tx to spend + err = binary.Write(&buf, binary.BigEndian, s.Op.Index) + if err != nil { + return nil, err + } + // write 4 byte height of utxo + err = binary.Write(&buf, binary.BigEndian, s.AtHeight) + if err != nil { + return nil, err + } + // write 4 byte key index of utxo + err = binary.Write(&buf, binary.BigEndian, s.KeyIdx) + if err != nil { + return nil, err + } + // write 8 byte amount of money at the utxo + err = binary.Write(&buf, binary.BigEndian, s.Value) + if err != nil { + return nil, err + } + // write 4 byte height where the txo was spent + err = binary.Write(&buf, binary.BigEndian, s.SpendHeight) + if err != nil { + return nil, err + } + // write 32 byte txid of the spending transaction + _, err = buf.Write(s.SpendTxid.Bytes()) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// StxoFromBytes turns bytes into a Stxo. +func StxoFromBytes(b []byte) (Stxo, error) { + var s Stxo + if b == nil { + return s, fmt.Errorf("nil input slice") + } + buf := bytes.NewBuffer(b) + if buf.Len() < 88 { // stxos are 88 bytes + return s, fmt.Errorf("Got %d bytes for stxo, expect 88", buf.Len()) + } + // read 32 byte txid + err := s.Op.Hash.SetBytes(buf.Next(32)) + if err != nil { + return s, err + } + // read 4 byte outpoint index within the tx to spend + err = binary.Read(buf, binary.BigEndian, &s.Op.Index) + if err != nil { + return s, err + } + // read 4 byte height of utxo + err = binary.Read(buf, binary.BigEndian, &s.AtHeight) + if err != nil { + return s, err + } + // read 4 byte key index of utxo + err = binary.Read(buf, binary.BigEndian, &s.KeyIdx) + if err != nil { + return s, err + } + // read 8 byte amount of money at the utxo + err = binary.Read(buf, binary.BigEndian, &s.Value) + if err != nil { + return s, err + } + // read 4 byte spend height + err = binary.Read(buf, binary.BigEndian, &s.SpendHeight) + if err != nil { + return s, err + } + // read 32 byte txid + err = s.SpendTxid.SetBytes(buf.Next(32)) + if err != nil { + return s, err + } + return s, nil +} From 6ef9dc3d4acb3597a20990a82410e915efccb096 Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Thu, 28 Jan 2016 19:35:49 -0800 Subject: [PATCH 19/30] not saying it works, but it works better redo how db and header sync works, somewhat simpler but a little recursve-ish. There is still an off by 1 error somewhere with headers. --- uspv/eight333.go | 82 +++++++++++++++++++++++++--------------------- uspv/msghandler.go | 44 +++++++++++++++++++++++-- uspv/txstore.go | 60 ++++++++++++++++++--------------- uspv/utxodb.go | 76 +++++++++++++++++++++++++++++++++--------- 4 files changed, 178 insertions(+), 84 deletions(-) diff --git a/uspv/eight333.go b/uspv/eight333.go index a8e592d5..275a6493 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -45,6 +45,8 @@ type SPVCon struct { // mBlockQueue is for keeping track of what height we've requested. mBlockQueue chan HashAndHeight + // fPositives is a channel to keep track of bloom filter false positives. + fPositives chan int32 } func OpenSPV(remoteNode string, hfn, tsfn string, @@ -123,11 +125,13 @@ func OpenSPV(remoteNode string, hfn, tsfn string, } s.WBytes += uint64(n) - s.inMsgQueue = make(chan wire.Message, 1) + s.inMsgQueue = make(chan wire.Message) go s.incomingMessageHandler() - s.outMsgQueue = make(chan wire.Message, 1) + s.outMsgQueue = make(chan wire.Message) go s.outgoingMessageHandler() s.mBlockQueue = make(chan HashAndHeight, 32) // queue depth 32 is a thing + s.fPositives = make(chan int32, 4000) // a block full, approx + go s.fPositiveHandler() return s, nil } @@ -223,18 +227,10 @@ func (s *SPVCon) AskForTx(txid wire.ShaHash) { // appending and checking the header, and checking spv proofs func (s *SPVCon) AskForBlock(hsh wire.ShaHash) { - fmt.Printf("mBlockQueue len %d\n", len(s.mBlockQueue)) - // wait until all mblocks are done before adding - for len(s.mBlockQueue) != 0 { - // fmt.Printf("mBlockQueue len %d\n", len(s.mBlockQueue)) - } - gdata := wire.NewMsgGetData() inv := wire.NewInvVect(wire.InvTypeFilteredBlock, &hsh) gdata.AddInvVect(inv) - // TODO - wait until headers are sync'd before checking height - info, err := s.headerFile.Stat() // get if err != nil { log.Fatal(err) // crash if header file disappears @@ -242,10 +238,9 @@ func (s *SPVCon) AskForBlock(hsh wire.ShaHash) { nextHeight := int32(info.Size() / 80) hah := NewRootAndHeight(hsh, nextHeight) - + fmt.Printf("AskForBlock - %s height %d\n", hsh.String(), nextHeight) s.mBlockQueue <- hah // push height and mroot of requested block on queue s.outMsgQueue <- gdata // push request to outbox - } func (s *SPVCon) AskForHeaders() error { @@ -304,19 +299,25 @@ func (s *SPVCon) IngestMerkleBlock(m *wire.MsgMerkleBlock) error { if !hah.blockhash.IsEqual(&newMerkBlockSha) { return fmt.Errorf("merkle block out of order error") } + for _, txid := range txids { err := s.TS.AddTxid(txid, hah.height) if err != nil { return fmt.Errorf("Txid store error: %s\n", err.Error()) } } + err = s.TS.SetDBSyncHeight(hah.height) + if err != nil { + return err + } + return nil } // IngestHeaders takes in a bunch of headers and appends them to the // local header file, checking that they fit. If there's no headers, // it assumes we're done and returns false. If it worked it assumes there's -// more to request and returns true.9 +// more to request and returns true. func (s *SPVCon) IngestHeaders(m *wire.MsgHeaders) (bool, error) { var err error // seek to last header @@ -361,7 +362,7 @@ func (s *SPVCon) IngestHeaders(m *wire.MsgHeaders) (bool, error) { return false, fmt.Errorf("couldn't truncate header file") } } - return false, fmt.Errorf("Truncated header file to try again") + return true, fmt.Errorf("Truncated header file to try again") } for _, resphdr := range m.Headers { @@ -393,7 +394,19 @@ func (s *SPVCon) IngestHeaders(m *wire.MsgHeaders) (bool, error) { } } log.Printf("Headers to height %d OK.", tip) + // if we got post DB syncheight headers, get merkleblocks for them + // this is always true except for first pre-birthday sync + syncTip, err := s.TS.GetDBSyncHeight() + if err != nil { + return false, err + } + if syncTip < tip { + err = s.AskForMerkBlocks(syncTip, tip) + if err != nil { + return false, err + } + } return true, nil } @@ -421,7 +434,7 @@ func (s *SPVCon) PushTx(tx *wire.MsgTx) error { if err != nil { return err } - err = s.TS.AckTx(tx) + _, err = s.TS.AckTx(tx) // our own tx so don't need to track relevance if err != nil { return err } @@ -429,27 +442,32 @@ func (s *SPVCon) PushTx(tx *wire.MsgTx) error { return nil } +func (s *SPVCon) GetNextHeaderHeight() (int32, error) { + info, err := s.headerFile.Stat() // get + if err != nil { + return 0, err // crash if header file disappears + } + nextHeight := int32(info.Size() / 80) + return nextHeight, nil +} + // AskForMerkBlocks requests blocks from current to last // right now this asks for 1 block per getData message. // Maybe it's faster to ask for many in a each message? func (s *SPVCon) AskForMerkBlocks(current, last int32) error { var hdr wire.BlockHeader - info, err := s.headerFile.Stat() // get + + nextHeight, err := s.GetNextHeaderHeight() if err != nil { - return err // crash if header file disappears + return err } - nextHeight := int32(info.Size() / 80) + fmt.Printf("have headers up to height %d\n", nextHeight-1) // if last is 0, that means go as far as we can if last == 0 { last = nextHeight - 1 } fmt.Printf("will request merkleblocks %d to %d\n", current, last) - // track number of utxos - track, err := s.TS.NumUtxos() - if err != nil { - return err - } // create initial filter filt, err := s.TS.GimmeFilter() @@ -467,21 +485,6 @@ func (s *SPVCon) AskForMerkBlocks(current, last int32) error { // loop through all heights where we want merkleblocks. for current < last { // check if we need to update filter... diff of 5 utxos...? - nTrack, err := s.TS.NumUtxos() - if err != nil { - return err - } - - if track < nTrack-4 || track > nTrack+4 { - track = nTrack - filt, err := s.TS.GimmeFilter() - if err != nil { - return err - } - s.SendFilter(filt) - - fmt.Printf("sent %d byte filter\n", len(filt.MsgFilterLoad().Filter)) - } // load header from file err = hdr.Deserialize(s.headerFile) @@ -503,5 +506,8 @@ func (s *SPVCon) AskForMerkBlocks(current, last int32) error { s.mBlockQueue <- hah // push height and mroot of requested block on queue current++ } + // done syncing blocks known in header file, ask for new headers we missed + s.AskForHeaders() + return nil } diff --git a/uspv/msghandler.go b/uspv/msghandler.go index a73e872b..1eeddb5e 100644 --- a/uspv/msghandler.go +++ b/uspv/msghandler.go @@ -1,6 +1,7 @@ package uspv import ( + "fmt" "log" "github.com/btcsuite/btcd/wire" @@ -25,7 +26,7 @@ func (s *SPVCon) incomingMessageHandler() { case *wire.MsgAddr: log.Printf("got %d addresses.\n", len(m.AddrList)) case *wire.MsgPing: - log.Printf("Got a ping message. We should pong back or they will kick us off.") + // log.Printf("Got a ping message. We should pong back or they will kick us off.") s.PongBack(m.Nonce) case *wire.MsgPong: log.Printf("Got a pong response. OK.\n") @@ -45,9 +46,15 @@ func (s *SPVCon) incomingMessageHandler() { s.AskForHeaders() } case *wire.MsgTx: - err := s.TS.AckTx(m) + hits, err := s.TS.AckTx(m) if err != nil { log.Printf("Incoming Tx error: %s\n", err.Error()) + continue + } + if hits == 0 { + log.Printf("tx %s had no hits, filter false positive.", + m.TxSha().String()) + s.fPositives <- 1 // add one false positive to chan } // log.Printf("Got tx %s\n", m.TxSha().String()) case *wire.MsgReject: @@ -83,6 +90,34 @@ func (s *SPVCon) outgoingMessageHandler() { return } +// fPositiveHandler monitors false positives and when it gets enough of them, +// +func (s *SPVCon) fPositiveHandler() { + var fpAccumulator int32 + for { + fpAccumulator += <-s.fPositives // blocks here + if fpAccumulator > 7 { + filt, err := s.TS.GimmeFilter() + if err != nil { + log.Printf("Filter creation error: %s\n", err.Error()) + log.Printf("uhoh, crashing filter handler") + return + } + + // send filter + s.SendFilter(filt) + fmt.Printf("sent filter %x\n", filt.MsgFilterLoad().Filter) + // clear the channel + for len(s.fPositives) != 0 { + fpAccumulator += <-s.fPositives + } + fmt.Printf("reset %d false positives\n", fpAccumulator) + // reset accumulator + fpAccumulator = 0 + } + } +} + func (s *SPVCon) InvHandler(m *wire.MsgInv) { log.Printf("got inv. Contains:\n") for i, thing := range m.InvList { @@ -93,7 +128,10 @@ func (s *SPVCon) InvHandler(m *wire.MsgInv) { s.AskForTx(thing.Hash) } if thing.Type == wire.InvTypeBlock { // new block, ingest - s.AskForBlock(thing.Hash) + if len(s.mBlockQueue) == 0 { + // don't ask directly; instead ask for header + s.AskForHeaders() + } } } } diff --git a/uspv/txstore.go b/uspv/txstore.go index c768567a..bfbf776f 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -77,7 +77,7 @@ func (t *TxStore) AddTxid(txid *wire.ShaHash, height int32) error { if txid == nil { return fmt.Errorf("tried to add nil txid") } - log.Printf("added %s at height %d\n", txid.String(), height) + log.Printf("added %s to OKTxids at height %d\n", txid.String(), height) t.OKTxids[*txid] = height return nil } @@ -94,7 +94,7 @@ func (t *TxStore) GimmeFilter() (*bloom.Filter, error) { } elem := uint32(len(t.Adrs)) + nutxo - f := bloom.NewFilter(elem, 0, 0.001, wire.BloomUpdateAll) + f := bloom.NewFilter(elem, 0, 0.000001, wire.BloomUpdateAll) for _, a := range t.Adrs { f.Add(a.PkhAdr.ScriptAddress()) } @@ -107,30 +107,35 @@ func (t *TxStore) GimmeFilter() (*bloom.Filter, error) { } // Ingest a tx into wallet, dealing with both gains and losses -func (t *TxStore) AckTx(tx *wire.MsgTx) error { +func (t *TxStore) AckTx(tx *wire.MsgTx) (uint32, error) { + var ioHits uint32 // number of utxos changed due to this tx + inTxid := tx.TxSha() height, ok := t.OKTxids[inTxid] if !ok { - log.Printf("False postive tx? %s", TxToString(tx)) - return fmt.Errorf("we don't care about tx %s", inTxid.String()) + log.Printf("%s", TxToString(tx)) + return 0, fmt.Errorf("tx %s not in OKTxids.", inTxid.String()) } + delete(t.OKTxids, inTxid) // don't need anymore + hitsGained, err := t.AbsorbTx(tx, height) + if err != nil { + return 0, err + } + hitsLost, err := t.ExpellTx(tx, height) + if err != nil { + return 0, err + } + ioHits = hitsGained + hitsLost - err := t.AbsorbTx(tx, height) - if err != nil { - return err - } - err = t.ExpellTx(tx, height) - if err != nil { - return err - } // fmt.Printf("ingested tx %s total amt %d\n", inTxid.String(), t.Sum) - return nil + return ioHits, nil } -// Absorb money into wallet from a tx -func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height int32) error { +// AbsorbTx Absorbs money into wallet from a tx. returns number of +// new utxos absorbed. +func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height int32) (uint32, error) { if tx == nil { - return fmt.Errorf("Tried to add nil tx") + return 0, fmt.Errorf("Tried to add nil tx") } newTxid := tx.TxSha() var hits uint32 // how many outputs of this tx are ours @@ -145,11 +150,12 @@ func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height int32) error { dup = OutPointsEqual(*newOp, u.Op) // is this outpoint known? if dup { // found dupe fmt.Printf(" %s is dupe\t", newOp.String()) + hits++ // thought a dupe, still a hit u.AtHeight = height // ONLY difference is height // save modified utxo to db, overwriting old one err := t.SaveUtxo(u) if err != nil { - return err + return 0, err } break // out of the t.Utxo range loop } @@ -164,7 +170,7 @@ func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height int32) error { // check for full script to eliminate false positives aPKscript, err := txscript.PayToAddrScript(a.PkhAdr) if err != nil { - return err + return 0, err } if bytes.Equal(out.PkScript, aPKscript) { // hit // already checked for dupes, so this must be a new outpoint @@ -179,7 +185,7 @@ func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height int32) error { newu.Op = newop err = t.SaveUtxo(&newu) if err != nil { - return err + return 0, err } acq += out.Value @@ -189,15 +195,15 @@ func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height int32) error { } } } - log.Printf("%d hits, acquired %d", hits, acq) + // log.Printf("%d hits, acquired %d", hits, acq) t.Sum += acq - return nil + return hits, nil } // Expell money from wallet due to a tx -func (t *TxStore) ExpellTx(tx *wire.MsgTx, height int32) error { +func (t *TxStore) ExpellTx(tx *wire.MsgTx, height int32) (uint32, error) { if tx == nil { - return fmt.Errorf("Tried to add nil tx") + return 0, fmt.Errorf("Tried to add nil tx") } var hits uint32 var loss int64 @@ -209,16 +215,16 @@ func (t *TxStore) ExpellTx(tx *wire.MsgTx, height int32) error { loss += myutxo.Value err := t.MarkSpent(*myutxo, height, tx) if err != nil { - return err + return 0, err } // delete from my in-ram utxo set t.Utxos = append(t.Utxos[:i], t.Utxos[i+1:]...) } } } - log.Printf("%d hits, lost %d", hits, loss) + // log.Printf("%d hits, lost %d", hits, loss) t.Sum -= loss - return nil + return hits, nil } // need this because before I was comparing pointers maybe? diff --git a/uspv/utxodb.go b/uspv/utxodb.go index d6d5763b..714ecf5f 100644 --- a/uspv/utxodb.go +++ b/uspv/utxodb.go @@ -17,8 +17,9 @@ var ( BKTStxos = []byte("SpentTxs") // for bookkeeping BKTTxns = []byte("Txns") // all txs we care about, for replays BKTState = []byte("MiscState") // last state of DB - - KEYNumKeys = []byte("NumKeys") // number of keys used + // these are in the state bucket + KEYNumKeys = []byte("NumKeys") // number of keys used + KEYTipHeight = []byte("TipHeight") // height synced to ) func (ts *TxStore) OpenDB(filename string) error { @@ -75,8 +76,8 @@ func (ts *TxStore) NewAdr() (*btcutil.AddressPubKeyHash, error) { // write to db file err = ts.StateDB.Update(func(btx *bolt.Tx) error { - stt := btx.Bucket(BKTState) - return stt.Put(KEYNumKeys, buf.Bytes()) + sta := btx.Bucket(BKTState) + return sta.Put(KEYNumKeys, buf.Bytes()) }) if err != nil { return nil, err @@ -86,6 +87,44 @@ func (ts *TxStore) NewAdr() (*btcutil.AddressPubKeyHash, error) { return newAdr, nil } +// SetBDay sets the birthday (birth height) of the db (really keyfile) +func (ts *TxStore) SetDBSyncHeight(n int32) error { + var buf bytes.Buffer + _ = binary.Write(&buf, binary.BigEndian, n) + + return ts.StateDB.Update(func(btx *bolt.Tx) error { + sta := btx.Bucket(BKTState) + return sta.Put(KEYTipHeight, buf.Bytes()) + }) +} + +// SyncHeight returns the chain height to which the db has synced +func (ts *TxStore) GetDBSyncHeight() (int32, error) { + var n int32 + err := ts.StateDB.View(func(btx *bolt.Tx) error { + sta := btx.Bucket(BKTState) + if sta == nil { + return fmt.Errorf("no state") + } + t := sta.Get(KEYTipHeight) + + if t == nil { // no height written, so 0 + return nil + } + + // read 4 byte tip height to n + err := binary.Read(bytes.NewBuffer(t), binary.BigEndian, &n) + if err != nil { + return err + } + return nil + }) + if err != nil { + return 0, err + } + return n, nil +} + // NumUtxos returns the number of utxos in the DB. func (ts *TxStore) NumUtxos() (uint32, error) { var n uint32 @@ -125,17 +164,22 @@ func (ts *TxStore) PopulateAdrs(lastKey uint32) error { // SaveToDB write a utxo to disk, overwriting an old utxo of the same outpoint func (ts *TxStore) SaveUtxo(u *Utxo) error { - err := ts.StateDB.Update(func(btx *bolt.Tx) error { + b, err := u.ToBytes() + if err != nil { + return err + } + + err = ts.StateDB.Update(func(btx *bolt.Tx) error { duf := btx.Bucket(BKTUtxos) - b, err := u.ToBytes() - if err != nil { - return err + sta := btx.Bucket(BKTState) + // kindof hack, height is 36:40 + // also not really tip height... + if u.AtHeight > 0 { // if confirmed + err = sta.Put(KEYTipHeight, b[36:40]) + if err != nil { + return err + } } - // don't check for dupes here, check in AbsorbTx(). here overwrite. - // if duf.Get(b[:36]) != nil { // already have tx - // dupe = true - // return nil - // } // key : val is txid:everything else return duf.Put(b[:36], b[36:]) @@ -207,12 +251,12 @@ func (ts *TxStore) LoadFromDB() error { if spent == nil { return fmt.Errorf("no spenttx bucket") } - state := btx.Bucket(BKTState) - if state == nil { + sta := btx.Bucket(BKTState) + if sta == nil { return fmt.Errorf("no state bucket") } // first populate addresses from state bucket - numKeysBytes := state.Get(KEYNumKeys) + numKeysBytes := sta.Get(KEYNumKeys) if numKeysBytes != nil { // NumKeys exists, read into uint32 buf := bytes.NewBuffer(numKeysBytes) var numKeys uint32 From 5c2bbff3eb042a862f73c3ed00ef653c59c8dc42 Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Thu, 28 Jan 2016 22:31:05 -0800 Subject: [PATCH 20/30] fix more sync problems. Seems to work OK One edge case is with long chains of unconfirmed txs. All self-signed txs are stored, but not used to negate false incoming utxos yet. --- uspv/eight333.go | 24 ++++++++++++++++++----- uspv/msghandler.go | 47 +++++++++++++++++++++++++--------------------- 2 files changed, 45 insertions(+), 26 deletions(-) diff --git a/uspv/eight333.go b/uspv/eight333.go index 275a6493..247d2815 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -401,7 +401,9 @@ func (s *SPVCon) IngestHeaders(m *wire.MsgHeaders) (bool, error) { if err != nil { return false, err } + if syncTip < tip { + fmt.Printf("syncTip %d headerTip %d\n", syncTip, tip) err = s.AskForMerkBlocks(syncTip, tip) if err != nil { return false, err @@ -451,6 +453,18 @@ func (s *SPVCon) GetNextHeaderHeight() (int32, error) { return nextHeight, nil } +func (s *SPVCon) RemoveHeaders(r int32) error { + endPos, err := s.headerFile.Seek(0, os.SEEK_END) + if err != nil { + return err + } + err = s.headerFile.Truncate(endPos - int64(r*80)) + if err != nil { + return fmt.Errorf("couldn't truncate header file") + } + return nil +} + // AskForMerkBlocks requests blocks from current to last // right now this asks for 1 block per getData message. // Maybe it's faster to ask for many in a each message? @@ -464,8 +478,8 @@ func (s *SPVCon) AskForMerkBlocks(current, last int32) error { fmt.Printf("have headers up to height %d\n", nextHeight-1) // if last is 0, that means go as far as we can - if last == 0 { - last = nextHeight - 1 + if last < current { + return fmt.Errorf("MBlock range %d < %d\n", last, current) } fmt.Printf("will request merkleblocks %d to %d\n", current, last) @@ -483,9 +497,7 @@ func (s *SPVCon) AskForMerkBlocks(current, last int32) error { return err } // loop through all heights where we want merkleblocks. - for current < last { - // check if we need to update filter... diff of 5 utxos...? - + for current <= last { // load header from file err = hdr.Deserialize(s.headerFile) if err != nil { @@ -506,6 +518,8 @@ func (s *SPVCon) AskForMerkBlocks(current, last int32) error { s.mBlockQueue <- hah // push height and mroot of requested block on queue current++ } + fmt.Printf("mblock reqs done, more headers\n") + // done syncing blocks known in header file, ask for new headers we missed s.AskForHeaders() diff --git a/uspv/msghandler.go b/uspv/msghandler.go index 1eeddb5e..88b6f4b1 100644 --- a/uspv/msghandler.go +++ b/uspv/msghandler.go @@ -27,7 +27,7 @@ func (s *SPVCon) incomingMessageHandler() { log.Printf("got %d addresses.\n", len(m.AddrList)) case *wire.MsgPing: // log.Printf("Got a ping message. We should pong back or they will kick us off.") - s.PongBack(m.Nonce) + go s.PongBack(m.Nonce) case *wire.MsgPong: log.Printf("Got a pong response. OK.\n") case *wire.MsgMerkleBlock: @@ -37,32 +37,14 @@ func (s *SPVCon) incomingMessageHandler() { continue } case *wire.MsgHeaders: - moar, err := s.IngestHeaders(m) - if err != nil { - log.Printf("Header error: %s\n", err.Error()) - return - } - if moar { - s.AskForHeaders() - } + go s.HeaderHandler(m) case *wire.MsgTx: - hits, err := s.TS.AckTx(m) - if err != nil { - log.Printf("Incoming Tx error: %s\n", err.Error()) - continue - } - if hits == 0 { - log.Printf("tx %s had no hits, filter false positive.", - m.TxSha().String()) - s.fPositives <- 1 // add one false positive to chan - } - // log.Printf("Got tx %s\n", m.TxSha().String()) + go s.TxHandler(m) case *wire.MsgReject: log.Printf("Rejected! cmd: %s code: %s tx: %s reason: %s", m.Cmd, m.Code.String(), m.Hash.String(), m.Reason) case *wire.MsgInv: go s.InvHandler(m) - case *wire.MsgNotFound: log.Printf("Got not found response from remote:") for i, thing := range m.InvList { @@ -118,6 +100,29 @@ func (s *SPVCon) fPositiveHandler() { } } +func (s *SPVCon) HeaderHandler(m *wire.MsgHeaders) { + moar, err := s.IngestHeaders(m) + if err != nil { + log.Printf("Header error: %s\n", err.Error()) + return + } + if moar { + s.AskForHeaders() + } +} + +func (s *SPVCon) TxHandler(m *wire.MsgTx) { + hits, err := s.TS.AckTx(m) + if err != nil { + log.Printf("Incoming Tx error: %s\n", err.Error()) + } + if hits == 0 { + log.Printf("tx %s had no hits, filter false positive.", + m.TxSha().String()) + s.fPositives <- 1 // add one false positive to chan + } +} + func (s *SPVCon) InvHandler(m *wire.MsgInv) { log.Printf("got inv. Contains:\n") for i, thing := range m.InvList { From 4cd9087f9fe274b01d3c97343d4afd1235c0d2da Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Fri, 29 Jan 2016 00:40:52 -0800 Subject: [PATCH 21/30] problems with concurrent disk access (header file) maybe add mutexes or something --- uspv/eight333.go | 23 ++++------------------- uspv/msghandler.go | 33 ++++++++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 20 deletions(-) diff --git a/uspv/eight333.go b/uspv/eight333.go index 247d2815..d7fedbb3 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -394,21 +394,7 @@ func (s *SPVCon) IngestHeaders(m *wire.MsgHeaders) (bool, error) { } } log.Printf("Headers to height %d OK.", tip) - // if we got post DB syncheight headers, get merkleblocks for them - // this is always true except for first pre-birthday sync - syncTip, err := s.TS.GetDBSyncHeight() - if err != nil { - return false, err - } - - if syncTip < tip { - fmt.Printf("syncTip %d headerTip %d\n", syncTip, tip) - err = s.AskForMerkBlocks(syncTip, tip) - if err != nil { - return false, err - } - } return true, nil } @@ -492,7 +478,7 @@ func (s *SPVCon) AskForMerkBlocks(current, last int32) error { s.SendFilter(filt) fmt.Printf("sent filter %x\n", filt.MsgFilterLoad().Filter) - _, err = s.headerFile.Seek(int64(current*80), os.SEEK_SET) + _, err = s.headerFile.Seek(int64((current-1)*80), os.SEEK_SET) if err != nil { return err } @@ -501,6 +487,7 @@ func (s *SPVCon) AskForMerkBlocks(current, last int32) error { // load header from file err = hdr.Deserialize(s.headerFile) if err != nil { + log.Printf("Deserialize err\n") return err } @@ -518,10 +505,8 @@ func (s *SPVCon) AskForMerkBlocks(current, last int32) error { s.mBlockQueue <- hah // push height and mroot of requested block on queue current++ } - fmt.Printf("mblock reqs done, more headers\n") - // done syncing blocks known in header file, ask for new headers we missed - s.AskForHeaders() - + // s.AskForHeaders() + // don't need this -- will sync to end regardless return nil } diff --git a/uspv/msghandler.go b/uspv/msghandler.go index 88b6f4b1..b0ccf99b 100644 --- a/uspv/msghandler.go +++ b/uspv/msghandler.go @@ -3,6 +3,7 @@ package uspv import ( "fmt" "log" + "os" "github.com/btcsuite/btcd/wire" ) @@ -106,9 +107,33 @@ func (s *SPVCon) HeaderHandler(m *wire.MsgHeaders) { log.Printf("Header error: %s\n", err.Error()) return } + // if we got post DB syncheight headers, get merkleblocks for them + // this is always true except for first pre-birthday sync + syncTip, err := s.TS.GetDBSyncHeight() + if err != nil { + log.Printf("Header error: %s", err.Error()) + return + } + endPos, err := s.headerFile.Seek(0, os.SEEK_END) + if err != nil { + log.Printf("Header error: %s", err.Error()) + return + } + tip := int32(endPos/80) - 1 // move back 1 header length to read + + // checked header lenght, start req for more if needed if moar { s.AskForHeaders() } + + if syncTip < tip { + fmt.Printf("syncTip %d headerTip %d\n", syncTip, tip) + err = s.AskForMerkBlocks(syncTip, tip) + if err != nil { + log.Printf("AskForMerkBlocks error: %s", err.Error()) + return + } + } } func (s *SPVCon) TxHandler(m *wire.MsgTx) { @@ -120,6 +145,9 @@ func (s *SPVCon) TxHandler(m *wire.MsgTx) { log.Printf("tx %s had no hits, filter false positive.", m.TxSha().String()) s.fPositives <- 1 // add one false positive to chan + } else { + log.Printf("tx %s ingested and matches utxo/adrs.", + m.TxSha().String()) } } @@ -135,7 +163,10 @@ func (s *SPVCon) InvHandler(m *wire.MsgInv) { if thing.Type == wire.InvTypeBlock { // new block, ingest if len(s.mBlockQueue) == 0 { // don't ask directly; instead ask for header - s.AskForHeaders() + fmt.Printf("asking for headers due to inv block\n") + // s.AskForHeaders() + } else { + fmt.Printf("inv block but ignoring, not synched\n") } } } From 83dff432b16f07bc82a1878f981747facc701a8a Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Fri, 29 Jan 2016 00:48:14 -0800 Subject: [PATCH 22/30] mutex helps, still some sequence errors from mblocks Seems that merkle blocks can return txs out of sequence, unlike regular blocks which have to be in temporal/sequential order. so maybe want to absorb ALL txs from mblock, then expel all? --- uspv/eight333.go | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/uspv/eight333.go b/uspv/eight333.go index d7fedbb3..73c21b99 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -7,6 +7,7 @@ import ( "log" "net" "os" + "sync" "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcd/wire" @@ -24,8 +25,10 @@ const ( ) type SPVCon struct { - con net.Conn // the (probably tcp) connection to the node - headerFile *os.File // file for SPV headers + con net.Conn // the (probably tcp) connection to the node + + headerMutex sync.Mutex + headerFile *os.File // file for SPV headers //[doesn't work without fancy mutexes, nevermind, just use header file] // localHeight int32 // block height we're on @@ -182,6 +185,8 @@ func (s *SPVCon) HeightFromHeader(query wire.BlockHeader) (uint32, error) { // happened recently. // seek to last header + s.headerMutex.Lock() + defer s.headerMutex.Unlock() lastPos, err := s.headerFile.Seek(-80, os.SEEK_END) if err != nil { return 0, err @@ -226,6 +231,8 @@ func (s *SPVCon) AskForTx(txid wire.ShaHash) { // We don't have it in our header file so when we get it we do both operations: // appending and checking the header, and checking spv proofs func (s *SPVCon) AskForBlock(hsh wire.ShaHash) { + s.headerMutex.Lock() + defer s.headerMutex.Unlock() gdata := wire.NewMsgGetData() inv := wire.NewInvVect(wire.InvTypeFilteredBlock, &hsh) @@ -241,9 +248,13 @@ func (s *SPVCon) AskForBlock(hsh wire.ShaHash) { fmt.Printf("AskForBlock - %s height %d\n", hsh.String(), nextHeight) s.mBlockQueue <- hah // push height and mroot of requested block on queue s.outMsgQueue <- gdata // push request to outbox + return } func (s *SPVCon) AskForHeaders() error { + s.headerMutex.Lock() + defer s.headerMutex.Unlock() + var hdr wire.BlockHeader ghdr := wire.NewMsgGetHeaders() ghdr.ProtocolVersion = s.localVersion @@ -319,6 +330,9 @@ func (s *SPVCon) IngestMerkleBlock(m *wire.MsgMerkleBlock) error { // it assumes we're done and returns false. If it worked it assumes there's // more to request and returns true. func (s *SPVCon) IngestHeaders(m *wire.MsgHeaders) (bool, error) { + s.headerMutex.Lock() + defer s.headerMutex.Unlock() + var err error // seek to last header _, err = s.headerFile.Seek(-80, os.SEEK_END) @@ -431,6 +445,8 @@ func (s *SPVCon) PushTx(tx *wire.MsgTx) error { } func (s *SPVCon) GetNextHeaderHeight() (int32, error) { + s.headerMutex.Lock() + defer s.headerMutex.Unlock() info, err := s.headerFile.Stat() // get if err != nil { return 0, err // crash if header file disappears @@ -478,6 +494,9 @@ func (s *SPVCon) AskForMerkBlocks(current, last int32) error { s.SendFilter(filt) fmt.Printf("sent filter %x\n", filt.MsgFilterLoad().Filter) + s.headerMutex.Lock() + defer s.headerMutex.Unlock() + _, err = s.headerFile.Seek(int64((current-1)*80), os.SEEK_SET) if err != nil { return err From 3b774ef3613c7f9b7c340de5c6f21a9e51f47434 Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Sun, 31 Jan 2016 01:05:31 -0800 Subject: [PATCH 23/30] move tx ingest to all db, fixes sync Sync was non-deterministic because ingest was concurrent. Now receiving tx messages is blocking, but that's OK, they really need to be in the right order because the whole point of bitcoin is to put txs in the right order. SendTx still has a problem that the change address may not be recognized by ingest. --- uspv/eight333.go | 4 +- uspv/msghandler.go | 10 ++-- uspv/txstore.go | 12 +++- uspv/utxodb.go | 134 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 150 insertions(+), 10 deletions(-) diff --git a/uspv/eight333.go b/uspv/eight333.go index 73c21b99..c2d896b3 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -402,7 +402,7 @@ func (s *SPVCon) IngestHeaders(m *wire.MsgHeaders) (bool, error) { } // probably should disconnect from spv node at this point, // since they're giving us invalid headers. - return false, fmt.Errorf( + return true, fmt.Errorf( "Header %d - %s doesn't fit, dropping 100 headers.", resphdr.BlockSha().String(), tip) } @@ -436,7 +436,7 @@ func (s *SPVCon) PushTx(tx *wire.MsgTx) error { if err != nil { return err } - _, err = s.TS.AckTx(tx) // our own tx so don't need to track relevance + _, err = s.TS.Ingest(tx) // our own tx so don't need to track relevance if err != nil { return err } diff --git a/uspv/msghandler.go b/uspv/msghandler.go index b0ccf99b..16aceb08 100644 --- a/uspv/msghandler.go +++ b/uspv/msghandler.go @@ -39,8 +39,8 @@ func (s *SPVCon) incomingMessageHandler() { } case *wire.MsgHeaders: go s.HeaderHandler(m) - case *wire.MsgTx: - go s.TxHandler(m) + case *wire.MsgTx: // can't be concurrent! out of order kills + s.TxHandler(m) case *wire.MsgReject: log.Printf("Rejected! cmd: %s code: %s tx: %s reason: %s", m.Cmd, m.Code.String(), m.Hash.String(), m.Reason) @@ -137,7 +137,7 @@ func (s *SPVCon) HeaderHandler(m *wire.MsgHeaders) { } func (s *SPVCon) TxHandler(m *wire.MsgTx) { - hits, err := s.TS.AckTx(m) + hits, err := s.TS.Ingest(m) if err != nil { log.Printf("Incoming Tx error: %s\n", err.Error()) } @@ -146,8 +146,8 @@ func (s *SPVCon) TxHandler(m *wire.MsgTx) { m.TxSha().String()) s.fPositives <- 1 // add one false positive to chan } else { - log.Printf("tx %s ingested and matches utxo/adrs.", - m.TxSha().String()) + log.Printf("tx %s ingested and matches utxo/adrs. sum %d", + m.TxSha().String(), s.TS.Sum) } } diff --git a/uspv/txstore.go b/uspv/txstore.go index bfbf776f..0a7d6417 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -98,8 +98,14 @@ func (t *TxStore) GimmeFilter() (*bloom.Filter, error) { for _, a := range t.Adrs { f.Add(a.PkhAdr.ScriptAddress()) } - // add txids of utxos to look for outgoing - for _, u := range t.Utxos { + + // get all utxos to add outpoints to filter + allUtxos, err := t.GetAllUtxos() + if err != nil { + return nil, err + } + + for _, u := range allUtxos { f.AddOutPoint(&u.Op) } @@ -107,7 +113,7 @@ func (t *TxStore) GimmeFilter() (*bloom.Filter, error) { } // Ingest a tx into wallet, dealing with both gains and losses -func (t *TxStore) AckTx(tx *wire.MsgTx) (uint32, error) { +func (t *TxStore) AckTxz(tx *wire.MsgTx) (uint32, error) { var ioHits uint32 // number of utxos changed due to this tx inTxid := tx.TxSha() diff --git a/uspv/utxodb.go b/uspv/utxodb.go index 714ecf5f..9949659e 100644 --- a/uspv/utxodb.go +++ b/uspv/utxodb.go @@ -5,6 +5,8 @@ import ( "encoding/binary" "fmt" + "github.com/btcsuite/btcd/txscript" + "github.com/btcsuite/btcd/wire" "github.com/btcsuite/btcutil" "github.com/btcsuite/btcutil/hdkeychain" @@ -143,6 +145,39 @@ func (ts *TxStore) NumUtxos() (uint32, error) { return n, nil } +func (ts *TxStore) GetAllUtxos() ([]*Utxo, error) { + var utxos []*Utxo + err := ts.StateDB.View(func(btx *bolt.Tx) error { + duf := btx.Bucket(BKTUtxos) + if duf == nil { + return fmt.Errorf("no duffel bag") + } + return duf.ForEach(func(k, v []byte) error { + // have to copy k and v here, otherwise append will crash it. + // not quite sure why but append does weird stuff I guess. + + // create a new utxo + x := make([]byte, len(k)+len(v)) + copy(x, k) + copy(x[len(k):], v) + newU, err := UtxoFromBytes(x) + if err != nil { + return err + } + // and add it to ram + utxos = append(utxos, &newU) + + return nil + }) + + return nil + }) + if err != nil { + return nil, err + } + return utxos, nil +} + // PopulateAdrs just puts a bunch of adrs in ram; it doesn't touch the DB func (ts *TxStore) PopulateAdrs(lastKey uint32) error { for k := uint32(0); k < lastKey; k++ { @@ -162,6 +197,105 @@ func (ts *TxStore) PopulateAdrs(lastKey uint32) error { return nil } +// Ingest puts a tx into the DB atomically. This can result in a +// gain, a loss, or no result. Gain or loss in satoshis is returned. +func (ts *TxStore) Ingest(tx *wire.MsgTx) (uint32, error) { + var hits uint32 + var err error + var spentOPs [][]byte + var nUtxoBytes [][]byte + + // check that we have a height and tx has been SPV OK'd + inTxid := tx.TxSha() + height, ok := ts.OKTxids[inTxid] + if !ok { + return hits, fmt.Errorf("Ingest error: tx %s not in OKTxids.", + inTxid.String()) + } + + // before entering into db, serialize all inputs of the ingested tx + for _, txin := range tx.TxIn { + nOP, err := outPointToBytes(&txin.PreviousOutPoint) + if err != nil { + return hits, err + } + spentOPs = append(spentOPs, nOP) + } + // also generate PKscripts for all addresses (maybe keep storing these?) + for _, adr := range ts.Adrs { + // iterate through all our addresses + aPKscript, err := txscript.PayToAddrScript(adr.PkhAdr) + if err != nil { + return hits, err + } + // iterate through all outputs of this tx + for i, out := range tx.TxOut { + if bytes.Equal(out.PkScript, aPKscript) { // new utxo for us + var newu Utxo + newu.AtHeight = height + newu.KeyIdx = adr.KeyIdx + newu.Value = out.Value + var newop wire.OutPoint + newop.Hash = tx.TxSha() + newop.Index = uint32(i) + newu.Op = newop + b, err := newu.ToBytes() + if err != nil { + return hits, err + } + nUtxoBytes = append(nUtxoBytes, b) + ts.Sum += newu.Value + hits++ + } + break // only one match + } + } + + err = ts.StateDB.Update(func(btx *bolt.Tx) error { + // get all 4 buckets + duf := btx.Bucket(BKTUtxos) + // sta := btx.Bucket(BKTState) + // old := btx.Bucket(BKTStxos) + // txns := btx.Bucket(BKTTxns) + + // first see if we lose utxos + // iterate through duffel bag and look for matches + // this makes us lose money, which is regrettable, but we need to know. + for _, nOP := range spentOPs { + duf.ForEach(func(k, v []byte) error { + if bytes.Equal(k, nOP) { // matched, we lost utxo + // do all this just to figure out value we lost + x := make([]byte, len(k)+len(v)) + copy(x, k) + copy(x[len(k):], v) + lostTxo, err := UtxoFromBytes(x) + if err != nil { + return err + } + ts.Sum -= lostTxo.Value + hits++ + // then delete the utxo from duf, save to old + err = duf.Delete(k) + if err != nil { + return err + } + return nil // matched utxo k, won't match another + } + return nil // no match + }) + } // done losing utxos + // next add all new utxos to db, this is quick as the work is above + for _, ub := range nUtxoBytes { + err = duf.Put(ub[:36], ub[36:]) + if err != nil { + return err + } + } + return nil + }) + return hits, err +} + // SaveToDB write a utxo to disk, overwriting an old utxo of the same outpoint func (ts *TxStore) SaveUtxo(u *Utxo) error { b, err := u.ToBytes() From cf01e02d64b25559a73db414dc5552888ce394fe Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Sun, 31 Jan 2016 02:08:39 -0800 Subject: [PATCH 24/30] fix dumb loop break error break was outside the if bytes.Equal {}, oops. only checked first output. Works now. Concurrency also seems OK but need to test more. --- uspv/eight333.go | 3 +- uspv/msghandler.go | 45 ++++++++-------- uspv/txstore.go | 124 --------------------------------------------- uspv/utxodb.go | 3 +- 4 files changed, 27 insertions(+), 148 deletions(-) diff --git a/uspv/eight333.go b/uspv/eight333.go index c2d896b3..49a8a176 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -230,7 +230,7 @@ func (s *SPVCon) AskForTx(txid wire.ShaHash) { // AskForBlock requests a merkle block we heard about from an inv message. // We don't have it in our header file so when we get it we do both operations: // appending and checking the header, and checking spv proofs -func (s *SPVCon) AskForBlock(hsh wire.ShaHash) { +func (s *SPVCon) AskForBlockx(hsh wire.ShaHash) { s.headerMutex.Lock() defer s.headerMutex.Unlock() @@ -493,7 +493,6 @@ func (s *SPVCon) AskForMerkBlocks(current, last int32) error { // send filter s.SendFilter(filt) fmt.Printf("sent filter %x\n", filt.MsgFilterLoad().Filter) - s.headerMutex.Lock() defer s.headerMutex.Unlock() diff --git a/uspv/msghandler.go b/uspv/msghandler.go index 16aceb08..3e1ffef9 100644 --- a/uspv/msghandler.go +++ b/uspv/msghandler.go @@ -37,9 +37,9 @@ func (s *SPVCon) incomingMessageHandler() { log.Printf("Merkle block error: %s\n", err.Error()) continue } - case *wire.MsgHeaders: + case *wire.MsgHeaders: // concurrent because we keep asking for blocks go s.HeaderHandler(m) - case *wire.MsgTx: // can't be concurrent! out of order kills + case *wire.MsgTx: // not concurrent! txs must be in order s.TxHandler(m) case *wire.MsgReject: log.Printf("Rejected! cmd: %s code: %s tx: %s reason: %s", @@ -109,30 +109,33 @@ func (s *SPVCon) HeaderHandler(m *wire.MsgHeaders) { } // if we got post DB syncheight headers, get merkleblocks for them // this is always true except for first pre-birthday sync - syncTip, err := s.TS.GetDBSyncHeight() - if err != nil { - log.Printf("Header error: %s", err.Error()) - return - } - endPos, err := s.headerFile.Seek(0, os.SEEK_END) - if err != nil { - log.Printf("Header error: %s", err.Error()) - return - } - tip := int32(endPos/80) - 1 // move back 1 header length to read - // checked header lenght, start req for more if needed + // checked header length, start req for more if needed if moar { s.AskForHeaders() - } - - if syncTip < tip { - fmt.Printf("syncTip %d headerTip %d\n", syncTip, tip) - err = s.AskForMerkBlocks(syncTip, tip) + } else { // no moar, done w/ headers, get merkleblocks + fmt.Printf("locks here...?? ") + s.headerMutex.Lock() + endPos, err := s.headerFile.Seek(0, os.SEEK_END) if err != nil { - log.Printf("AskForMerkBlocks error: %s", err.Error()) + log.Printf("Header error: %s", err.Error()) return } + s.headerMutex.Unlock() + tip := int32(endPos/80) - 1 // move back 1 header length to read + syncTip, err := s.TS.GetDBSyncHeight() + if err != nil { + log.Printf("syncTip error: %s", err.Error()) + return + } + if syncTip < tip { + fmt.Printf("syncTip %d headerTip %d\n", syncTip, tip) + err = s.AskForMerkBlocks(syncTip+1, tip) + if err != nil { + log.Printf("AskForMerkBlocks error: %s", err.Error()) + return + } + } } } @@ -164,7 +167,7 @@ func (s *SPVCon) InvHandler(m *wire.MsgInv) { if len(s.mBlockQueue) == 0 { // don't ask directly; instead ask for header fmt.Printf("asking for headers due to inv block\n") - // s.AskForHeaders() + s.AskForHeaders() } else { fmt.Printf("inv block but ignoring, not synched\n") } diff --git a/uspv/txstore.go b/uspv/txstore.go index 0a7d6417..8eab9947 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -1,12 +1,9 @@ package uspv import ( - "bytes" "fmt" "log" - "github.com/btcsuite/btcd/txscript" - "github.com/btcsuite/btcd/chaincfg" "github.com/boltdb/bolt" @@ -112,127 +109,6 @@ func (t *TxStore) GimmeFilter() (*bloom.Filter, error) { return f, nil } -// Ingest a tx into wallet, dealing with both gains and losses -func (t *TxStore) AckTxz(tx *wire.MsgTx) (uint32, error) { - var ioHits uint32 // number of utxos changed due to this tx - - inTxid := tx.TxSha() - height, ok := t.OKTxids[inTxid] - if !ok { - log.Printf("%s", TxToString(tx)) - return 0, fmt.Errorf("tx %s not in OKTxids.", inTxid.String()) - } - delete(t.OKTxids, inTxid) // don't need anymore - hitsGained, err := t.AbsorbTx(tx, height) - if err != nil { - return 0, err - } - hitsLost, err := t.ExpellTx(tx, height) - if err != nil { - return 0, err - } - ioHits = hitsGained + hitsLost - - // fmt.Printf("ingested tx %s total amt %d\n", inTxid.String(), t.Sum) - return ioHits, nil -} - -// AbsorbTx Absorbs money into wallet from a tx. returns number of -// new utxos absorbed. -func (t *TxStore) AbsorbTx(tx *wire.MsgTx, height int32) (uint32, error) { - if tx == nil { - return 0, fmt.Errorf("Tried to add nil tx") - } - newTxid := tx.TxSha() - var hits uint32 // how many outputs of this tx are ours - var acq int64 // total acquirement from this tx - // check if any of the tx's outputs match my known outpoints - for i, out := range tx.TxOut { // in each output of tx - dup := false // start by assuming its new until found duplicate - newOp := wire.NewOutPoint(&newTxid, uint32(i)) - // first look for dupes -- already known outpoints. - // if we find a dupe here overwrite it to the DB. - for _, u := range t.Utxos { - dup = OutPointsEqual(*newOp, u.Op) // is this outpoint known? - if dup { // found dupe - fmt.Printf(" %s is dupe\t", newOp.String()) - hits++ // thought a dupe, still a hit - u.AtHeight = height // ONLY difference is height - // save modified utxo to db, overwriting old one - err := t.SaveUtxo(u) - if err != nil { - return 0, err - } - break // out of the t.Utxo range loop - } - } - if dup { - // if we found the outpoint to be a dup above, don't add it again - // when it matches an address, just go to the next outpoint - continue - } - // check if this is a new txout matching one of my addresses - for _, a := range t.Adrs { // compare to each adr we have - // check for full script to eliminate false positives - aPKscript, err := txscript.PayToAddrScript(a.PkhAdr) - if err != nil { - return 0, err - } - if bytes.Equal(out.PkScript, aPKscript) { // hit - // already checked for dupes, so this must be a new outpoint - var newu Utxo - newu.AtHeight = height - newu.KeyIdx = a.KeyIdx - newu.Value = out.Value - - var newop wire.OutPoint - newop.Hash = tx.TxSha() - newop.Index = uint32(i) - newu.Op = newop - err = t.SaveUtxo(&newu) - if err != nil { - return 0, err - } - - acq += out.Value - hits++ - t.Utxos = append(t.Utxos, &newu) // always add new utxo - break - } - } - } - // log.Printf("%d hits, acquired %d", hits, acq) - t.Sum += acq - return hits, nil -} - -// Expell money from wallet due to a tx -func (t *TxStore) ExpellTx(tx *wire.MsgTx, height int32) (uint32, error) { - if tx == nil { - return 0, fmt.Errorf("Tried to add nil tx") - } - var hits uint32 - var loss int64 - - for _, in := range tx.TxIn { - for i, myutxo := range t.Utxos { - if OutPointsEqual(myutxo.Op, in.PreviousOutPoint) { - hits++ - loss += myutxo.Value - err := t.MarkSpent(*myutxo, height, tx) - if err != nil { - return 0, err - } - // delete from my in-ram utxo set - t.Utxos = append(t.Utxos[:i], t.Utxos[i+1:]...) - } - } - } - // log.Printf("%d hits, lost %d", hits, loss) - t.Sum -= loss - return hits, nil -} - // need this because before I was comparing pointers maybe? // so they were the same outpoint but stored in 2 places so false negative? func OutPointsEqual(a, b wire.OutPoint) bool { diff --git a/uspv/utxodb.go b/uspv/utxodb.go index 9949659e..34e48527 100644 --- a/uspv/utxodb.go +++ b/uspv/utxodb.go @@ -246,8 +246,9 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx) (uint32, error) { nUtxoBytes = append(nUtxoBytes, b) ts.Sum += newu.Value hits++ + break // only one match } - break // only one match + } } From f231113b9076b0ba4eb823ccc0eb79c80bac58df Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Sun, 31 Jan 2016 15:02:38 -0800 Subject: [PATCH 25/30] some cleanup, store spent txs pretty much everything is db based now. Still some concurrency issues when multiple block inv messages come in (which they do) where we used len(mBlockQueue) as a check for being synched up, which doesn't quite work. Find a better way to do that... --- uspv/eight333.go | 2 +- uspv/msghandler.go | 3 +- uspv/txstore.go | 9 --- uspv/utxodb.go | 149 ++++++++++++++++----------------------------- 4 files changed, 55 insertions(+), 108 deletions(-) diff --git a/uspv/eight333.go b/uspv/eight333.go index 49a8a176..0d9bb007 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -76,11 +76,11 @@ func OpenSPV(remoteNode string, hfn, tsfn string, s.localVersion = VERSION // transaction store for this SPV connection + inTs.Param = p err = inTs.OpenDB(tsfn) if err != nil { return s, err } - inTs.Param = p s.TS = inTs // copy pointer of txstore into spvcon myMsgVer, err := wire.NewMsgVersionFromConn(s.con, 0, 0) diff --git a/uspv/msghandler.go b/uspv/msghandler.go index 3e1ffef9..a6a8e0cc 100644 --- a/uspv/msghandler.go +++ b/uspv/msghandler.go @@ -114,7 +114,6 @@ func (s *SPVCon) HeaderHandler(m *wire.MsgHeaders) { if moar { s.AskForHeaders() } else { // no moar, done w/ headers, get merkleblocks - fmt.Printf("locks here...?? ") s.headerMutex.Lock() endPos, err := s.headerFile.Seek(0, os.SEEK_END) if err != nil { @@ -164,7 +163,7 @@ func (s *SPVCon) InvHandler(m *wire.MsgInv) { s.AskForTx(thing.Hash) } if thing.Type == wire.InvTypeBlock { // new block, ingest - if len(s.mBlockQueue) == 0 { + if len(s.mBlockQueue) == 0 { // this is not a good check... // don't ask directly; instead ask for header fmt.Printf("asking for headers due to inv block\n") s.AskForHeaders() diff --git a/uspv/txstore.go b/uspv/txstore.go index 8eab9947..15b6d562 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -60,15 +60,6 @@ func NewTxStore(rootkey *hdkeychain.ExtendedKey) TxStore { return txs } -// add addresses into the TxStore in memory -func (t *TxStore) AddAdr(a btcutil.Address, kidx uint32) { - var ma MyAdr - ma.PkhAdr = a - ma.KeyIdx = kidx - t.Adrs = append(t.Adrs, ma) - return -} - // add txid of interest func (t *TxStore) AddTxid(txid *wire.ShaHash, height int32) error { if txid == nil { diff --git a/uspv/utxodb.go b/uspv/utxodb.go index 34e48527..25e0bad8 100644 --- a/uspv/utxodb.go +++ b/uspv/utxodb.go @@ -26,12 +26,13 @@ var ( func (ts *TxStore) OpenDB(filename string) error { var err error + var numKeys uint32 ts.StateDB, err = bolt.Open(filename, 0644, nil) if err != nil { return err } // create buckets if they're not already there - return ts.StateDB.Update(func(btx *bolt.Tx) error { + err = ts.StateDB.Update(func(btx *bolt.Tx) error { _, err = btx.CreateBucketIfNotExists(BKTUtxos) if err != nil { return err @@ -44,12 +45,37 @@ func (ts *TxStore) OpenDB(filename string) error { if err != nil { return err } - _, err = btx.CreateBucketIfNotExists(BKTState) + sta, err := btx.CreateBucketIfNotExists(BKTState) if err != nil { return err } + + numKeysBytes := sta.Get(KEYNumKeys) + if numKeysBytes != nil { // NumKeys exists, read into uint32 + buf := bytes.NewBuffer(numKeysBytes) + err := binary.Read(buf, binary.BigEndian, &numKeys) + if err != nil { + return err + } + fmt.Printf("db says %d keys\n", numKeys) + } else { // no adrs yet, make it 1 (why...?) + numKeys = 1 + var buf bytes.Buffer + err = binary.Write(&buf, binary.BigEndian, numKeys) + if err != nil { + return err + } + err = sta.Put(KEYNumKeys, buf.Bytes()) + if err != nil { + return err + } + } return nil }) + if err != nil { + return err + } + return ts.PopulateAdrs(numKeys) } // NewAdr creates a new, never before seen address, and increments the @@ -85,7 +111,11 @@ func (ts *TxStore) NewAdr() (*btcutil.AddressPubKeyHash, error) { return nil, err } // add in to ram. - ts.AddAdr(newAdr, n) + var ma MyAdr + ma.PkhAdr = newAdr + ma.KeyIdx = n + ts.Adrs = append(ts.Adrs, ma) + return newAdr, nil } @@ -191,8 +221,11 @@ func (ts *TxStore) PopulateAdrs(lastKey uint32) error { if err != nil { return err } + var ma MyAdr + ma.PkhAdr = newAdr + ma.KeyIdx = k + ts.Adrs = append(ts.Adrs, ma) - ts.AddAdr(newAdr, k) } return nil } @@ -256,7 +289,7 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx) (uint32, error) { // get all 4 buckets duf := btx.Bucket(BKTUtxos) // sta := btx.Bucket(BKTState) - // old := btx.Bucket(BKTStxos) + old := btx.Bucket(BKTStxos) // txns := btx.Bucket(BKTTxns) // first see if we lose utxos @@ -280,11 +313,25 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx) (uint32, error) { if err != nil { return err } + // after deletion, save stxo to old bucket + var st Stxo // generate spent txo + st.Utxo = lostTxo // assign outpoint + st.SpendHeight = height // spent at height + st.SpendTxid = tx.TxSha() // spent by txid + stxb, err := st.ToBytes() // serialize + if err != nil { + return err + } + err = old.Put(k, stxb) // write k:v outpoint:stxo bytes + if err != nil { + return err + } + return nil // matched utxo k, won't match another } return nil // no match }) - } // done losing utxos + } // done losing utxos, next gain utxos // next add all new utxos to db, this is quick as the work is above for _, ub := range nUtxoBytes { err = duf.Put(ub[:36], ub[36:]) @@ -297,34 +344,6 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx) (uint32, error) { return hits, err } -// SaveToDB write a utxo to disk, overwriting an old utxo of the same outpoint -func (ts *TxStore) SaveUtxo(u *Utxo) error { - b, err := u.ToBytes() - if err != nil { - return err - } - - err = ts.StateDB.Update(func(btx *bolt.Tx) error { - duf := btx.Bucket(BKTUtxos) - sta := btx.Bucket(BKTState) - // kindof hack, height is 36:40 - // also not really tip height... - if u.AtHeight > 0 { // if confirmed - err = sta.Put(KEYTipHeight, b[36:40]) - if err != nil { - return err - } - } - - // key : val is txid:everything else - return duf.Put(b[:36], b[36:]) - }) - if err != nil { - return err - } - return nil -} - func (ts *TxStore) MarkSpent(ut Utxo, h int32, stx *wire.MsgTx) error { // we write in key = outpoint (32 hash, 4 index) // value = spending txid @@ -370,68 +389,6 @@ func (ts *TxStore) MarkSpent(ut Utxo, h int32, stx *wire.MsgTx) error { }) } -// LoadFromDB loads everything in the db file into ram, rebuilding the TxStore -// (except the rootPrivKey, that should be done before calling this -- -// this will error if ts.rootPrivKey hasn't been loaded) -func (ts *TxStore) LoadFromDB() error { - if ts.rootPrivKey == nil { - return fmt.Errorf("LoadFromDB needs rootPrivKey loaded") - } - return ts.StateDB.View(func(btx *bolt.Tx) error { - duf := btx.Bucket(BKTUtxos) - if duf == nil { - return fmt.Errorf("no duffel bag") - } - spent := btx.Bucket(BKTStxos) - if spent == nil { - return fmt.Errorf("no spenttx bucket") - } - sta := btx.Bucket(BKTState) - if sta == nil { - return fmt.Errorf("no state bucket") - } - // first populate addresses from state bucket - numKeysBytes := sta.Get(KEYNumKeys) - if numKeysBytes != nil { // NumKeys exists, read into uint32 - buf := bytes.NewBuffer(numKeysBytes) - var numKeys uint32 - err := binary.Read(buf, binary.BigEndian, &numKeys) - if err != nil { - return err - } - fmt.Printf("db says %d keys\n", numKeys) - err = ts.PopulateAdrs(numKeys) - if err != nil { - return err - } - } - // next load all utxos from db into ram - duf.ForEach(func(k, v []byte) error { - // have to copy k and v here, otherwise append will crash it. - // not quite sure why but append does weird stuff I guess. - stx := spent.Get(k) - if stx == nil { // if it's not in the spent bucket - // create a new utxo - x := make([]byte, len(k)+len(v)) - copy(x, k) - copy(x[len(k):], v) - newU, err := UtxoFromBytes(x) - if err != nil { - return err - } - // and add it to ram - ts.Utxos = append(ts.Utxos, &newU) - ts.Sum += newU.Value - } else { - fmt.Printf("had utxo %x but spent by tx %x...\n", - k, stx[:8]) - } - return nil - }) - return nil - }) -} - // outPointToBytes turns an outpoint into 36 bytes. func outPointToBytes(op *wire.OutPoint) ([]byte, error) { var buf bytes.Buffer From e56b1993fc1739dac65aa3a3576b0f0f17a81d0a Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Mon, 1 Feb 2016 19:03:01 -0800 Subject: [PATCH 26/30] move ser/des from utxodb to txstore --- uspv/txstore.go | 203 ++++++++++++++++++++++++++++++++++++++--- uspv/utxodb.go | 234 ++---------------------------------------------- 2 files changed, 201 insertions(+), 236 deletions(-) diff --git a/uspv/txstore.go b/uspv/txstore.go index 15b6d562..5937bc85 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -1,6 +1,8 @@ package uspv import ( + "bytes" + "encoding/binary" "fmt" "log" @@ -16,10 +18,8 @@ import ( type TxStore struct { OKTxids map[wire.ShaHash]int32 // known good txids and their heights - Utxos []*Utxo // stacks on stacks Sum int64 // racks on racks Adrs []MyAdr // endeavouring to acquire capital - LastIdx uint32 // should equal len(Adrs) StateDB *bolt.DB // place to write all this down // this is redundant with the SPVCon param... ugly and should be taken out Param *chaincfg.Params // network parameters (testnet3, testnetL) @@ -100,15 +100,6 @@ func (t *TxStore) GimmeFilter() (*bloom.Filter, error) { return f, nil } -// need this because before I was comparing pointers maybe? -// so they were the same outpoint but stored in 2 places so false negative? -func OutPointsEqual(a, b wire.OutPoint) bool { - if !a.Hash.IsEqual(&b.Hash) { - return false - } - return a.Index == b.Index -} - // TxToString prints out some info about a transaction. for testing / debugging func TxToString(tx *wire.MsgTx) string { str := "\t\t\t - Tx - \n" @@ -126,3 +117,193 @@ func TxToString(tx *wire.MsgTx) string { } return str } + +// need this because before I was comparing pointers maybe? +// so they were the same outpoint but stored in 2 places so false negative? +func OutPointsEqual(a, b wire.OutPoint) bool { + if !a.Hash.IsEqual(&b.Hash) { + return false + } + return a.Index == b.Index +} + +/*----- serialization for tx outputs ------- */ + +// outPointToBytes turns an outpoint into 36 bytes. +func outPointToBytes(op *wire.OutPoint) ([]byte, error) { + var buf bytes.Buffer + _, err := buf.Write(op.Hash.Bytes()) + if err != nil { + return nil, err + } + // write 4 byte outpoint index within the tx to spend + err = binary.Write(&buf, binary.BigEndian, op.Index) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// ToBytes turns a Utxo into some bytes. +// note that the txid is the first 36 bytes and in our use cases will be stripped +// off, but is left here for other applications +func (u *Utxo) ToBytes() ([]byte, error) { + var buf bytes.Buffer + // write 32 byte txid of the utxo + _, err := buf.Write(u.Op.Hash.Bytes()) + if err != nil { + return nil, err + } + // write 4 byte outpoint index within the tx to spend + err = binary.Write(&buf, binary.BigEndian, u.Op.Index) + if err != nil { + return nil, err + } + // write 4 byte height of utxo + err = binary.Write(&buf, binary.BigEndian, u.AtHeight) + if err != nil { + return nil, err + } + // write 4 byte key index of utxo + err = binary.Write(&buf, binary.BigEndian, u.KeyIdx) + if err != nil { + return nil, err + } + // write 8 byte amount of money at the utxo + err = binary.Write(&buf, binary.BigEndian, u.Value) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +// UtxoFromBytes turns bytes into a Utxo. Note it wants the txid and outindex +// in the first 36 bytes, which isn't stored that way in the boldDB, +// but can be easily appended. +func UtxoFromBytes(b []byte) (Utxo, error) { + var u Utxo + if b == nil { + return u, fmt.Errorf("nil input slice") + } + buf := bytes.NewBuffer(b) + if buf.Len() < 52 { // utxos are 52 bytes + return u, fmt.Errorf("Got %d bytes for utxo, expect 52", buf.Len()) + } + // read 32 byte txid + err := u.Op.Hash.SetBytes(buf.Next(32)) + if err != nil { + return u, err + } + // read 4 byte outpoint index within the tx to spend + err = binary.Read(buf, binary.BigEndian, &u.Op.Index) + if err != nil { + return u, err + } + // read 4 byte height of utxo + err = binary.Read(buf, binary.BigEndian, &u.AtHeight) + if err != nil { + return u, err + } + // read 4 byte key index of utxo + err = binary.Read(buf, binary.BigEndian, &u.KeyIdx) + if err != nil { + return u, err + } + // read 8 byte amount of money at the utxo + err = binary.Read(buf, binary.BigEndian, &u.Value) + if err != nil { + return u, err + } + return u, nil +} + +// ToBytes turns an Stxo into some bytes. +// outpoint txid, outpoint idx, height, key idx, amt, spendheight, spendtxid +func (s *Stxo) ToBytes() ([]byte, error) { + var buf bytes.Buffer + // write 32 byte txid of the utxo + _, err := buf.Write(s.Op.Hash.Bytes()) + if err != nil { + return nil, err + } + // write 4 byte outpoint index within the tx to spend + err = binary.Write(&buf, binary.BigEndian, s.Op.Index) + if err != nil { + return nil, err + } + // write 4 byte height of utxo + err = binary.Write(&buf, binary.BigEndian, s.AtHeight) + if err != nil { + return nil, err + } + // write 4 byte key index of utxo + err = binary.Write(&buf, binary.BigEndian, s.KeyIdx) + if err != nil { + return nil, err + } + // write 8 byte amount of money at the utxo + err = binary.Write(&buf, binary.BigEndian, s.Value) + if err != nil { + return nil, err + } + // write 4 byte height where the txo was spent + err = binary.Write(&buf, binary.BigEndian, s.SpendHeight) + if err != nil { + return nil, err + } + // write 32 byte txid of the spending transaction + _, err = buf.Write(s.SpendTxid.Bytes()) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// StxoFromBytes turns bytes into a Stxo. +func StxoFromBytes(b []byte) (Stxo, error) { + var s Stxo + if b == nil { + return s, fmt.Errorf("nil input slice") + } + buf := bytes.NewBuffer(b) + if buf.Len() < 88 { // stxos are 88 bytes + return s, fmt.Errorf("Got %d bytes for stxo, expect 88", buf.Len()) + } + // read 32 byte txid + err := s.Op.Hash.SetBytes(buf.Next(32)) + if err != nil { + return s, err + } + // read 4 byte outpoint index within the tx to spend + err = binary.Read(buf, binary.BigEndian, &s.Op.Index) + if err != nil { + return s, err + } + // read 4 byte height of utxo + err = binary.Read(buf, binary.BigEndian, &s.AtHeight) + if err != nil { + return s, err + } + // read 4 byte key index of utxo + err = binary.Read(buf, binary.BigEndian, &s.KeyIdx) + if err != nil { + return s, err + } + // read 8 byte amount of money at the utxo + err = binary.Read(buf, binary.BigEndian, &s.Value) + if err != nil { + return s, err + } + // read 4 byte spend height + err = binary.Read(buf, binary.BigEndian, &s.SpendHeight) + if err != nil { + return s, err + } + // read 32 byte txid + err = s.SpendTxid.SetBytes(buf.Next(32)) + if err != nil { + return s, err + } + return s, nil +} diff --git a/uspv/utxodb.go b/uspv/utxodb.go index 25e0bad8..7a3af432 100644 --- a/uspv/utxodb.go +++ b/uspv/utxodb.go @@ -290,7 +290,7 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx) (uint32, error) { duf := btx.Bucket(BKTUtxos) // sta := btx.Bucket(BKTState) old := btx.Bucket(BKTStxos) - // txns := btx.Bucket(BKTTxns) + txns := btx.Bucket(BKTTxns) // first see if we lose utxos // iterate through duffel bag and look for matches @@ -326,6 +326,14 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx) (uint32, error) { if err != nil { return err } + // store this relevant tx + sha := tx.TxSha() + var buf bytes.Buffer + tx.Serialize(&buf) + err = txns.Put(sha.Bytes(), buf.Bytes()) + if err != nil { + return err + } return nil // matched utxo k, won't match another } @@ -343,227 +351,3 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx) (uint32, error) { }) return hits, err } - -func (ts *TxStore) MarkSpent(ut Utxo, h int32, stx *wire.MsgTx) error { - // we write in key = outpoint (32 hash, 4 index) - // value = spending txid - // if we care about the spending tx we can store that in another bucket. - - var st Stxo - st.Utxo = ut - st.SpendHeight = h - st.SpendTxid = stx.TxSha() - - return ts.StateDB.Update(func(btx *bolt.Tx) error { - duf := btx.Bucket(BKTUtxos) - old := btx.Bucket(BKTStxos) - txns := btx.Bucket(BKTTxns) - - opb, err := outPointToBytes(&st.Op) - if err != nil { - return err - } - - err = duf.Delete(opb) // not utxo anymore - if err != nil { - return err - } - - stxb, err := st.ToBytes() - if err != nil { - return err - } - - err = old.Put(opb, stxb) // write k:v outpoint:stxo bytes - if err != nil { - return err - } - - // store spending tx - sha := stx.TxSha() - var buf bytes.Buffer - stx.Serialize(&buf) - txns.Put(sha.Bytes(), buf.Bytes()) - - return nil - }) -} - -// outPointToBytes turns an outpoint into 36 bytes. -func outPointToBytes(op *wire.OutPoint) ([]byte, error) { - var buf bytes.Buffer - _, err := buf.Write(op.Hash.Bytes()) - if err != nil { - return nil, err - } - // write 4 byte outpoint index within the tx to spend - err = binary.Write(&buf, binary.BigEndian, op.Index) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// ToBytes turns a Utxo into some bytes. -// note that the txid is the first 36 bytes and in our use cases will be stripped -// off, but is left here for other applications -func (u *Utxo) ToBytes() ([]byte, error) { - var buf bytes.Buffer - // write 32 byte txid of the utxo - _, err := buf.Write(u.Op.Hash.Bytes()) - if err != nil { - return nil, err - } - // write 4 byte outpoint index within the tx to spend - err = binary.Write(&buf, binary.BigEndian, u.Op.Index) - if err != nil { - return nil, err - } - // write 4 byte height of utxo - err = binary.Write(&buf, binary.BigEndian, u.AtHeight) - if err != nil { - return nil, err - } - // write 4 byte key index of utxo - err = binary.Write(&buf, binary.BigEndian, u.KeyIdx) - if err != nil { - return nil, err - } - // write 8 byte amount of money at the utxo - err = binary.Write(&buf, binary.BigEndian, u.Value) - if err != nil { - return nil, err - } - - return buf.Bytes(), nil -} - -// UtxoFromBytes turns bytes into a Utxo. Note it wants the txid and outindex -// in the first 36 bytes, which isn't stored that way in the boldDB, -// but can be easily appended. -func UtxoFromBytes(b []byte) (Utxo, error) { - var u Utxo - if b == nil { - return u, fmt.Errorf("nil input slice") - } - buf := bytes.NewBuffer(b) - if buf.Len() < 52 { // utxos are 52 bytes - return u, fmt.Errorf("Got %d bytes for utxo, expect 52", buf.Len()) - } - // read 32 byte txid - err := u.Op.Hash.SetBytes(buf.Next(32)) - if err != nil { - return u, err - } - // read 4 byte outpoint index within the tx to spend - err = binary.Read(buf, binary.BigEndian, &u.Op.Index) - if err != nil { - return u, err - } - // read 4 byte height of utxo - err = binary.Read(buf, binary.BigEndian, &u.AtHeight) - if err != nil { - return u, err - } - // read 4 byte key index of utxo - err = binary.Read(buf, binary.BigEndian, &u.KeyIdx) - if err != nil { - return u, err - } - // read 8 byte amount of money at the utxo - err = binary.Read(buf, binary.BigEndian, &u.Value) - if err != nil { - return u, err - } - return u, nil -} - -// ToBytes turns an Stxo into some bytes. -// outpoint txid, outpoint idx, height, key idx, amt, spendheight, spendtxid -func (s *Stxo) ToBytes() ([]byte, error) { - var buf bytes.Buffer - // write 32 byte txid of the utxo - _, err := buf.Write(s.Op.Hash.Bytes()) - if err != nil { - return nil, err - } - // write 4 byte outpoint index within the tx to spend - err = binary.Write(&buf, binary.BigEndian, s.Op.Index) - if err != nil { - return nil, err - } - // write 4 byte height of utxo - err = binary.Write(&buf, binary.BigEndian, s.AtHeight) - if err != nil { - return nil, err - } - // write 4 byte key index of utxo - err = binary.Write(&buf, binary.BigEndian, s.KeyIdx) - if err != nil { - return nil, err - } - // write 8 byte amount of money at the utxo - err = binary.Write(&buf, binary.BigEndian, s.Value) - if err != nil { - return nil, err - } - // write 4 byte height where the txo was spent - err = binary.Write(&buf, binary.BigEndian, s.SpendHeight) - if err != nil { - return nil, err - } - // write 32 byte txid of the spending transaction - _, err = buf.Write(s.SpendTxid.Bytes()) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -// StxoFromBytes turns bytes into a Stxo. -func StxoFromBytes(b []byte) (Stxo, error) { - var s Stxo - if b == nil { - return s, fmt.Errorf("nil input slice") - } - buf := bytes.NewBuffer(b) - if buf.Len() < 88 { // stxos are 88 bytes - return s, fmt.Errorf("Got %d bytes for stxo, expect 88", buf.Len()) - } - // read 32 byte txid - err := s.Op.Hash.SetBytes(buf.Next(32)) - if err != nil { - return s, err - } - // read 4 byte outpoint index within the tx to spend - err = binary.Read(buf, binary.BigEndian, &s.Op.Index) - if err != nil { - return s, err - } - // read 4 byte height of utxo - err = binary.Read(buf, binary.BigEndian, &s.AtHeight) - if err != nil { - return s, err - } - // read 4 byte key index of utxo - err = binary.Read(buf, binary.BigEndian, &s.KeyIdx) - if err != nil { - return s, err - } - // read 8 byte amount of money at the utxo - err = binary.Read(buf, binary.BigEndian, &s.Value) - if err != nil { - return s, err - } - // read 4 byte spend height - err = binary.Read(buf, binary.BigEndian, &s.SpendHeight) - if err != nil { - return s, err - } - // read 32 byte txid - err = s.SpendTxid.SetBytes(buf.Next(32)) - if err != nil { - return s, err - } - return s, nil -} From 4513d772166f4d83679bb97c41e557c8e8e4a78f Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Tue, 2 Feb 2016 02:19:26 -0800 Subject: [PATCH 27/30] add readme documentation --- uspv/README.md | 59 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 uspv/README.md diff --git a/uspv/README.md b/uspv/README.md new file mode 100644 index 00000000..44ff72d9 --- /dev/null +++ b/uspv/README.md @@ -0,0 +1,59 @@ +# uspv - micro-SPV library + +The uspv library implements simplified SPV wallet functionality. +It connects to full nodes using the standard port 8333 bitcoin protocol, +gets headers, uses bloom filters, gets blocks and transactions, and has +functions to send and receive coins. + +## Files + +Three files are used by the library: + +#### Key file (currently testkey.hex) + +This file contains the secret seed which creates all private keys used by the wallet. It is stored in ascii hexadecimal for easy copying and pasting. If you don't enter a password when prompted, you'll get a warning and the key file will be saved in the clear with no encryption. You shouldn't do that though. When using a password, the key file will be longer and use the scrypt KDF and nacl/secretbox to protect the secret seed. + +#### Header file (currently headers.bin) + +This is a file storing all the block headers. Headers are 80 bytes long, so this file's size will always be an even multiple of 80. All blockchain-technology verifications are performed when appending headers to the file. In the case of re-orgs, since it's so quick to get headers, it just truncates a bit and tries again. + +#### Database file (currently utxo.db) + +This file more complex. It uses bolt DB to store wallet information needed to send and receive bitcoins. The database file is organized into 4 main "buckets": + +* Utxos ("DuffelBag") + +This bucket stores all the utxos. The goal of bitcoin is to get lots of utxos, earning a high score. + +* Stxos ("SpentTxs") + +For record keeping, this bucket stores what used to be utxos, but are no longer "u"txos, and are spent outpoints. It references the spending txid. + +* Txns ("Txns") + +This bucket stores full serialized transactions which are refenced in the Stxos bucket. These can be used to re-play transactions in the case of re-orgs. + +* State ("MiscState") + +This has describes some miscellaneous global state variables of the database, such as what height it has synchronized up to, and how many addresses have been created. (Currently those are the only 2 things stored) + +## Synchronization overview + +Currently uspv only connects to one hard-coded node, as address messages and storage are not yet implemented. It first asks for headers, providing the last known header (writing the genesis header if needed). It loops through asking for headers until it receives an empty header message, which signals that headers are fully synchronized. + +After header synchronization is complete, it requests merkle blocks starting at the keyfile birthday. (This is currently hard-coded; add new db key?) Bloom filters are generated for the addresses and utxos known to the wallet. If too many false positives are received, a new filter is generated and sent. (This happens fairly often because the filter exponentially saturates with false positives when using BloomUpdateAll.) Once the merkle blocks have been received up to the header height, the wallet is considered synchronized and it will listen for new inv messages from the remote node. An inv message describing a block will trigger a request for headers, starting the same synchronization process of headers then merkle-blocks. + +## TODO + +There's still quite a bit left, though most of it hopefully won't be too hard. + +Problems / still to do: + +* Only connects to one node, and that node is hard-coded. +* Currently doesn't check signatures of received transactions. +* Re-orgs affect only headers, and don't evict confirmed transactions. +* Double spends are not detected; Double spent txs will stay at height 0. +* Tx creation and signing is still very rudimentary. +* There may be wire-protocol irregularities which can get it kicked off. + +Hopefully I can get most of that list deleted soon. From a9cf239ec3b4f2124f109b7f327ad0c000900844 Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Tue, 2 Feb 2016 17:14:13 -0800 Subject: [PATCH 28/30] fix readme, better state machine for sync --- uspv/README.md | 13 ++++++++++++- uspv/eight333.go | 12 ++++++++++-- uspv/msghandler.go | 12 +++++++----- uspv/utxodb.go | 14 +++++++++++++- 4 files changed, 42 insertions(+), 9 deletions(-) diff --git a/uspv/README.md b/uspv/README.md index 44ff72d9..08cade5e 100644 --- a/uspv/README.md +++ b/uspv/README.md @@ -50,10 +50,21 @@ There's still quite a bit left, though most of it hopefully won't be too hard. Problems / still to do: * Only connects to one node, and that node is hard-coded. -* Currently doesn't check signatures of received transactions. * Re-orgs affect only headers, and don't evict confirmed transactions. * Double spends are not detected; Double spent txs will stay at height 0. * Tx creation and signing is still very rudimentary. * There may be wire-protocol irregularities which can get it kicked off. Hopefully I can get most of that list deleted soon. + +(Now sanity checks txs, but can't check sigs... because it's SPV. Right.) + +Later functionality to implement: + +* "Desktop Mode" SPV, or "Unfiltered" SPV or some other name + +This would be a mode where uspv doesn't use bloom filters and request merkle blocks, but instead grabs everything in the block and discards most of the data. This prevents nodes from learning about your utxo set. To further enhance this, it should connect to multiple nodes and relay txs and inv messages to blend in. + +* Ironman SPV + +Never request txs. Only merkleBlocks (or in above mode, blocks). No unconfirmed transactions are presented to the user, which makes a whole lot of sense as with unconfirmed SPV transactions you're relying completely on the honesty of the reporting node. \ No newline at end of file diff --git a/uspv/eight333.go b/uspv/eight333.go index 0d9bb007..eb2bbd85 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -50,6 +50,10 @@ type SPVCon struct { mBlockQueue chan HashAndHeight // fPositives is a channel to keep track of bloom filter false positives. fPositives chan int32 + + // waitState is a channel that is empty while in the header and block + // sync modes, but when in the idle state has a "true" in it. + inWaitState chan bool } func OpenSPV(remoteNode string, hfn, tsfn string, @@ -134,6 +138,7 @@ func OpenSPV(remoteNode string, hfn, tsfn string, go s.outgoingMessageHandler() s.mBlockQueue = make(chan HashAndHeight, 32) // queue depth 32 is a thing s.fPositives = make(chan int32, 4000) // a block full, approx + s.inWaitState = make(chan bool) go s.fPositiveHandler() return s, nil @@ -317,11 +322,16 @@ func (s *SPVCon) IngestMerkleBlock(m *wire.MsgMerkleBlock) error { return fmt.Errorf("Txid store error: %s\n", err.Error()) } } + err = s.TS.SetDBSyncHeight(hah.height) if err != nil { return err } + // not super comfortable with this but it seems to work. + if len(s.mBlockQueue) == 0 { // done and fully sync'd + s.inWaitState <- true + } return nil } @@ -385,7 +395,6 @@ func (s *SPVCon) IngestHeaders(m *wire.MsgHeaders) (bool, error) { if err != nil { return false, err } - // advance chain tip tip++ // check last header @@ -408,7 +417,6 @@ func (s *SPVCon) IngestHeaders(m *wire.MsgHeaders) (bool, error) { } } log.Printf("Headers to height %d OK.", tip) - return true, nil } diff --git a/uspv/msghandler.go b/uspv/msghandler.go index a6a8e0cc..8c6ecfbe 100644 --- a/uspv/msghandler.go +++ b/uspv/msghandler.go @@ -45,7 +45,7 @@ func (s *SPVCon) incomingMessageHandler() { log.Printf("Rejected! cmd: %s code: %s tx: %s reason: %s", m.Cmd, m.Code.String(), m.Hash.String(), m.Reason) case *wire.MsgInv: - go s.InvHandler(m) + s.InvHandler(m) case *wire.MsgNotFound: log.Printf("Got not found response from remote:") for i, thing := range m.InvList { @@ -162,12 +162,14 @@ func (s *SPVCon) InvHandler(m *wire.MsgInv) { s.TS.OKTxids[thing.Hash] = 0 // unconfirmed s.AskForTx(thing.Hash) } - if thing.Type == wire.InvTypeBlock { // new block, ingest - if len(s.mBlockQueue) == 0 { // this is not a good check... - // don't ask directly; instead ask for header + if thing.Type == wire.InvTypeBlock { // new block what to do? + select { + case <-s.inWaitState: + // start getting headers fmt.Printf("asking for headers due to inv block\n") s.AskForHeaders() - } else { + default: + // drop it as if its component particles had high thermal energies fmt.Printf("inv block but ignoring, not synched\n") } } diff --git a/uspv/utxodb.go b/uspv/utxodb.go index 7a3af432..07f3ee82 100644 --- a/uspv/utxodb.go +++ b/uspv/utxodb.go @@ -5,6 +5,8 @@ import ( "encoding/binary" "fmt" + "github.com/btcsuite/btcd/blockchain" + "github.com/btcsuite/btcd/txscript" "github.com/btcsuite/btcd/wire" @@ -238,7 +240,7 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx) (uint32, error) { var spentOPs [][]byte var nUtxoBytes [][]byte - // check that we have a height and tx has been SPV OK'd + // first check that we have a height and tx has been SPV OK'd inTxid := tx.TxSha() height, ok := ts.OKTxids[inTxid] if !ok { @@ -246,6 +248,16 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx) (uint32, error) { inTxid.String()) } + // tx has been OK'd by SPV; check tx sanity + utilTx := btcutil.NewTx(tx) // convert for validation + // checks stuff like inputs >= ouputs + err = blockchain.CheckTransactionSanity(utilTx) + if err != nil { + return hits, err + } + // note that you can't check signatures; this is SPV. + // 0 conf SPV means pretty much nothing. Anyone can say anything. + // before entering into db, serialize all inputs of the ingested tx for _, txin := range tx.TxIn { nOP, err := outPointToBytes(&txin.PreviousOutPoint) From 2f9fc87636f705ef4319d01a92f542d5317c7d0d Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Tue, 2 Feb 2016 19:04:03 -0800 Subject: [PATCH 29/30] fix sync, remove sum from TxStore Synchronization now seems to work well even with the rapid fire many inv-block messages that I'm seeing often on testnet3. I'm not 100% sure measuring the len() of a buffered channel is safe but it seems to work fine. Got rid of 'sum' in the TxStore; can be computed from GetAllUtxos() Might want to merge SCon and TxStore since there's not much going on in TxStore any more... --- uspv/eight333.go | 19 +++++++++++++------ uspv/msghandler.go | 11 +++++++---- uspv/txstore.go | 1 - uspv/utxodb.go | 2 -- 4 files changed, 20 insertions(+), 13 deletions(-) diff --git a/uspv/eight333.go b/uspv/eight333.go index eb2bbd85..26dd0a96 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -138,7 +138,7 @@ func OpenSPV(remoteNode string, hfn, tsfn string, go s.outgoingMessageHandler() s.mBlockQueue = make(chan HashAndHeight, 32) // queue depth 32 is a thing s.fPositives = make(chan int32, 4000) // a block full, approx - s.inWaitState = make(chan bool) + s.inWaitState = make(chan bool, 1) go s.fPositiveHandler() return s, nil @@ -308,7 +308,18 @@ func (s *SPVCon) IngestMerkleBlock(m *wire.MsgMerkleBlock) error { if err != nil { return err } - hah := <-s.mBlockQueue // pop height off mblock queue + var hah HashAndHeight + select { + case hah = <-s.mBlockQueue: // pop height off mblock queue + // not super comfortable with this but it seems to work. + if len(s.mBlockQueue) == 0 { // done and fully sync'd + s.inWaitState <- true + } + break + default: + return fmt.Errorf("Unrequested merkle block") + } + // this verifies order, and also that the returned header fits // into our SPV header file newMerkBlockSha := m.Header.BlockSha() @@ -328,10 +339,6 @@ func (s *SPVCon) IngestMerkleBlock(m *wire.MsgMerkleBlock) error { return err } - // not super comfortable with this but it seems to work. - if len(s.mBlockQueue) == 0 { // done and fully sync'd - s.inWaitState <- true - } return nil } diff --git a/uspv/msghandler.go b/uspv/msghandler.go index 8c6ecfbe..5b648e0a 100644 --- a/uspv/msghandler.go +++ b/uspv/msghandler.go @@ -148,8 +148,8 @@ func (s *SPVCon) TxHandler(m *wire.MsgTx) { m.TxSha().String()) s.fPositives <- 1 // add one false positive to chan } else { - log.Printf("tx %s ingested and matches utxo/adrs. sum %d", - m.TxSha().String(), s.TS.Sum) + log.Printf("tx %s ingested and matches %d utxo/adrs.", + m.TxSha().String(), hits) } } @@ -167,10 +167,13 @@ func (s *SPVCon) InvHandler(m *wire.MsgInv) { case <-s.inWaitState: // start getting headers fmt.Printf("asking for headers due to inv block\n") - s.AskForHeaders() + err := s.AskForHeaders() + if err != nil { + log.Printf("AskForHeaders error: %s", err.Error()) + } default: // drop it as if its component particles had high thermal energies - fmt.Printf("inv block but ignoring, not synched\n") + fmt.Printf("inv block but ignoring; not synched\n") } } } diff --git a/uspv/txstore.go b/uspv/txstore.go index 5937bc85..505586ff 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -18,7 +18,6 @@ import ( type TxStore struct { OKTxids map[wire.ShaHash]int32 // known good txids and their heights - Sum int64 // racks on racks Adrs []MyAdr // endeavouring to acquire capital StateDB *bolt.DB // place to write all this down // this is redundant with the SPVCon param... ugly and should be taken out diff --git a/uspv/utxodb.go b/uspv/utxodb.go index 07f3ee82..28475470 100644 --- a/uspv/utxodb.go +++ b/uspv/utxodb.go @@ -289,7 +289,6 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx) (uint32, error) { return hits, err } nUtxoBytes = append(nUtxoBytes, b) - ts.Sum += newu.Value hits++ break // only one match } @@ -318,7 +317,6 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx) (uint32, error) { if err != nil { return err } - ts.Sum -= lostTxo.Value hits++ // then delete the utxo from duf, save to old err = duf.Delete(k) From 624e776987e3b8c199b74013f22e3e1e32bcb9bb Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Tue, 2 Feb 2016 23:37:29 -0800 Subject: [PATCH 30/30] add shell spv testing mode To try uspv, do ./lnd -spv The remote node is hardcoded in shell.go. If you aren't running a full node on localhost, specify where to connect to. Nearby nodes will be much faster but random testnet nodes on the internet should also work. --- .gitignore | 4 + cmd/lnshell/commands.go | 4 +- lnd.go | 6 + shell.go | 328 ++++++++++++++++++++++++++++++++++++++++ uspv/eight333.go | 26 ++-- uspv/msghandler.go | 4 +- uspv/txstore.go | 6 +- 7 files changed, 358 insertions(+), 20 deletions(-) create mode 100644 shell.go diff --git a/.gitignore b/.gitignore index feb149c7..b55095c7 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,7 @@ test_wal/* # vim **.swp + +*.hex +*.db +*.bin diff --git a/cmd/lnshell/commands.go b/cmd/lnshell/commands.go index a883c9be..dc4e1506 100644 --- a/cmd/lnshell/commands.go +++ b/cmd/lnshell/commands.go @@ -22,8 +22,8 @@ func RpcConnect(args []string) error { } fmt.Printf("connection state: %s\n", state.String()) time.Sleep(time.Second * 2) - // lnClient := lnrpc.NewLightningClient(conn) - // lnClient.NewAddress(nil, nil, nil) // crashes + // lnClient := lnrpc.NewLightningClient(conn) + // lnClient.NewAddress(nil, nil, nil) // crashes state, err = conn.State() if err != nil { diff --git a/lnd.go b/lnd.go index afcc516f..5751379b 100644 --- a/lnd.go +++ b/lnd.go @@ -21,11 +21,17 @@ var ( rpcPort = flag.Int("rpcport", 10009, "The port for the rpc server") peerPort = flag.String("peerport", "10011", "The port to listen on for incoming p2p connections") dataDir = flag.String("datadir", "test_wal", "The directory to store lnd's data within") + spvMode = flag.Bool("spv", false, "assert to enter spv wallet mode") ) func main() { flag.Parse() + if *spvMode == true { + shell() + return + } + go func() { listenAddr := net.JoinHostPort("", "5009") profileRedirect := http.RedirectHandler("/debug/pprof", diff --git a/shell.go b/shell.go new file mode 100644 index 00000000..5602a18c --- /dev/null +++ b/shell.go @@ -0,0 +1,328 @@ +package main + +import ( + "bufio" + "bytes" + "fmt" + "log" + "os" + "strconv" + "strings" + + "github.com/btcsuite/btcd/txscript" + "github.com/btcsuite/btcd/wire" + + "github.com/btcsuite/btcd/chaincfg" + "github.com/btcsuite/btcutil" + + "github.com/lightningnetwork/lnd/uspv" +) + +/* this is a CLI shell for testing out LND. Right now it's only for uspv +testing. It can send and receive coins. +*/ + +const ( + keyFileName = "testkey.hex" + headerFileName = "headers.bin" + dbFileName = "utxo.db" + // this is my local testnet node, replace it with your own close by. + // Random internet testnet nodes usually work but sometimes don't, so + // maybe I should test against different versions out there. + SPVHostAdr = "127.0.0.1:18333" +) + +var ( + Params = &chaincfg.TestNet3Params + SCon uspv.SPVCon // global here for now +) + +func shell() { + fmt.Printf("LND spv shell v0.0\n") + fmt.Printf("Not yet well integrated, but soon.\n") + + // read key file (generate if not found) + rootPriv, err := uspv.ReadKeyFileToECPriv(keyFileName, Params) + if err != nil { + log.Fatal(err) + } + // setup TxStore first (before spvcon) + Store := uspv.NewTxStore(rootPriv, Params) + // setup spvCon + + SCon, err = uspv.OpenSPV( + SPVHostAdr, headerFileName, dbFileName, &Store, Params) + if err != nil { + log.Fatal(err) + } + + tip, err := SCon.TS.GetDBSyncHeight() // ask for sync height + if err != nil { + log.Fatal(err) + } + if tip == 0 { // DB has never been used, set to birthday + tip = 675000 // hardcoded; later base on keyfile date? + err = SCon.TS.SetDBSyncHeight(tip) + if err != nil { + log.Fatal(err) + } + } + + // once we're connected, initiate headers sync + err = Hdr() + if err != nil { + log.Fatal(err) + } + + // main shell loop + for { + // setup reader with max 4K input chars + reader := bufio.NewReaderSize(os.Stdin, 4000) + fmt.Printf("LND# ") // prompt + msg, err := reader.ReadString('\n') // input finishes on enter key + if err != nil { + log.Fatal(err) + } + + cmdslice := strings.Fields(msg) // chop input up on whitespace + if len(cmdslice) < 1 { + continue // no input, just prompt again + } + fmt.Printf("entered command: %s\n", msg) // immediate feedback + err = Shellparse(cmdslice) + if err != nil { // only error should be user exit + log.Fatal(err) + } + } + return +} + +// Shellparse parses user input and hands it to command functions if matching +func Shellparse(cmdslice []string) error { + var err error + var args []string + cmd := cmdslice[0] + if len(cmdslice) > 1 { + args = cmdslice[1:] + } + if cmd == "exit" || cmd == "quit" { + return fmt.Errorf("User exit") + } + + // help gives you really terse help. Just a list of commands. + if cmd == "help" { + err = Help(args) + if err != nil { + fmt.Printf("help error: %s\n", err) + } + return nil + } + + // adr generates a new address and displays it + if cmd == "adr" { + err = Adr(args) + if err != nil { + fmt.Printf("adr error: %s\n", err) + } + return nil + } + + // bal shows the current set of utxos, addresses and score + if cmd == "bal" { + err = Bal(args) + if err != nil { + fmt.Printf("bal error: %s\n", err) + } + return nil + } + + // send sends coins to the address specified + if cmd == "send" { + err = Send(args) + if err != nil { + fmt.Printf("send error: %s\n", err) + } + return nil + } + + fmt.Printf("Command not recognized. type help for command list.\n") + return nil +} + +// Hdr asks for headers. +func Hdr() error { + if SCon.RBytes == 0 { + return fmt.Errorf("No SPV connection, can't get headers.") + } + err := SCon.AskForHeaders() + if err != nil { + return err + } + return nil +} + +// Bal prints out your score. +func Bal(args []string) error { + if SCon.TS == nil { + return fmt.Errorf("Can't get balance, spv connection broken") + } + fmt.Printf(" ----- Account Balance ----- \n") + allUtxos, err := SCon.TS.GetAllUtxos() + if err != nil { + return err + } + var score int64 + for i, u := range allUtxos { + fmt.Printf("\tutxo %d height %d %s key: %d amt %d\n", + i, u.AtHeight, u.Op.String(), u.KeyIdx, u.Value) + score += u.Value + } + height, _ := SCon.TS.GetDBSyncHeight() + + for i, a := range SCon.TS.Adrs { + fmt.Printf("address %d %s\n", i, a.PkhAdr.String()) + } + fmt.Printf("Total known utxos: %d\n", len(allUtxos)) + fmt.Printf("Total spendable coin: %d\n", score) + fmt.Printf("DB sync height: %d\n", height) + return nil +} + +// Adr makes a new address. +func Adr(args []string) error { + a, err := SCon.TS.NewAdr() + if err != nil { + return err + } + fmt.Printf("made new address %s, %d addresses total\n", + a.String(), len(SCon.TS.Adrs)) + + return nil +} + +// Send sends coins. +func Send(args []string) error { + // get all utxos from the database + allUtxos, err := SCon.TS.GetAllUtxos() + if err != nil { + return err + } + var score int64 // score is the sum of all utxo amounts. highest score wins. + // add all the utxos up to get the score + for _, u := range allUtxos { + score += u.Value + } + + // score is 0, cannot unlock 'send coins' acheivement + if score == 0 { + return fmt.Errorf("You don't have money. Work hard.") + } + // need args, fail + if len(args) < 2 { + return fmt.Errorf("need args: ssend amount(satoshis) address") + } + amt, err := strconv.ParseInt(args[0], 10, 64) + if err != nil { + return err + } + if amt < 1000 { + return fmt.Errorf("can't send %d, too small", amt) + } + adr, err := btcutil.DecodeAddress(args[1], SCon.TS.Param) + if err != nil { + fmt.Printf("error parsing %s as address\t", args[1]) + return err + } + fmt.Printf("send %d to address: %s \n", + amt, adr.String()) + err = SendCoins(SCon, adr, amt) + if err != nil { + return err + } + return nil +} + +// SendCoins does send coins, but it's very rudimentary +func SendCoins(s uspv.SPVCon, adr btcutil.Address, sendAmt int64) error { + var err error + var score int64 + allUtxos, err := s.TS.GetAllUtxos() + if err != nil { + return err + } + + for _, utxo := range allUtxos { + score += utxo.Value + } + // important rule in bitcoin, output total > input total is invalid. + if sendAmt > score { + return fmt.Errorf("trying to send %d but %d available.", + sendAmt, score) + } + + tx := wire.NewMsgTx() // make new tx + // make address script 76a914...88ac + adrScript, err := txscript.PayToAddrScript(adr) + if err != nil { + return err + } + // make user specified txout and add to tx + txout := wire.NewTxOut(sendAmt, adrScript) + tx.AddTxOut(txout) + + nokori := sendAmt // nokori is how much is needed on input side + for _, utxo := range allUtxos { + // generate pkscript to sign + prevPKscript, err := txscript.PayToAddrScript( + s.TS.Adrs[utxo.KeyIdx].PkhAdr) + if err != nil { + return err + } + // make new input from this utxo + thisInput := wire.NewTxIn(&utxo.Op, prevPKscript) + tx.AddTxIn(thisInput) + nokori -= utxo.Value + if nokori < -10000 { // minimum overage / fee is 1K now + break + } + } + // there's enough left to make a change output + if nokori < -200000 { + change, err := s.TS.NewAdr() + if err != nil { + return err + } + + changeScript, err := txscript.PayToAddrScript(change) + if err != nil { + return err + } + changeOut := wire.NewTxOut((-100000)-nokori, changeScript) + tx.AddTxOut(changeOut) + } + + // use txstore method to sign + err = s.TS.SignThis(tx) + if err != nil { + return err + } + + fmt.Printf("tx: %s", uspv.TxToString(tx)) + buf := bytes.NewBuffer(make([]byte, 0, tx.SerializeSize())) + tx.Serialize(buf) + fmt.Printf("tx: %x\n", buf.Bytes()) + + // send it out on the wire. hope it gets there. + // we should deal with rejects. Don't yet. + err = s.PushTx(tx) + if err != nil { + return err + } + return nil +} + +func Help(args []string) error { + fmt.Printf("commands:\n") + fmt.Printf("help adr bal send exit\n") + return nil +} diff --git a/uspv/eight333.go b/uspv/eight333.go index 26dd0a96..a757c4ab 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -43,8 +43,7 @@ type SPVCon struct { WBytes uint64 // total bytes written RBytes uint64 // total bytes read - TS *TxStore // transaction store to write to - param *chaincfg.Params // network parameters (testnet3, testnetL) + TS *TxStore // transaction store to write to // mBlockQueue is for keeping track of what height we've requested. mBlockQueue chan HashAndHeight @@ -56,16 +55,17 @@ type SPVCon struct { inWaitState chan bool } -func OpenSPV(remoteNode string, hfn, tsfn string, +func OpenSPV(remoteNode string, hfn, dbfn string, inTs *TxStore, p *chaincfg.Params) (SPVCon, error) { // create new SPVCon var s SPVCon - // assign network parameters to SPVCon - s.param = p + // I should really merge SPVCon and TxStore, they're basically the same + inTs.Param = p + s.TS = inTs // copy pointer of txstore into spvcon // open header file - err := s.openHeaderFile(headerFileName) + err := s.openHeaderFile(hfn) if err != nil { return s, err } @@ -80,12 +80,10 @@ func OpenSPV(remoteNode string, hfn, tsfn string, s.localVersion = VERSION // transaction store for this SPV connection - inTs.Param = p - err = inTs.OpenDB(tsfn) + err = inTs.OpenDB(dbfn) if err != nil { return s, err } - s.TS = inTs // copy pointer of txstore into spvcon myMsgVer, err := wire.NewMsgVersionFromConn(s.con, 0, 0) if err != nil { @@ -99,7 +97,7 @@ func OpenSPV(remoteNode string, hfn, tsfn string, myMsgVer.AddService(wire.SFNodeBloom) // this actually sends - n, err := wire.WriteMessageN(s.con, myMsgVer, s.localVersion, s.param.Net) + n, err := wire.WriteMessageN(s.con, myMsgVer, s.localVersion, s.TS.Param.Net) if err != nil { return s, err } @@ -107,7 +105,7 @@ func OpenSPV(remoteNode string, hfn, tsfn string, log.Printf("wrote %d byte version message to %s\n", n, s.con.RemoteAddr().String()) - n, m, b, err := wire.ReadMessageN(s.con, s.localVersion, s.param.Net) + n, m, b, err := wire.ReadMessageN(s.con, s.localVersion, s.TS.Param.Net) if err != nil { return s, err } @@ -126,7 +124,7 @@ func OpenSPV(remoteNode string, hfn, tsfn string, s.remoteHeight = mv.LastBlock mva := wire.NewMsgVerAck() - n, err = wire.WriteMessageN(s.con, mva, s.localVersion, s.param.Net) + n, err = wire.WriteMessageN(s.con, mva, s.localVersion, s.TS.Param.Net) if err != nil { return s, err } @@ -149,7 +147,7 @@ func (s *SPVCon) openHeaderFile(hfn string) error { if err != nil { if os.IsNotExist(err) { var b bytes.Buffer - err = s.param.GenesisBlock.Header.Serialize(&b) + err = s.TS.Param.GenesisBlock.Header.Serialize(&b) if err != nil { return err } @@ -405,7 +403,7 @@ func (s *SPVCon) IngestHeaders(m *wire.MsgHeaders) (bool, error) { // advance chain tip tip++ // check last header - worked := CheckHeader(s.headerFile, tip, s.param) + worked := CheckHeader(s.headerFile, tip, s.TS.Param) if !worked { if endPos < 8080 { // jeez I give up, back to genesis diff --git a/uspv/msghandler.go b/uspv/msghandler.go index 5b648e0a..926ae5da 100644 --- a/uspv/msghandler.go +++ b/uspv/msghandler.go @@ -10,7 +10,7 @@ import ( func (s *SPVCon) incomingMessageHandler() { for { - n, xm, _, err := wire.ReadMessageN(s.con, s.localVersion, s.param.Net) + n, xm, _, err := wire.ReadMessageN(s.con, s.localVersion, s.TS.Param.Net) if err != nil { log.Printf("ReadMessageN error. Disconnecting: %s\n", err.Error()) return @@ -64,7 +64,7 @@ func (s *SPVCon) incomingMessageHandler() { func (s *SPVCon) outgoingMessageHandler() { for { msg := <-s.outMsgQueue - n, err := wire.WriteMessageN(s.con, msg, s.localVersion, s.param.Net) + n, err := wire.WriteMessageN(s.con, msg, s.localVersion, s.TS.Param.Net) if err != nil { log.Printf("Write message error: %s", err.Error()) } diff --git a/uspv/txstore.go b/uspv/txstore.go index 505586ff..a5062a3b 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -20,7 +20,8 @@ type TxStore struct { Adrs []MyAdr // endeavouring to acquire capital StateDB *bolt.DB // place to write all this down - // this is redundant with the SPVCon param... ugly and should be taken out + + // Params live here, not SCon Param *chaincfg.Params // network parameters (testnet3, testnetL) // From here, comes everything. It's a secret to everybody. @@ -52,9 +53,10 @@ type MyAdr struct { // an address I have the private key for // inside the Adrs slice, right? leave for now } -func NewTxStore(rootkey *hdkeychain.ExtendedKey) TxStore { +func NewTxStore(rootkey *hdkeychain.ExtendedKey, p *chaincfg.Params) TxStore { var txs TxStore txs.rootPrivKey = rootkey + txs.Param = p txs.OKTxids = make(map[wire.ShaHash]int32) return txs }