From fbd17846d41e0878cfdf9ec15bc7b412aa2e1d42 Mon Sep 17 00:00:00 2001 From: Tadge Dryja Date: Fri, 19 Feb 2016 00:32:22 -0800 Subject: [PATCH] split adrs up into two slices? here is code to keep track of two sets of addresses -- the regular ones and the witness ones. But maybe those should be merged becuase it's easy for other wallets to switch between them and send to the same 20 byte pubkey hash but wintessified. So maybe I'll change this stuff... --- shell.go | 124 +++++++--------------------------- uspv/eight333.go | 2 +- uspv/sortsignsend.go | 155 ++++++++++++++++++++++++++++++++++++++++--- uspv/txstore.go | 104 ++++++++++++++--------------- uspv/utxodb.go | 93 ++++++++++++++++++++------ 5 files changed, 293 insertions(+), 185 deletions(-) diff --git a/shell.go b/shell.go index c820c740..bcc84647 100644 --- a/shell.go +++ b/shell.go @@ -2,16 +2,12 @@ package main import ( "bufio" - "bytes" "fmt" "log" "os" "strconv" "strings" - "github.com/btcsuite/btcd/txscript" - "github.com/btcsuite/btcd/wire" - "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcutil" @@ -205,8 +201,12 @@ func Bal(args []string) error { } var score int64 for i, u := range allUtxos { - fmt.Printf("\tutxo %d height %d %s key: %d amt %d\n", + fmt.Printf("\tutxo %d height %d %s key:%d amt %d", i, u.AtHeight, u.Op.String(), u.KeyIdx, u.Value) + if u.IsWit { + fmt.Printf(" WIT") + } + fmt.Printf("\n") score += u.Value } height, _ := SCon.TS.GetDBSyncHeight() @@ -214,6 +214,9 @@ func Bal(args []string) error { for i, a := range SCon.TS.Adrs { fmt.Printf("address %d %s\n", i, a.PkhAdr.String()) } + for i, a := range SCon.TS.WitAdrs { + fmt.Printf("address %d %s [WIT]\n", i, a.PkhAdr.String()) + } fmt.Printf("Total known utxos: %d\n", len(allUtxos)) fmt.Printf("Total spendable coin: %d\n", score) fmt.Printf("DB sync height: %d\n", height) @@ -222,12 +225,24 @@ func Bal(args []string) error { // Adr makes a new address. func Adr(args []string) error { - a, err := SCon.TS.NewAdr() + + // if there's an arg, make 10 regular adrs + if len(args) > 0 { + for i := 0; i < 10; i++ { + _, err := SCon.TS.NewAdr(false) + if err != nil { + return err + } + } + } + + // always make one segwit + a, err := SCon.TS.NewAdr(true) if err != nil { return err } - fmt.Printf("made new address %s, %d addresses total\n", - a.String(), len(SCon.TS.Adrs)) + fmt.Printf("made new address %s\n", + a.String()) return nil } @@ -275,98 +290,7 @@ func Send(args []string) error { fmt.Printf("send %d to address: %s \n", amt, adr.String()) - err = SendCoins(SCon, adr, amt, wit) - if err != nil { - return err - } - return nil -} - -// SendCoins does send coins, but it's very rudimentary -// wit makes it into p2wpkh. Which is not yet spendable. -func SendCoins( - s uspv.SPVCon, adr btcutil.Address, sendAmt int64, wit bool) error { - - var err error - var score int64 - allUtxos, err := s.TS.GetAllUtxos() - if err != nil { - return err - } - - for _, utxo := range allUtxos { - score += utxo.Value - } - // important rule in bitcoin, output total > input total is invalid. - if sendAmt > score { - return fmt.Errorf("trying to send %d but %d available.", - sendAmt, score) - } - - tx := wire.NewMsgTx() // make new tx - // make address script 76a914...88ac - var adrScript []byte - if wit { - adrScript, err = uspv.P2wpkhScript(adr) - if err != nil { - return err - } - } else { // non-wit, use old p2pkh - adrScript, err = txscript.PayToAddrScript(adr) - if err != nil { - return err - } - } - - // make user specified txout and add to tx - txout := wire.NewTxOut(sendAmt, adrScript) - tx.AddTxOut(txout) - - nokori := sendAmt // nokori is how much is needed on input side - for _, utxo := range allUtxos { - // generate pkscript to sign - prevPKscript, err := txscript.PayToAddrScript( - s.TS.Adrs[utxo.KeyIdx].PkhAdr) - if err != nil { - return err - } - // make new input from this utxo - thisInput := wire.NewTxIn(&utxo.Op, prevPKscript, nil) - tx.AddTxIn(thisInput) - nokori -= utxo.Value - if nokori < -10000 { // minimum overage / fee is 1K now - break - } - } - // there's enough left to make a change output - if nokori < -200000 { - change, err := s.TS.NewAdr() - if err != nil { - return err - } - - changeScript, err := txscript.PayToAddrScript(change) - if err != nil { - return err - } - changeOut := wire.NewTxOut((-100000)-nokori, changeScript) - tx.AddTxOut(changeOut) - } - - // use txstore method to sign - err = s.TS.SignThis(tx) - if err != nil { - return err - } - - fmt.Printf("tx: %s", uspv.TxToString(tx)) - buf := bytes.NewBuffer(make([]byte, 0, tx.SerializeSize())) - tx.Serialize(buf) - fmt.Printf("tx: %x\n", buf.Bytes()) - - // send it out on the wire. hope it gets there. - // we should deal with rejects. Don't yet. - err = s.NewOutgoingTx(tx) + err = SCon.SendCoins(adr, amt, wit) if err != nil { return err } diff --git a/uspv/eight333.go b/uspv/eight333.go index ca88e18f..18356226 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -329,7 +329,7 @@ func (s *SPVCon) AskForBlocks() error { s.headerMutex.Unlock() // checked, unlock endPos := stat.Size() - headerTip := int32(endPos/80) - 1 // move back 1 header length to read + headerTip := int32(endPos / 80) // DONT move back 1 header length to read dbTip, err := s.TS.GetDBSyncHeight() if err != nil { diff --git a/uspv/sortsignsend.go b/uspv/sortsignsend.go index d7b91610..894978e9 100644 --- a/uspv/sortsignsend.go +++ b/uspv/sortsignsend.go @@ -4,6 +4,7 @@ import ( "bytes" "fmt" "log" + "sort" "github.com/btcsuite/btcd/txscript" "github.com/btcsuite/btcd/wire" @@ -42,15 +43,29 @@ func (s *SPVCon) Rebroadcast() { return } -func P2wpkhScript(adr btcutil.Address) ([]byte, error) { - switch adr := adr.(type) { - case *btcutil.AddressPubKeyHash: - sb := txscript.NewScriptBuilder() - sb.AddOp(txscript.OP_0) - sb.AddData(adr.ScriptAddress()) - return sb.Script() +// make utxo slices sortable +type utxoSlice []Utxo + +// Sort utxos just like txins -- Len, Less, Swap +func (s utxoSlice) Len() int { return len(s) } +func (s utxoSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// outpoint sort; First input hash (reversed / rpc-style), then index. +func (s utxoSlice) Less(i, j int) bool { + // Input hashes are the same, so compare the index. + ihash := s[i].Op.Hash + jhash := s[j].Op.Hash + if ihash == jhash { + return s[i].Op.Index < s[j].Op.Index } - return nil, fmt.Errorf("%s is not pkh address", adr.String()) + // At this point, the hashes are not equal, so reverse them to + // big-endian and return the result of the comparison. + const hashSize = wire.HashSize + for b := 0; b < hashSize/2; b++ { + ihash[b], ihash[hashSize-1-b] = ihash[hashSize-1-b], ihash[b] + jhash[b], jhash[hashSize-1-b] = jhash[hashSize-1-b], jhash[b] + } + return bytes.Compare(ihash[:], jhash[:]) == -1 } func (s *SPVCon) NewOutgoingTx(tx *wire.MsgTx) error { @@ -75,6 +90,130 @@ func (s *SPVCon) NewOutgoingTx(tx *wire.MsgTx) error { return nil } +// SendCoins does send coins, but it's very rudimentary +// wit makes it into p2wpkh. Which is not yet spendable. +func (s *SPVCon) SendCoins(adr btcutil.Address, sendAmt int64, wit bool) error { + + var err error + var score int64 + allUtxos, err := s.TS.GetAllUtxos() + if err != nil { + return err + } + + for _, utxo := range allUtxos { + score += utxo.Value + } + // important rule in bitcoin, output total > input total is invalid. + if sendAmt > score { + return fmt.Errorf("trying to send %d but %d available.", + sendAmt, score) + } + + /////////////////// + tx := wire.NewMsgTx() // make new tx + // make address script 76a914...88ac or 0014... + outAdrScript, err := txscript.PayToAddrScript(adr) + if err != nil { + return err + } + + // make user specified txout and add to tx + txout := wire.NewTxOut(sendAmt, outAdrScript) + tx.AddTxOut(txout) + //////////////////////////// + + // generate a utxo slice for your inputs + nokori := sendAmt // nokori is how much is needed on input side + var ins utxoSlice + for _, utxo := range allUtxos { + // yeah, lets add this utxo! + ins = append(ins, *utxo) + nokori -= utxo.Value + if nokori < -10000 { // minimum overage / fee is 1K now + break + } + } + // sort utxos on the input side + sort.Sort(ins) + + // add all the utxos as txins + for _, in := range ins { + var prevPKscript []byte + + prevPKscript, err := txscript.PayToAddrScript( + s.TS.Adrs[in.KeyIdx].PkhAdr) + if err != nil { + return err + } + tx.AddTxIn(wire.NewTxIn(&in.Op, prevPKscript, nil)) + } + + // there's enough left to make a change output + if nokori < -200000 { + change, err := s.TS.NewAdr(true) // change is witnessy + if err != nil { + return err + } + + changeScript, err := txscript.PayToAddrScript(change) + if err != nil { + return err + } + changeOut := wire.NewTxOut((-100000)-nokori, changeScript) + tx.AddTxOut(changeOut) + } + + for _, utxo := range allUtxos { + // generate pkscript to sign + prevPKscript, err := txscript.PayToAddrScript( + s.TS.Adrs[utxo.KeyIdx].PkhAdr) + if err != nil { + return err + } + // make new input from this utxo + thisInput := wire.NewTxIn(&utxo.Op, prevPKscript, nil) + tx.AddTxIn(thisInput) + nokori -= utxo.Value + if nokori < -10000 { // minimum overage / fee is 1K now + break + } + } + // there's enough left to make a change output + if nokori < -200000 { + change, err := s.TS.NewAdr(true) // change is witnessy + if err != nil { + return err + } + + changeScript, err := txscript.PayToAddrScript(change) + if err != nil { + return err + } + changeOut := wire.NewTxOut((-100000)-nokori, changeScript) + tx.AddTxOut(changeOut) + } + + // use txstore method to sign + err = s.TS.SignThis(tx) + if err != nil { + return err + } + + fmt.Printf("tx: %s", TxToString(tx)) + buf := bytes.NewBuffer(make([]byte, 0, tx.SerializeSize())) + tx.Serialize(buf) + fmt.Printf("tx: %x\n", buf.Bytes()) + + // send it out on the wire. hope it gets there. + // we should deal with rejects. Don't yet. + err = s.NewOutgoingTx(tx) + if err != nil { + return err + } + return nil +} + func (t *TxStore) SignThis(tx *wire.MsgTx) error { fmt.Printf("-= SignThis =-\n") diff --git a/uspv/txstore.go b/uspv/txstore.go index 0cf6b028..ab93f0f3 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -21,6 +21,7 @@ type TxStore struct { OKMutex sync.Mutex Adrs []MyAdr // endeavouring to acquire capital + WitAdrs []MyAdr // separate segwit address slice StateDB *bolt.DB // place to write all this down // Params live here, not SCon @@ -38,7 +39,8 @@ type Utxo struct { // cash money. KeyIdx uint32 // index for private key needed to sign / spend Value int64 // higher is better - // IsCoinbase bool // can't spend for a while + // IsCoinbase bool // can't spend for a while + IsWit bool // true if p2wpkh output } // Stxo is a utxo that has moved on. @@ -77,8 +79,8 @@ func (t *TxStore) AddTxid(txid *wire.ShaHash, height int32) error { // ... or I'm gonna fade away func (t *TxStore) GimmeFilter() (*bloom.Filter, error) { - if len(t.Adrs) == 0 { - return nil, fmt.Errorf("no addresses to filter for") + if len(t.Adrs) == 0 && len(t.WitAdrs) == 9 { + return nil, fmt.Errorf("no address to filter for") } // get all utxos to add outpoints to filter @@ -87,9 +89,15 @@ func (t *TxStore) GimmeFilter() (*bloom.Filter, error) { return nil, err } - elem := uint32(len(t.Adrs) + len(allUtxos)) + elem := uint32(len(t.Adrs) + len(t.WitAdrs) + len(allUtxos)) f := bloom.NewFilter(elem, 0, 0.000001, wire.BloomUpdateAll) - for _, a := range t.Adrs { + + // note there could be false positives since we're just looking + // for the 20 byte PKH without the opcodes. + for _, a := range t.Adrs { // add 20-byte pubkeyhash + f.Add(a.PkhAdr.ScriptAddress()) + } + for _, a := range t.WitAdrs { // add witness 20-byte pubkeyhash f.Add(a.PkhAdr.ScriptAddress()) } @@ -211,6 +219,16 @@ func (u *Utxo) ToBytes() ([]byte, error) { if err != nil { return nil, err } + // last byte indicates tx witness flags ( tx[5] from serialized tx) + // write a 1 at the end for p2wpkh (same as flags byte) + witByte := byte(0x00) + if u.IsWit { + witByte = 0x01 + } + err = buf.WriteByte(witByte) + if err != nil { + return nil, err + } return buf.Bytes(), nil } @@ -224,8 +242,8 @@ func UtxoFromBytes(b []byte) (Utxo, error) { return u, fmt.Errorf("nil input slice") } buf := bytes.NewBuffer(b) - if buf.Len() < 52 { // utxos are 52 bytes - return u, fmt.Errorf("Got %d bytes for utxo, expect 52", buf.Len()) + if buf.Len() < 53 { // utxos are 53 bytes + return u, fmt.Errorf("Got %d bytes for utxo, expect 53", buf.Len()) } // read 32 byte txid err := u.Op.Hash.SetBytes(buf.Next(32)) @@ -252,38 +270,33 @@ func UtxoFromBytes(b []byte) (Utxo, error) { if err != nil { return u, err } + // read 1 byte witness flags + witByte, err := buf.ReadByte() + if err != nil { + return u, err + } + if witByte != 0x00 { + u.IsWit = true + } + return u, nil } // ToBytes turns an Stxo into some bytes. -// outpoint txid, outpoint idx, height, key idx, amt, spendheight, spendtxid +// prevUtxo serialization, then spendheight [4], spendtxid [32] func (s *Stxo) ToBytes() ([]byte, error) { var buf bytes.Buffer - // write 32 byte txid of the utxo - _, err := buf.Write(s.Op.Hash.Bytes()) + // first serialize the utxo part + uBytes, err := s.Utxo.ToBytes() if err != nil { return nil, err } - // write 4 byte outpoint index within the tx to spend - err = binary.Write(&buf, binary.BigEndian, s.Op.Index) - if err != nil { - return nil, err - } - // write 4 byte height of utxo - err = binary.Write(&buf, binary.BigEndian, s.AtHeight) - if err != nil { - return nil, err - } - // write 4 byte key index of utxo - err = binary.Write(&buf, binary.BigEndian, s.KeyIdx) - if err != nil { - return nil, err - } - // write 8 byte amount of money at the utxo - err = binary.Write(&buf, binary.BigEndian, s.Value) + // write that into the buffer first + _, err = buf.Write(uBytes) if err != nil { return nil, err } + // write 4 byte height where the txo was spent err = binary.Write(&buf, binary.BigEndian, s.SpendHeight) if err != nil { @@ -298,40 +311,21 @@ func (s *Stxo) ToBytes() ([]byte, error) { } // StxoFromBytes turns bytes into a Stxo. +// first take the first 53 bytes as a utxo, then the next 36 for how it's spent. func StxoFromBytes(b []byte) (Stxo, error) { var s Stxo - if b == nil { - return s, fmt.Errorf("nil input slice") + if len(b) < 89 { + return s, fmt.Errorf("Got %d bytes for stxo, expect 89", len(b)) } - buf := bytes.NewBuffer(b) - if buf.Len() < 88 { // stxos are 88 bytes - return s, fmt.Errorf("Got %d bytes for stxo, expect 88", buf.Len()) - } - // read 32 byte txid - err := s.Op.Hash.SetBytes(buf.Next(32)) - if err != nil { - return s, err - } - // read 4 byte outpoint index within the tx to spend - err = binary.Read(buf, binary.BigEndian, &s.Op.Index) - if err != nil { - return s, err - } - // read 4 byte height of utxo - err = binary.Read(buf, binary.BigEndian, &s.AtHeight) - if err != nil { - return s, err - } - // read 4 byte key index of utxo - err = binary.Read(buf, binary.BigEndian, &s.KeyIdx) - if err != nil { - return s, err - } - // read 8 byte amount of money at the utxo - err = binary.Read(buf, binary.BigEndian, &s.Value) + + u, err := UtxoFromBytes(b[:53]) if err != nil { return s, err } + s.Utxo = u // assign the utxo + + buf := bytes.NewBuffer(b[53:]) // make buffer for spend data + // read 4 byte spend height err = binary.Read(buf, binary.BigEndian, &s.SpendHeight) if err != nil { diff --git a/uspv/utxodb.go b/uspv/utxodb.go index 8550b50c..b4010103 100644 --- a/uspv/utxodb.go +++ b/uspv/utxodb.go @@ -22,7 +22,8 @@ var ( BKTTxns = []byte("Txns") // all txs we care about, for replays BKTState = []byte("MiscState") // last state of DB // these are in the state bucket - KEYNumKeys = []byte("NumKeys") // number of keys used + KEYStdKeys = []byte("StdKeys") // number of p2pkh keys used + KEYWitKeys = []byte("WitKeys") // number of p2wpkh keys used KEYTipHeight = []byte("TipHeight") // height synced to ) @@ -52,7 +53,7 @@ func (ts *TxStore) OpenDB(filename string) error { return err } - numKeysBytes := sta.Get(KEYNumKeys) + numKeysBytes := sta.Get(KEYStdKeys) if numKeysBytes != nil { // NumKeys exists, read into uint32 buf := bytes.NewBuffer(numKeysBytes) err := binary.Read(buf, binary.BigEndian, &numKeys) @@ -67,7 +68,7 @@ func (ts *TxStore) OpenDB(filename string) error { if err != nil { return err } - err = sta.Put(KEYNumKeys, buf.Bytes()) + err = sta.Put(KEYStdKeys, buf.Bytes()) if err != nil { return err } @@ -82,19 +83,42 @@ func (ts *TxStore) OpenDB(filename string) error { // NewAdr creates a new, never before seen address, and increments the // DB counter as well as putting it in the ram Adrs store, and returns it -func (ts *TxStore) NewAdr() (*btcutil.AddressPubKeyHash, error) { +func (ts *TxStore) NewAdr(wit bool) (btcutil.Address, error) { if ts.Param == nil { - return nil, fmt.Errorf("nil param") - } - n := uint32(len(ts.Adrs)) - priv, err := ts.rootPrivKey.Child(n + hdkeychain.HardenedKeyStart) - if err != nil { - return nil, err + return nil, fmt.Errorf("NewAdr error: nil param") } - newAdr, err := priv.Address(ts.Param) - if err != nil { - return nil, err + priv := new(hdkeychain.ExtendedKey) + var err error + var n uint32 + var nAdr btcutil.Address + + if wit { + n = uint32(len(ts.WitAdrs)) + + // witness keys are another branch down from the rootpriv + ephpriv, err := ts.rootPrivKey.Child(1<<30 + hdkeychain.HardenedKeyStart) + if err != nil { + return nil, err + } + priv, err = ephpriv.Child(n + hdkeychain.HardenedKeyStart) + if err != nil { + return nil, err + } + nAdr, err = priv.WitnessAddress(ts.Param) + if err != nil { + return nil, err + } + } else { // regular p2pkh + n = uint32(len(ts.Adrs)) + priv, err = ts.rootPrivKey.Child(n + hdkeychain.HardenedKeyStart) + if err != nil { + return nil, err + } + nAdr, err = priv.Address(ts.Param) + if err != nil { + return nil, err + } } // total number of keys (now +1) into 4 bytes @@ -107,18 +131,24 @@ func (ts *TxStore) NewAdr() (*btcutil.AddressPubKeyHash, error) { // write to db file err = ts.StateDB.Update(func(btx *bolt.Tx) error { sta := btx.Bucket(BKTState) - return sta.Put(KEYNumKeys, buf.Bytes()) + if wit { + return sta.Put(KEYStdKeys, buf.Bytes()) + } + return sta.Put(KEYWitKeys, buf.Bytes()) }) if err != nil { return nil, err } // add in to ram. var ma MyAdr - ma.PkhAdr = newAdr + ma.PkhAdr = nAdr ma.KeyIdx = n - ts.Adrs = append(ts.Adrs, ma) - - return newAdr, nil + if wit { + ts.WitAdrs = append(ts.WitAdrs, ma) + } else { + ts.Adrs = append(ts.Adrs, ma) + } + return nAdr, nil } // SetDBSyncHeight sets sync height of the db, indicated the latest block @@ -369,17 +399,38 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) { // also generate PKscripts for all addresses (maybe keep storing these?) for _, adr := range ts.Adrs { // iterate through all our addresses + + // convert regular address to witness address. (split adrs later) + wa, err := btcutil.NewAddressWitnessPubKeyHash( + adr.PkhAdr.ScriptAddress(), ts.Param) + if err != nil { + return hits, err + } + + wPKscript, err := txscript.PayToAddrScript(wa) + if err != nil { + return hits, err + } aPKscript, err := txscript.PayToAddrScript(adr.PkhAdr) if err != nil { return hits, err } - // iterate through all outputs of this tx + + // iterate through all outputs of this tx, see if we gain for i, out := range tx.TxOut { - if bytes.Equal(out.PkScript, aPKscript) { // new utxo for us - var newu Utxo + + // detect p2wpkh + witBool := false + if bytes.Equal(out.PkScript, wPKscript) { + witBool = true + } + + if bytes.Equal(out.PkScript, aPKscript) || witBool { // new utxo found + var newu Utxo // create new utxo and copy into it newu.AtHeight = height newu.KeyIdx = adr.KeyIdx newu.Value = out.Value + newu.IsWit = witBool // copy witness version from pkscript var newop wire.OutPoint newop.Hash = tx.TxSha() newop.Index = uint32(i)