diff --git a/.gitignore b/.gitignore index b55095c7..2d4d343a 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,9 @@ cmd/cmd cmd/lncli/lncli cmd/lnshell/lnshell +sighash143/sighash143 +sighash143/*.hex + test_wal/* # vim diff --git a/lnwallet/channel.go b/lnwallet/channel.go index 93ad4938..8bb4fa62 100644 --- a/lnwallet/channel.go +++ b/lnwallet/channel.go @@ -86,7 +86,7 @@ func newLightningChannel(wallet *LightningWallet, events chainntnfs.ChainNotifie return nil, err } _, multiSigIndex := findScriptOutputIndex(state.FundingTx, fundingPkScript) - lc.fundingTxIn = wire.NewTxIn(wire.NewOutPoint(&fundingTxId, multiSigIndex), nil) + lc.fundingTxIn = wire.NewTxIn(wire.NewOutPoint(&fundingTxId, multiSigIndex), nil, nil) lc.fundingP2SH = fundingPkScript return lc, nil diff --git a/lnwallet/wallet.go b/lnwallet/wallet.go index 372b6cf3..8aaed0ac 100644 --- a/lnwallet/wallet.go +++ b/lnwallet/wallet.go @@ -529,7 +529,7 @@ func (l *LightningWallet) handleFundingReserveRequest(req *initFundingReserveMsg // Empty sig script, we'll actually sign if this reservation is // queued up to be completed (the other side accepts). outPoint := wire.NewOutPoint(coin.Hash(), coin.Index()) - ourContribution.Inputs[i] = wire.NewTxIn(outPoint, nil) + ourContribution.Inputs[i] = wire.NewTxIn(outPoint, nil, nil) } l.coinSelectMtx.Unlock() @@ -794,7 +794,7 @@ func (l *LightningWallet) handleContributionMsg(req *addContributionMsg) { // since the outputs are cannonically sorted. fundingNTxid := fundingTx.TxSha() // NOTE: assumes testnet-L _, multiSigIndex := findScriptOutputIndex(fundingTx, multiSigOut.PkScript) - fundingTxIn := wire.NewTxIn(wire.NewOutPoint(&fundingNTxid, multiSigIndex), nil) + fundingTxIn := wire.NewTxIn(wire.NewOutPoint(&fundingNTxid, multiSigIndex), nil, nil) // With the funding tx complete, create both commitment transactions. initialBalance := ourContribution.FundingAmount diff --git a/lnwire/lnwire.go b/lnwire/lnwire.go index 3e5be3df..39909fe6 100644 --- a/lnwire/lnwire.go +++ b/lnwire/lnwire.go @@ -4,11 +4,12 @@ import ( "bytes" "encoding/binary" "fmt" + "io" + "io/ioutil" + "github.com/btcsuite/btcd/btcec" "github.com/btcsuite/btcd/wire" "github.com/btcsuite/btcutil" - "io" - "io/ioutil" ) var MAX_SLICE_LENGTH = 65535 @@ -557,7 +558,7 @@ func readElement(r io.Reader, element interface{}) error { var txins []*wire.TxIn for i := uint8(0); i < numScripts; i++ { outpoint := new(wire.OutPoint) - txin := wire.NewTxIn(outpoint, nil) + txin := wire.NewTxIn(outpoint, nil, nil) err = readElement(r, &txin) if err != nil { return err diff --git a/shell.go b/shell.go index 9438e7de..2ec9a50b 100644 --- a/shell.go +++ b/shell.go @@ -2,16 +2,13 @@ package main import ( "bufio" - "bytes" "fmt" "log" "os" + "sort" "strconv" "strings" - "github.com/btcsuite/btcd/txscript" - "github.com/btcsuite/btcd/wire" - "github.com/btcsuite/btcd/chaincfg" "github.com/btcsuite/btcutil" @@ -25,15 +22,15 @@ testing. It can send and receive coins. const ( keyFileName = "testkey.hex" headerFileName = "headers.bin" - dbFileName = "/dev/shm/utxo.db" + dbFileName = "utxo.db" // this is my local testnet node, replace it with your own close by. // Random internet testnet nodes usually work but sometimes don't, so // maybe I should test against different versions out there. - SPVHostAdr = "127.0.0.1:18333" + SPVHostAdr = "127.0.0.1:28333" ) var ( - Params = &chaincfg.TestNet3Params + Params = &chaincfg.SegNetParams SCon uspv.SPVCon // global here for now ) @@ -51,7 +48,7 @@ func shell() { // setup spvCon SCon, err = uspv.OpenSPV( - SPVHostAdr, headerFileName, dbFileName, &Store, false, false, Params) + SPVHostAdr, headerFileName, dbFileName, &Store, true, false, Params) if err != nil { log.Fatal(err) } @@ -61,14 +58,14 @@ func shell() { log.Fatal(err) } if tip == 0 { // DB has never been used, set to birthday - tip = 675000 // hardcoded; later base on keyfile date? + tip = 21900 // hardcoded; later base on keyfile date? err = SCon.TS.SetDBSyncHeight(tip) if err != nil { log.Fatal(err) } } - // once we're connected, initiate headers sync + // once we're connected, initiate headers sync err = SCon.AskForHeaders() if err != nil { log.Fatal(err) @@ -144,6 +141,20 @@ func Shellparse(cmdslice []string) error { } return nil } + if cmd == "fan" { + err = Fan(args) + if err != nil { + fmt.Printf("fan error: %s\n", err) + } + return nil + } + if cmd == "sweep" { + err = Sweep(args) + if err != nil { + fmt.Printf("sweep error: %s\n", err) + } + return nil + } if cmd == "txs" { err = Txs(args) if err != nil { @@ -199,39 +210,164 @@ func Bal(args []string) error { return fmt.Errorf("Can't get balance, spv connection broken") } fmt.Printf(" ----- Account Balance ----- \n") - allUtxos, err := SCon.TS.GetAllUtxos() + rawUtxos, err := SCon.TS.GetAllUtxos() if err != nil { return err } - var score int64 + var allUtxos uspv.SortableUtxoSlice + for _, utxo := range rawUtxos { + allUtxos = append(allUtxos, *utxo) + } + // smallest and unconfirmed last (because it's reversed) + sort.Sort(sort.Reverse(allUtxos)) + + var score, confScore int64 for i, u := range allUtxos { - fmt.Printf("\tutxo %d height %d %s key: %d amt %d\n", + fmt.Printf("\tutxo %d height %d %s key:%d amt %d", i, u.AtHeight, u.Op.String(), u.KeyIdx, u.Value) + if u.IsWit { + fmt.Printf(" WIT") + } + fmt.Printf("\n") score += u.Value + if u.AtHeight != 0 { + confScore += u.Value + } } height, _ := SCon.TS.GetDBSyncHeight() + atx, err := SCon.TS.GetAllTxs() + + stxos, err := SCon.TS.GetAllStxos() + for i, a := range SCon.TS.Adrs { - fmt.Printf("address %d %s\n", i, a.PkhAdr.String()) + wa, err := btcutil.NewAddressWitnessPubKeyHash( + a.PkhAdr.ScriptAddress(), Params) + if err != nil { + return err + } + fmt.Printf("address %d %s OR %s\n", i, a.PkhAdr.String(), wa.String()) } - fmt.Printf("Total known utxos: %d\n", len(allUtxos)) - fmt.Printf("Total spendable coin: %d\n", score) + fmt.Printf("Total known txs: %d\n", len(atx)) + fmt.Printf("Known utxos: %d\tPreviously spent txos: %d\n", + len(allUtxos), len(stxos)) + fmt.Printf("Total coin: %d confirmed: %d\n", score, confScore) fmt.Printf("DB sync height: %d\n", height) return nil } // Adr makes a new address. func Adr(args []string) error { + + // if there's an arg, make 10 adrs + if len(args) > 0 { + for i := 0; i < 10; i++ { + _, err := SCon.TS.NewAdr() + if err != nil { + return err + } + } + } + if len(args) > 1 { + for i := 0; i < 1000; i++ { + _, err := SCon.TS.NewAdr() + if err != nil { + return err + } + } + } + + // always make one a, err := SCon.TS.NewAdr() if err != nil { return err } - fmt.Printf("made new address %s, %d addresses total\n", - a.String(), len(SCon.TS.Adrs)) + fmt.Printf("made new address %s\n", + a.String()) return nil } +// Sweep sends every confirmed uxto in your wallet to an address. +// it does them all individually to there are a lot of txs generated. +// syntax: sweep adr +func Sweep(args []string) error { + if len(args) < 2 { + return fmt.Errorf("sweep syntax: sweep adr") + } + + adr, err := btcutil.DecodeAddress(args[0], SCon.TS.Param) + if err != nil { + fmt.Printf("error parsing %s as address\t", args[0]) + return err + } + numTxs, err := strconv.ParseInt(args[1], 10, 64) + if err != nil { + return err + } + if numTxs < 1 { + return fmt.Errorf("can't send %d txs", numTxs) + } + + rawUtxos, err := SCon.TS.GetAllUtxos() + if err != nil { + return err + } + var allUtxos uspv.SortableUtxoSlice + for _, utxo := range rawUtxos { + allUtxos = append(allUtxos, *utxo) + } + // smallest and unconfirmed last (because it's reversed) + sort.Sort(sort.Reverse(allUtxos)) + + for i, u := range allUtxos { + if u.AtHeight != 0 { + err = SCon.SendOne(allUtxos[i], adr) + if err != nil { + return err + } + numTxs-- + if numTxs == 0 { + return nil + } + } + } + + fmt.Printf("spent all confirmed utxos; not enough by %d\n", numTxs) + + return nil +} + +// Fan generates a bunch of fanout. Only for testing, can be expensive. +// syntax: fan adr numOutputs valOutputs witty +func Fan(args []string) error { + if len(args) < 3 { + return fmt.Errorf("fan syntax: fan adr numOutputs valOutputs") + } + adr, err := btcutil.DecodeAddress(args[0], SCon.TS.Param) + if err != nil { + fmt.Printf("error parsing %s as address\t", args[0]) + return err + } + numOutputs, err := strconv.ParseInt(args[1], 10, 64) + if err != nil { + return err + } + valOutputs, err := strconv.ParseInt(args[2], 10, 64) + if err != nil { + return err + } + + adrs := make([]btcutil.Address, numOutputs) + amts := make([]int64, numOutputs) + + for i := int64(0); i < numOutputs; i++ { + adrs[i] = adr + amts[i] = valOutputs + i + } + return SCon.SendCoins(adrs, amts) +} + // Send sends coins. func Send(args []string) error { if SCon.RBytes == 0 { @@ -254,102 +390,30 @@ func Send(args []string) error { } // need args, fail if len(args) < 2 { - return fmt.Errorf("need args: ssend amount(satoshis) address") + return fmt.Errorf("need args: ssend address amount(satoshis) wit?") } - amt, err := strconv.ParseInt(args[0], 10, 64) + adr, err := btcutil.DecodeAddress(args[0], SCon.TS.Param) + if err != nil { + fmt.Printf("error parsing %s as address\t", args[0]) + return err + } + amt, err := strconv.ParseInt(args[1], 10, 64) if err != nil { return err } if amt < 1000 { return fmt.Errorf("can't send %d, too small", amt) } - adr, err := btcutil.DecodeAddress(args[1], SCon.TS.Param) - if err != nil { - fmt.Printf("error parsing %s as address\t", args[1]) - return err - } + fmt.Printf("send %d to address: %s \n", amt, adr.String()) - err = SendCoins(SCon, adr, amt) - if err != nil { - return err - } - return nil -} -// SendCoins does send coins, but it's very rudimentary -func SendCoins(s uspv.SPVCon, adr btcutil.Address, sendAmt int64) error { - var err error - var score int64 - allUtxos, err := s.TS.GetAllUtxos() - if err != nil { - return err - } + var adrs []btcutil.Address + var amts []int64 - for _, utxo := range allUtxos { - score += utxo.Value - } - // important rule in bitcoin, output total > input total is invalid. - if sendAmt > score { - return fmt.Errorf("trying to send %d but %d available.", - sendAmt, score) - } - - tx := wire.NewMsgTx() // make new tx - // make address script 76a914...88ac - adrScript, err := txscript.PayToAddrScript(adr) - if err != nil { - return err - } - // make user specified txout and add to tx - txout := wire.NewTxOut(sendAmt, adrScript) - tx.AddTxOut(txout) - - nokori := sendAmt // nokori is how much is needed on input side - for _, utxo := range allUtxos { - // generate pkscript to sign - prevPKscript, err := txscript.PayToAddrScript( - s.TS.Adrs[utxo.KeyIdx].PkhAdr) - if err != nil { - return err - } - // make new input from this utxo - thisInput := wire.NewTxIn(&utxo.Op, prevPKscript) - tx.AddTxIn(thisInput) - nokori -= utxo.Value - if nokori < -10000 { // minimum overage / fee is 1K now - break - } - } - // there's enough left to make a change output - if nokori < -200000 { - change, err := s.TS.NewAdr() - if err != nil { - return err - } - - changeScript, err := txscript.PayToAddrScript(change) - if err != nil { - return err - } - changeOut := wire.NewTxOut((-100000)-nokori, changeScript) - tx.AddTxOut(changeOut) - } - - // use txstore method to sign - err = s.TS.SignThis(tx) - if err != nil { - return err - } - - fmt.Printf("tx: %s", uspv.TxToString(tx)) - buf := bytes.NewBuffer(make([]byte, 0, tx.SerializeSize())) - tx.Serialize(buf) - fmt.Printf("tx: %x\n", buf.Bytes()) - - // send it out on the wire. hope it gets there. - // we should deal with rejects. Don't yet. - err = s.NewOutgoingTx(tx) + adrs = append(adrs, adr) + amts = append(amts, amt) + err = SCon.SendCoins(adrs, amts) if err != nil { return err } diff --git a/sighash143/sighash143.go b/sighash143/sighash143.go new file mode 100644 index 00000000..419bf54e --- /dev/null +++ b/sighash143/sighash143.go @@ -0,0 +1,317 @@ +package main + +import ( + "bytes" + "encoding/binary" + "encoding/hex" + "fmt" + "io" + "io/ioutil" + "log" + "math" + "strings" + + "github.com/btcsuite/btcd/btcec" + "github.com/btcsuite/btcd/txscript" + "github.com/btcsuite/btcd/wire" + "github.com/lightningnetwork/lnd/uspv" +) + +const ( + inspk0 = "2103c9f4836b9a4f77fc0d81f7bcb01b7f1b35916864b9476c241ce9fc198bd25432ac" + inamt0 = int64(625000000) + + inspk1 = "00141d0f172a0ecb48aee1be1f2687d2963ae33f71a1" + inamt1 = int64(600000000) + + xpecthash = "c37af31116d1b27caf68aae9e3ac82f1477929014d5b917657d0eb49478cb670" +) + +// calcWitnessSignatureHash is the witnessified version of calcSignatureHash +// put your pkscripts in the sigscript slot before handing the tx to this +// function. Also you're clearly supposed to cache the 3 sub-hashes generated +// here, because they apply to the tx, not a txin. But this doesn't yet. +func calcWitnessSignatureHash(sigscript []byte, + hashType txscript.SigHashType, tx *wire.MsgTx, idx int, amt int64) []byte { + // in the script.go calcSignatureHash(), idx is assumed safe, so I guess + // that's OK here too...? Nah I'm gonna check + if idx > len(tx.TxIn)-1 { + fmt.Printf("calcWitnessSignatureHash error: idx %d but %d txins", + idx, len(tx.TxIn)) + return nil + } + + // first get hashPrevOuts, hashSequence, and hashOutputs + hashPrevOuts := calcHashPrevOuts(tx, hashType) + hashSequence := calcHashSequence(tx, hashType) + hashOutputs := calcHashOutputs(tx, idx, hashType) + + var buf4 [4]byte // buffer for 4-byte stuff + var buf8 [8]byte // buffer for 8-byte stuff + var pre []byte // the pre-image we're generating + + binary.LittleEndian.PutUint32(buf4[:], uint32(tx.Version)) + pre = append(pre, buf4[:]...) + + pre = append(pre, hashPrevOuts.Bytes()...) + pre = append(pre, hashSequence.Bytes()...) + + // outpoint being spent + pre = append(pre, tx.TxIn[idx].PreviousOutPoint.Hash.Bytes()...) + binary.LittleEndian.PutUint32(buf4[:], tx.TxIn[idx].PreviousOutPoint.Index) + pre = append(pre, buf4[:]...) + + // scriptCode which is some new thing + + // detect wpkh mode + fmt.Printf("txin %d sscript len %d\n", idx, len(tx.TxIn[idx].SignatureScript)) + if len(sigscript) == 22 && sigscript[0] == 0x00 && sigscript[1] == 0x14 { + // wpkh mode .... recreate op_dup codes here + sCode := []byte{0x19, 0x76, 0xa9, 0x14} + sCode = append(sCode, sigscript[2:22]...) + sCode = append(sCode, []byte{0x88, 0xac}...) + pre = append(pre, sCode...) + } else if len(sigscript) == 34 && + sigscript[0] == 0x00 && sigscript[1] == 0x20 { + // whs mode- need to remove codeseparators. this doesn't yet. + var buf bytes.Buffer + writeVarBytes(&buf, 0, sigscript) + pre = append(pre, buf.Bytes()...) + } else { + // ?? this is not witness tx! fail + fmt.Printf("Non witness error ") + return nil + } + + // amount being signed off + binary.LittleEndian.PutUint64(buf8[:], uint64(amt)) + pre = append(pre, buf8[:]...) + + // nsequence of input + binary.LittleEndian.PutUint32(buf4[:], tx.TxIn[idx].Sequence) + pre = append(pre, buf4[:]...) + + pre = append(pre, hashOutputs.Bytes()...) + + // locktime + binary.LittleEndian.PutUint32(buf4[:], tx.LockTime) + pre = append(pre, buf4[:]...) + + // hashType... in 4 bytes, instead of 1, because reasons. + binary.LittleEndian.PutUint32(buf4[:], uint32(hashType)) + pre = append(pre, buf4[:]...) + + fmt.Printf("calcWitnessSignatureHash pre: %x\n", pre) + hsh := wire.DoubleSha256SH(pre) + return hsh.Bytes() +} + +// calcHashPrevOuts makes a single hash of all previous outputs in the tx +func calcHashPrevOuts(tx *wire.MsgTx, hType txscript.SigHashType) wire.ShaHash { + // skip this (0x00) for anyonecanpay + if hType == txscript.SigHashAnyOneCanPay { + var empty [32]byte + return empty + } + + var pre []byte + for _, in := range tx.TxIn { + // first append 32 byte hash + pre = append(pre, in.PreviousOutPoint.Hash.Bytes()...) + // then make a buffer, put 4 byte index in lil' endian and append that + var buf [4]byte + binary.LittleEndian.PutUint32(buf[:], in.PreviousOutPoint.Index) + pre = append(pre, buf[:]...) + } + fmt.Printf("calcHashPrevOuts pre: %x\n", pre) + return wire.DoubleSha256SH(pre) +} + +// calcHashSequence is hash of txins' seq numbers, lil' endian, stuck together +func calcHashSequence(tx *wire.MsgTx, hType txscript.SigHashType) wire.ShaHash { + // skip (0x00) for single, none, anyonecanpay + if hType == txscript.SigHashSingle || hType == txscript.SigHashNone || + hType == txscript.SigHashAnyOneCanPay { + var empty [32]byte + return empty + } + + var pre []byte + for _, in := range tx.TxIn { + var buf [4]byte + binary.LittleEndian.PutUint32(buf[:], in.Sequence) + pre = append(pre, buf[:]...) + } + fmt.Printf("calcHashSequence pre: %x\n", pre) + return wire.DoubleSha256SH(pre) +} + +// calcHashOutputs also wants a input index, which it only uses for +// sighash single. If it's not sighash single, just put a 0 or whatever. +func calcHashOutputs( + tx *wire.MsgTx, inIndex int, hType txscript.SigHashType) wire.ShaHash { + if hType == txscript.SigHashNone || + (hType == txscript.SigHashSingle && inIndex <= len(tx.TxOut)) { + var empty [32]byte + return empty + } + if hType == txscript.SigHashSingle { + var buf bytes.Buffer + writeTxOut(&buf, 0, 0, tx.TxOut[inIndex]) + return wire.DoubleSha256SH(buf.Bytes()) + } + + var pre []byte + for _, out := range tx.TxOut { + var buf bytes.Buffer + writeTxOut(&buf, 0, 0, out) + pre = append(pre, buf.Bytes()...) + } + fmt.Printf("calcHashOutputs pre: %x\n", pre) + return wire.DoubleSha256SH(pre) +} + +func main() { + fmt.Printf("sighash 143\n") + + // get previous pkscripts for inputs 0 and 1 from hex + in0spk, err := hex.DecodeString(string(inspk0)) + if err != nil { + log.Fatal(err) + } + in1spk, err := hex.DecodeString(string(inspk1)) + if err != nil { + log.Fatal(err) + } + xpkt, err := hex.DecodeString(string(xpecthash)) + if err != nil { + log.Fatal(err) + } + + // load tx skeleton from local file + fmt.Printf("loading tx from file tx.hex\n") + txhex, err := ioutil.ReadFile("tx.hex") + if err != nil { + log.Fatal(err) + } + + txhex = []byte(strings.TrimSpace(string(txhex))) + txbytes, err := hex.DecodeString(string(txhex)) + if err != nil { + log.Fatal(err) + } + + fmt.Printf("loaded %d byte tx %x\n", len(txbytes), txbytes) + + // make tx + ttx := wire.NewMsgTx() + // deserialize into tx + buf := bytes.NewBuffer(txbytes) + ttx.Deserialize(buf) + + fmt.Printf("=====tx locktime: %x\n", ttx.LockTime) + ttx.TxIn[0].SignatureScript = in0spk + // ttx.TxIn[1].SignatureScript = in1spk + + fmt.Printf(uspv.TxToString(ttx)) + + priv, err := btcec.NewPrivateKey(btcec.S256()) + if err != nil { + log.Fatal(err) + } + // assert flag before writing witness + ttx.Flags = 0x01 + ttx.TxIn[1].Witness, err = txscript.WitnessScript(ttx, 1, inamt1, in1spk, + txscript.SigHashAll, priv, true) + // ttx.TxIn[1].SignatureScript = nil + + hxh := calcWitnessSignatureHash(in1spk, txscript.SigHashAll, ttx, 1, inamt1) + + fmt.Printf("got sigHash %x NON LIB\n", hxh) + fmt.Printf("expect hash %x\n ", xpkt) + fmt.Printf("\n%s\n", uspv.TxToString(ttx)) + + fmt.Printf("loading tx from file xtx.hex\n") + txhex, err = ioutil.ReadFile("xtx.hex") + if err != nil { + log.Fatal(err) + } + txhex = []byte(strings.TrimSpace(string(txhex))) + txbytes, err = hex.DecodeString(string(txhex)) + if err != nil { + log.Fatal(err) + } + // make tx + xtx := wire.NewMsgTx() + // deserialize into tx + buf = bytes.NewBuffer(txbytes) + xtx.Deserialize(buf) + + // ttx.Flags = 0x01 + fmt.Printf("\n%s\n", uspv.TxToString(xtx)) +} + +// pver can be 0, doesn't do anything in these. Same for msg.Version +// writeVarInt serializes val to w using a variable number of bytes depending +// on its value. +func writeVarInt(w io.Writer, pver uint32, val uint64) error { + if val < 0xfd { + _, err := w.Write([]byte{uint8(val)}) + return err + } + + if val <= math.MaxUint16 { + var buf [3]byte + buf[0] = 0xfd + binary.LittleEndian.PutUint16(buf[1:], uint16(val)) + _, err := w.Write(buf[:]) + return err + } + + if val <= math.MaxUint32 { + var buf [5]byte + buf[0] = 0xfe + binary.LittleEndian.PutUint32(buf[1:], uint32(val)) + _, err := w.Write(buf[:]) + return err + } + + var buf [9]byte + buf[0] = 0xff + binary.LittleEndian.PutUint64(buf[1:], val) + _, err := w.Write(buf[:]) + return err +} + +// writeVarBytes serializes a variable length byte array to w as a varInt +// containing the number of bytes, followed by the bytes themselves. +func writeVarBytes(w io.Writer, pver uint32, bytes []byte) error { + slen := uint64(len(bytes)) + err := writeVarInt(w, pver, slen) + if err != nil { + return err + } + + _, err = w.Write(bytes) + if err != nil { + return err + } + return nil +} + +// writeTxOut encodes to into the bitcoin protocol encoding for a transaction +// output (TxOut) to w. +func writeTxOut(w io.Writer, pver uint32, version int32, to *wire.TxOut) error { + var buf [8]byte + binary.LittleEndian.PutUint64(buf[:], uint64(to.Value)) + _, err := w.Write(buf[:]) + if err != nil { + return err + } + + err = writeVarBytes(w, pver, to.PkScript) + if err != nil { + return err + } + return nil +} diff --git a/uspv/eight333.go b/uspv/eight333.go index 3c40f2c2..4eb525ac 100644 --- a/uspv/eight333.go +++ b/uspv/eight333.go @@ -15,7 +15,8 @@ const ( headerFileName = "headers.bin" // version hardcoded for now, probably ok...? - VERSION = 70011 + // 70012 is for segnet... make this a init var? + VERSION = 70012 ) type SPVCon struct { @@ -115,7 +116,10 @@ func (s *SPVCon) IngestMerkleBlock(m *wire.MsgMerkleBlock) { // into our SPV header file newMerkBlockSha := m.Header.BlockSha() if !hah.blockhash.IsEqual(&newMerkBlockSha) { - log.Printf("merkle block out of order error") + log.Printf("merkle block out of order got %s expect %s", + m.Header.BlockSha().String(), hah.blockhash.String()) + log.Printf("has %d hashes %d txs flags: %x", + len(m.Hashes), m.Transactions, m.Flags) return } @@ -280,6 +284,40 @@ func (s *SPVCon) AskForHeaders() error { return nil } +// AskForOneBlock is for testing only, so you can ask for a specific block height +// and see what goes wrong +func (s *SPVCon) AskForOneBlock(h int32) error { + var hdr wire.BlockHeader + var err error + + dbTip := int32(h) + s.headerMutex.Lock() // seek to header we need + _, err = s.headerFile.Seek(int64((dbTip)*80), os.SEEK_SET) + if err != nil { + return err + } + err = hdr.Deserialize(s.headerFile) // read header, done w/ file for now + s.headerMutex.Unlock() // unlock after reading 1 header + if err != nil { + log.Printf("header deserialize error!\n") + return err + } + + bHash := hdr.BlockSha() + // create inventory we're asking for + iv1 := wire.NewInvVect(wire.InvTypeWitnessBlock, &bHash) + gdataMsg := wire.NewMsgGetData() + // add inventory + err = gdataMsg.AddInvVect(iv1) + if err != nil { + return err + } + hah := NewRootAndHeight(bHash, h) + s.outMsgQueue <- gdataMsg + s.blockQueue <- hah // push height and mroot of requested block on queue + return nil +} + // AskForMerkBlocks requests blocks from current to last // right now this asks for 1 block per getData message. // Maybe it's faster to ask for many in a each message? @@ -311,7 +349,7 @@ func (s *SPVCon) AskForBlocks() error { return nil } - fmt.Printf("will request merkleblocks %d to %d\n", dbTip, headerTip) + fmt.Printf("will request blocks %d to %d\n", dbTip+1, headerTip) if !s.HardMode { // don't send this in hardmode! that's the whole point // create initial filter @@ -324,11 +362,12 @@ func (s *SPVCon) AskForBlocks() error { fmt.Printf("sent filter %x\n", filt.MsgFilterLoad().Filter) } // loop through all heights where we want merkleblocks. - for dbTip <= headerTip { - // load header from file + for dbTip < headerTip { + dbTip++ // we're requesting the next header + // load header from file s.headerMutex.Lock() // seek to header we need - _, err = s.headerFile.Seek(int64((dbTip-1)*80), os.SEEK_SET) + _, err = s.headerFile.Seek(int64((dbTip)*80), os.SEEK_SET) if err != nil { return err } @@ -344,9 +383,9 @@ func (s *SPVCon) AskForBlocks() error { iv1 := new(wire.InvVect) // if hardmode, ask for legit blocks, none of this ralphy stuff if s.HardMode { - iv1 = wire.NewInvVect(wire.InvTypeBlock, &bHash) + iv1 = wire.NewInvVect(wire.InvTypeWitnessBlock, &bHash) } else { // ah well - iv1 = wire.NewInvVect(wire.InvTypeFilteredBlock, &bHash) + iv1 = wire.NewInvVect(wire.InvTypeFilteredWitnessBlock, &bHash) } gdataMsg := wire.NewMsgGetData() // add inventory @@ -354,14 +393,14 @@ func (s *SPVCon) AskForBlocks() error { if err != nil { return err } + hah := NewRootAndHeight(hdr.BlockSha(), dbTip) if dbTip == headerTip { // if this is the last block, indicate finality hah.final = true } - s.outMsgQueue <- gdataMsg // waits here most of the time for the queue to empty out s.blockQueue <- hah // push height and mroot of requested block on queue - dbTip++ + s.outMsgQueue <- gdataMsg } return nil } diff --git a/uspv/hardmode.go b/uspv/hardmode.go index 7cd62c1e..bc40e2c8 100644 --- a/uspv/hardmode.go +++ b/uspv/hardmode.go @@ -1,38 +1,156 @@ package uspv import ( + "bytes" "fmt" "log" - "os" - "sync" "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" + "github.com/btcsuite/btcutil/bloom" ) -// BlockRootOK checks that all the txs in the block match the merkle root. -// Only checks merkle root; it doesn't look at txs themselves. -func BlockRootOK(blk wire.MsgBlock) bool { - var shas []*wire.ShaHash - for _, tx := range blk.Transactions { // make slice of txids - nSha := tx.TxSha() - shas = append(shas, &nSha) +var ( + WitMagicBytes = []byte{0x6a, 0x24, 0xaa, 0x21, 0xa9, 0xed} +) + +// BlockRootOK checks for block self-consistency. +// If the block has no wintess txs, and no coinbase witness commitment, +// it only checks the tx merkle root. If either a witness commitment or +// any witnesses are detected, it also checks that as well. +// Returns false if anything goes wrong, true if everything is fine. +func BlockOK(blk wire.MsgBlock) bool { + var txids, wtxids []*wire.ShaHash // txids and wtxids + // witMode true if any tx has a wintess OR coinbase has wit commit + var witMode bool + + for _, tx := range blk.Transactions { // make slice of (w)/txids + txid := tx.TxSha() + wtxid := tx.WTxSha() + if !witMode && !txid.IsEqual(&wtxid) { + witMode = true + } + txids = append(txids, &txid) + wtxids = append(wtxids, &wtxid) } - neededLen := int(nextPowerOfTwo(uint32(len(shas)))) // kindof ugly - for len(shas) < neededLen { - shas = append(shas, nil) // pad out tx slice to get the full tree base + + var commitBytes []byte + // try to extract coinbase witness commitment (even if !witMode) + cb := blk.Transactions[0] // get coinbase tx + for i := len(cb.TxOut) - 1; i >= 0; i-- { // start at the last txout + if bytes.HasPrefix(cb.TxOut[i].PkScript, WitMagicBytes) && + len(cb.TxOut[i].PkScript) > 37 { + // 38 bytes or more, and starts with WitMagicBytes is a hit + commitBytes = cb.TxOut[i].PkScript[6:38] + witMode = true // it there is a wit commit it must be valid + } } - for len(shas) > 1 { // calculate merkle root. Terse, eh? - shas = append(shas[2:], MakeMerkleParent(shas[0], shas[1])) - } // auto recognizes coinbase-only blocks - return blk.Header.MerkleRoot.IsEqual(shas[0]) + + if witMode { // witmode, so check witness tree + // first find ways witMode can be disqualified + if len(commitBytes) != 32 { + // witness in block but didn't find a wintess commitment; fail + log.Printf("block %s has witness but no witcommit", + blk.BlockSha().String()) + return false + } + if len(cb.TxIn) != 1 { + log.Printf("block %s coinbase tx has %d txins (must be 1)", + blk.BlockSha().String(), len(cb.TxIn)) + return false + } + if len(cb.TxIn[0].Witness) != 1 { + log.Printf("block %s coinbase has %d witnesses (must be 1)", + blk.BlockSha().String(), len(cb.TxIn[0].Witness)) + return false + } + + if len(cb.TxIn[0].Witness[0]) != 32 { + log.Printf("block %s coinbase has %d byte witness nonce (not 32)", + blk.BlockSha().String(), len(cb.TxIn[0].Witness[0])) + return false + } + // witness nonce is the cb's witness, subject to above constraints + witNonce, err := wire.NewShaHash(cb.TxIn[0].Witness[0]) + if err != nil { + log.Printf("Witness nonce error: %s", err.Error()) + return false // not sure why that'd happen but fail + } + + var empty [32]byte + wtxids[0].SetBytes(empty[:]) // coinbase wtxid is 0x00...00 + + // witness root calculated from wtixds + witRoot := calcRoot(wtxids) + + calcWitCommit := wire.DoubleSha256SH( + append(witRoot.Bytes(), witNonce.Bytes()...)) + + // witness root given in coinbase op_return + givenWitCommit, err := wire.NewShaHash(commitBytes) + if err != nil { + log.Printf("Witness root error: %s", err.Error()) + return false // not sure why that'd happen but fail + } + // they should be the same. If not, fail. + if !calcWitCommit.IsEqual(givenWitCommit) { + log.Printf("Block %s witRoot error: calc %s given %s", + blk.BlockSha().String(), + calcWitCommit.String(), givenWitCommit.String()) + return false + } + } + + // got through witMode check so that should be OK; + // check regular txid merkleroot. Which is, like, trivial. + return blk.Header.MerkleRoot.IsEqual(calcRoot(txids)) +} + +// calcRoot calculates the merkle root of a slice of hashes. +func calcRoot(hashes []*wire.ShaHash) *wire.ShaHash { + for len(hashes) < int(nextPowerOfTwo(uint32(len(hashes)))) { + hashes = append(hashes, nil) // pad out hash slice to get the full base + } + for len(hashes) > 1 { // calculate merkle root. Terse, eh? + hashes = append(hashes[2:], MakeMerkleParent(hashes[0], hashes[1])) + } + return hashes[0] +} + +func (ts *TxStore) Refilter() error { + allUtxos, err := ts.GetAllUtxos() + if err != nil { + return err + } + filterElements := uint32(len(allUtxos) + len(ts.Adrs)) + + ts.localFilter = bloom.NewFilter(filterElements, 0, 0, wire.BloomUpdateAll) + + for _, u := range allUtxos { + ts.localFilter.AddOutPoint(&u.Op) + } + for _, a := range ts.Adrs { + ts.localFilter.Add(a.PkhAdr.ScriptAddress()) + } + + msg := ts.localFilter.MsgFilterLoad() + fmt.Printf("made %d element filter: %x\n", filterElements, msg.Filter) + return nil } // IngestBlock is like IngestMerkleBlock but aralphic // different enough that it's better to have 2 separate functions func (s *SPVCon) IngestBlock(m *wire.MsgBlock) { var err error - - ok := BlockRootOK(*m) // check block self-consistency + // var buf bytes.Buffer + // m.SerializeWitness(&buf) + // fmt.Printf("block hex %x\n", buf.Bytes()) + // for _, tx := range m.Transactions { + // fmt.Printf("wtxid: %s\n", tx.WTxSha()) + // fmt.Printf(" txid: %s\n", tx.TxSha()) + // fmt.Printf("%d %s", i, TxToString(tx)) + // } + ok := BlockOK(*m) // check block self-consistency if !ok { fmt.Printf("block %s not OK!!11\n", m.BlockSha().String()) return @@ -53,28 +171,39 @@ func (s *SPVCon) IngestBlock(m *wire.MsgBlock) { return } - // iterate through all txs in the block, looking for matches. - // this is slow and can be sped up by doing in-ram filters client side. - // kindof a pain to implement though and it's fast enough for now. - var wg sync.WaitGroup - wg.Add(len(m.Transactions)) - for i, tx := range m.Transactions { + fPositive := 0 // local filter false positives + reFilter := 10 // after that many false positives, regenerate filter. + // 10? Making it up. False positives have disk i/o cost, and regenning + // the filter also has costs. With a large local filter, false positives + // should be rare. - go func() { + // iterate through all txs in the block, looking for matches. + // use a local bloom filter to ignore txs that don't affect us + for _, tx := range m.Transactions { + utilTx := btcutil.NewTx(tx) + if s.TS.localFilter.MatchTxAndUpdate(utilTx) { hits, err := s.TS.Ingest(tx, hah.height) if err != nil { log.Printf("Incoming Tx error: %s\n", err.Error()) return } if hits > 0 { - log.Printf("block %d tx %d %s ingested and matches %d utxo/adrs.", - hah.height, i, tx.TxSha().String(), hits) + // log.Printf("block %d tx %d %s ingested and matches %d utxo/adrs.", + // hah.height, i, tx.TxSha().String(), hits) + } else { + fPositive++ // matched filter but no hits } - wg.Done() - }() + } } - wg.Wait() + if fPositive > reFilter { + fmt.Printf("%d filter false positives in this block\n", fPositive) + err = s.TS.Refilter() + if err != nil { + log.Printf("Refilter error: %s\n", err.Error()) + return + } + } // write to db that we've sync'd to the height indicated in the // merkle block. This isn't QUITE true since we haven't actually gotten // the txs yet but if there are problems with the txs we should backtrack. @@ -83,6 +212,7 @@ func (s *SPVCon) IngestBlock(m *wire.MsgBlock) { log.Printf("full block sync error: %s\n", err.Error()) return } + fmt.Printf("ingested full block %s height %d OK\n", m.Header.BlockSha().String(), hah.height) @@ -99,35 +229,3 @@ func (s *SPVCon) IngestBlock(m *wire.MsgBlock) { } return } - -func (s *SPVCon) AskForOneBlock(h int32) error { - var hdr wire.BlockHeader - var err error - - dbTip := int32(h) - s.headerMutex.Lock() // seek to header we need - _, err = s.headerFile.Seek(int64((dbTip)*80), os.SEEK_SET) - if err != nil { - return err - } - err = hdr.Deserialize(s.headerFile) // read header, done w/ file for now - s.headerMutex.Unlock() // unlock after reading 1 header - if err != nil { - log.Printf("header deserialize error!\n") - return err - } - - bHash := hdr.BlockSha() - // create inventory we're asking for - iv1 := wire.NewInvVect(wire.InvTypeBlock, &bHash) - gdataMsg := wire.NewMsgGetData() - // add inventory - err = gdataMsg.AddInvVect(iv1) - if err != nil { - return err - } - - s.outMsgQueue <- gdataMsg - - return nil -} diff --git a/uspv/init.go b/uspv/init.go index 4dc84556..0e723daa 100644 --- a/uspv/init.go +++ b/uspv/init.go @@ -49,6 +49,8 @@ func OpenSPV(remoteNode string, hfn, dbfn string, } // must set this to enable SPV stuff myMsgVer.AddService(wire.SFNodeBloom) + // set this to enable segWit + myMsgVer.AddService(wire.SFNodeWitness) // this actually sends n, err := wire.WriteMessageN(s.con, myMsgVer, s.localVersion, s.TS.Param.Net) if err != nil { @@ -89,6 +91,13 @@ func OpenSPV(remoteNode string, hfn, dbfn string, s.inWaitState = make(chan bool, 1) go s.fPositiveHandler() + if hard { + err = s.TS.Refilter() + if err != nil { + return s, err + } + } + return s, nil } diff --git a/uspv/msghandler.go b/uspv/msghandler.go index 06d10c4a..2365f993 100644 --- a/uspv/msghandler.go +++ b/uspv/msghandler.go @@ -5,6 +5,7 @@ import ( "log" "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" ) func (s *SPVCon) incomingMessageHandler() { @@ -88,10 +89,18 @@ func (s *SPVCon) fPositiveHandler() { // send filter s.SendFilter(filt) fmt.Printf("sent filter %x\n", filt.MsgFilterLoad().Filter) + // clear the channel - for len(s.fPositives) != 0 { - fpAccumulator += <-s.fPositives + finClear: + for { + select { + case x := <-s.fPositives: + fpAccumulator += x + default: + break finClear + } } + fmt.Printf("reset %d false positives\n", fpAccumulator) // reset accumulator fpAccumulator = 0 @@ -128,40 +137,43 @@ func (s *SPVCon) TxHandler(m *wire.MsgTx) { height, ok := s.TS.OKTxids[m.TxSha()] s.TS.OKMutex.Unlock() if !ok { - log.Printf("Tx %s unknown, will not ingest\n") + log.Printf("Tx %s unknown, will not ingest\n", m.TxSha().String()) return } // check for double spends - allTxs, err := s.TS.GetAllTxs() - if err != nil { - log.Printf("Can't get txs from db: %s", err.Error()) - return - } - dubs, err := CheckDoubleSpends(m, allTxs) - if err != nil { - log.Printf("CheckDoubleSpends error: %s", err.Error()) - return - } - if len(dubs) > 0 { - for i, dub := range dubs { - fmt.Printf("dub %d known tx %s and new tx %s are exclusive!!!\n", - i, dub.String(), m.TxSha().String()) + // allTxs, err := s.TS.GetAllTxs() + // if err != nil { + // log.Printf("Can't get txs from db: %s", err.Error()) + // return + // } + // dubs, err := CheckDoubleSpends(m, allTxs) + // if err != nil { + // log.Printf("CheckDoubleSpends error: %s", err.Error()) + // return + // } + // if len(dubs) > 0 { + // for i, dub := range dubs { + // fmt.Printf("dub %d known tx %s and new tx %s are exclusive!!!\n", + // i, dub.String(), m.TxSha().String()) + // } + // } + utilTx := btcutil.NewTx(m) + if !s.HardMode || s.TS.localFilter.MatchTxAndUpdate(utilTx) { + hits, err := s.TS.Ingest(m, height) + if err != nil { + log.Printf("Incoming Tx error: %s\n", err.Error()) + return } + if hits == 0 && !s.HardMode { + log.Printf("tx %s had no hits, filter false positive.", + m.TxSha().String()) + s.fPositives <- 1 // add one false positive to chan + return + } + log.Printf("tx %s ingested and matches %d utxo/adrs.", + m.TxSha().String(), hits) } - hits, err := s.TS.Ingest(m, height) - if err != nil { - log.Printf("Incoming Tx error: %s\n", err.Error()) - return - } - if hits == 0 && !s.HardMode { - log.Printf("tx %s had no hits, filter false positive.", - m.TxSha().String()) - s.fPositives <- 1 // add one false positive to chan - return - } - log.Printf("tx %s ingested and matches %d utxo/adrs.", - m.TxSha().String(), hits) } // GetDataHandler responds to requests for tx data, which happen after @@ -172,17 +184,33 @@ func (s *SPVCon) GetDataHandler(m *wire.MsgGetData) { for i, thing := range m.InvList { log.Printf("\t%d)%s : %s", i, thing.Type.String(), thing.Hash.String()) - if thing.Type != wire.InvTypeTx { // refuse non-tx reqs - log.Printf("We only respond to tx requests, ignoring") + + // separate wittx and tx. needed / combine? + // does the same thing right now + if thing.Type == wire.InvTypeWitnessTx { + tx, err := s.TS.GetTx(&thing.Hash) + if err != nil { + log.Printf("error getting tx %s: %s", + thing.Hash.String(), err.Error()) + } + s.outMsgQueue <- tx + sent++ continue } - tx, err := s.TS.GetTx(&thing.Hash) - if err != nil { - log.Printf("error getting tx %s: %s", - thing.Hash.String(), err.Error()) + if thing.Type == wire.InvTypeTx { + tx, err := s.TS.GetTx(&thing.Hash) + if err != nil { + log.Printf("error getting tx %s: %s", + thing.Hash.String(), err.Error()) + } + tx.Flags = 0x00 // dewitnessify + s.outMsgQueue <- tx + sent++ + continue } - s.outMsgQueue <- tx - sent++ + // didn't match, so it's not something we're responding to + log.Printf("We only respond to tx requests, ignoring") + } log.Printf("sent %d of %d requested items", sent, len(m.InvList)) } diff --git a/uspv/sortsignsend.go b/uspv/sortsignsend.go index ec3e3ce0..bb51218e 100644 --- a/uspv/sortsignsend.go +++ b/uspv/sortsignsend.go @@ -4,9 +4,11 @@ import ( "bytes" "fmt" "log" + "sort" "github.com/btcsuite/btcd/txscript" "github.com/btcsuite/btcd/wire" + "github.com/btcsuite/btcutil" "github.com/btcsuite/btcutil/bloom" "github.com/btcsuite/btcutil/hdkeychain" "github.com/btcsuite/btcutil/txsort" @@ -41,6 +43,48 @@ func (s *SPVCon) Rebroadcast() { return } +// make utxo slices sortable -- same as txsort +type utxoSlice []Utxo + +// Sort utxos just like txins -- Len, Less, Swap +func (s utxoSlice) Len() int { return len(s) } +func (s utxoSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// outpoint sort; First input hash (reversed / rpc-style), then index. +func (s utxoSlice) Less(i, j int) bool { + // Input hashes are the same, so compare the index. + ihash := s[i].Op.Hash + jhash := s[j].Op.Hash + if ihash == jhash { + return s[i].Op.Index < s[j].Op.Index + } + // At this point, the hashes are not equal, so reverse them to + // big-endian and return the result of the comparison. + const hashSize = wire.HashSize + for b := 0; b < hashSize/2; b++ { + ihash[b], ihash[hashSize-1-b] = ihash[hashSize-1-b], ihash[b] + jhash[b], jhash[hashSize-1-b] = jhash[hashSize-1-b], jhash[b] + } + return bytes.Compare(ihash[:], jhash[:]) == -1 +} + +type SortableUtxoSlice []Utxo + +// utxoByAmts get sorted by utxo value. also put unconfirmed last +func (s SortableUtxoSlice) Len() int { return len(s) } +func (s SortableUtxoSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// height 0 means your lesser +func (s SortableUtxoSlice) Less(i, j int) bool { + if s[i].AtHeight == 0 && s[j].AtHeight > 0 { + return true + } + if s[j].AtHeight == 0 && s[i].AtHeight > 0 { + return false + } + return s[i].Value < s[j].Value +} + func (s *SPVCon) NewOutgoingTx(tx *wire.MsgTx) error { txid := tx.TxSha() // assign height of zero for txs we create @@ -53,7 +97,7 @@ func (s *SPVCon) NewOutgoingTx(tx *wire.MsgTx) error { return err } // make an inv message instead of a tx message to be polite - iv1 := wire.NewInvVect(wire.InvTypeTx, &txid) + iv1 := wire.NewInvVect(wire.InvTypeWitnessTx, &txid) invMsg := wire.NewMsgInv() err = invMsg.AddInvVect(iv1) if err != nil { @@ -63,6 +107,313 @@ func (s *SPVCon) NewOutgoingTx(tx *wire.MsgTx) error { return nil } +func (s *SPVCon) SendOne(u Utxo, adr btcutil.Address) error { + // fixed fee + fee := int64(5000) + + sendAmt := u.Value - fee + tx := wire.NewMsgTx() // make new tx + // add single output + outAdrScript, err := txscript.PayToAddrScript(adr) + if err != nil { + return err + } + // make user specified txout and add to tx + txout := wire.NewTxOut(sendAmt, outAdrScript) + tx.AddTxOut(txout) + + var prevPKs []byte + if u.IsWit { + tx.Flags = 0x01 + wa, err := btcutil.NewAddressWitnessPubKeyHash( + s.TS.Adrs[u.KeyIdx].PkhAdr.ScriptAddress(), s.TS.Param) + prevPKs, err = txscript.PayToAddrScript(wa) + if err != nil { + return err + } + } else { // otherwise generate directly + prevPKs, err = txscript.PayToAddrScript( + s.TS.Adrs[u.KeyIdx].PkhAdr) + if err != nil { + return err + } + } + + tx.AddTxIn(wire.NewTxIn(&u.Op, prevPKs, nil)) + + var sig []byte + var wit [][]byte + hCache := txscript.CalcHashCache(tx, 0, txscript.SigHashAll) + + child, err := s.TS.rootPrivKey.Child(u.KeyIdx + hdkeychain.HardenedKeyStart) + + if err != nil { + return err + } + priv, err := child.ECPrivKey() + if err != nil { + return err + } + + // This is where witness based sighash types need to happen + // sign into stash + if u.IsWit { + wit, err = txscript.WitnessScript( + tx, hCache, 0, u.Value, tx.TxIn[0].SignatureScript, + txscript.SigHashAll, priv, true) + if err != nil { + return err + } + } else { + sig, err = txscript.SignatureScript( + tx, 0, tx.TxIn[0].SignatureScript, + txscript.SigHashAll, priv, true) + if err != nil { + return err + } + } + + // swap sigs into sigScripts in txins + + if sig != nil { + tx.TxIn[0].SignatureScript = sig + } + if wit != nil { + tx.TxIn[0].Witness = wit + tx.TxIn[0].SignatureScript = nil + } + return s.NewOutgoingTx(tx) +} + +// SendCoins does send coins, but it's very rudimentary +// wit makes it into p2wpkh. Which is not yet spendable. +func (s *SPVCon) SendCoins(adrs []btcutil.Address, sendAmts []int64) error { + if len(adrs) != len(sendAmts) { + return fmt.Errorf("%d addresses and %d amounts", len(adrs), len(sendAmts)) + } + var err error + var score, totalSend, fee int64 + dustCutoff := int64(20000) // below this amount, just give to miners + satPerByte := int64(80) // satoshis per byte fee; have as arg later + rawUtxos, err := s.TS.GetAllUtxos() + if err != nil { + return err + } + var allUtxos SortableUtxoSlice + // start with utxos sorted by value. + + for _, utxo := range rawUtxos { + score += utxo.Value + allUtxos = append(allUtxos, *utxo) + } + // smallest and unconfirmed last (because it's reversed) + sort.Sort(sort.Reverse(allUtxos)) + + // sort.Reverse(allUtxos) + for _, amt := range sendAmts { + totalSend += amt + } + // important rule in bitcoin, output total > input total is invalid. + if totalSend > score { + return fmt.Errorf("trying to send %d but %d available.", + totalSend, score) + } + + tx := wire.NewMsgTx() // make new tx + // add non-change (arg) outputs + for i, adr := range adrs { + // make address script 76a914...88ac or 0014... + outAdrScript, err := txscript.PayToAddrScript(adr) + if err != nil { + return err + } + // make user specified txout and add to tx + txout := wire.NewTxOut(sendAmts[i], outAdrScript) + tx.AddTxOut(txout) + } + + // generate a utxo slice for your inputs + var ins utxoSlice + + // add utxos until we've had enough + nokori := totalSend // nokori is how much is needed on input side + for _, utxo := range allUtxos { + // skip unconfirmed. Or de-prioritize? + // if utxo.AtHeight == 0 { + // continue + // } + + // yeah, lets add this utxo! + ins = append(ins, utxo) + // as we add utxos, fill in sigscripts + // generate previous pkscripts (subscritpt?) for all utxos + // then make txins with the utxo and prevpk, and insert them into the tx + // these are all zeroed out during signing but it's an easy way to keep track + var prevPKs []byte + if utxo.IsWit { + tx.Flags = 0x01 + wa, err := btcutil.NewAddressWitnessPubKeyHash( + s.TS.Adrs[utxo.KeyIdx].PkhAdr.ScriptAddress(), s.TS.Param) + prevPKs, err = txscript.PayToAddrScript(wa) + if err != nil { + return err + } + } else { // otherwise generate directly + prevPKs, err = txscript.PayToAddrScript( + s.TS.Adrs[utxo.KeyIdx].PkhAdr) + if err != nil { + return err + } + } + tx.AddTxIn(wire.NewTxIn(&utxo.Op, prevPKs, nil)) + nokori -= utxo.Value + // if nokori is positive, don't bother checking fee yet. + if nokori < 0 { + fee = EstFee(tx, satPerByte) + if nokori < -fee { // done adding utxos: nokori below negative est. fee + break + } + } + } + + // see if there's enough left to also add a change output + + changeOld, err := s.TS.NewAdr() // change is witnessy + if err != nil { + return err + } + changeAdr, err := btcutil.NewAddressWitnessPubKeyHash( + changeOld.ScriptAddress(), s.TS.Param) + if err != nil { + return err + } + + changeScript, err := txscript.PayToAddrScript(changeAdr) + if err != nil { + return err + } + + changeOut := wire.NewTxOut(0, changeScript) + tx.AddTxOut(changeOut) + fee = EstFee(tx, satPerByte) + changeOut.Value = -(nokori + fee) + if changeOut.Value < dustCutoff { + // remove last output (change) : not worth it + tx.TxOut = tx.TxOut[:len(tx.TxOut)-1] + } + + // sort utxos on the input side. use this instead of txsort + // because we want to remember which keys are associated with which inputs + sort.Sort(ins) + + // sort tx -- this only will change txouts since inputs are already sorted + txsort.InPlaceSort(tx) + + // tx is ready for signing, + sigStash := make([][]byte, len(ins)) + witStash := make([][][]byte, len(ins)) + + // generate tx-wide hashCache for segwit stuff + // middle index number doesn't matter for sighashAll. + hCache := txscript.CalcHashCache(tx, 0, txscript.SigHashAll) + + for i, txin := range tx.TxIn { + // pick key + child, err := s.TS.rootPrivKey.Child( + ins[i].KeyIdx + hdkeychain.HardenedKeyStart) + if err != nil { + return err + } + priv, err := child.ECPrivKey() + if err != nil { + return err + } + + // This is where witness based sighash types need to happen + // sign into stash + if ins[i].IsWit { + witStash[i], err = txscript.WitnessScript( + tx, hCache, i, ins[i].Value, txin.SignatureScript, + txscript.SigHashAll, priv, true) + if err != nil { + return err + } + } else { + sigStash[i], err = txscript.SignatureScript( + tx, i, txin.SignatureScript, + txscript.SigHashAll, priv, true) + if err != nil { + return err + } + } + } + // swap sigs into sigScripts in txins + for i, txin := range tx.TxIn { + if sigStash[i] != nil { + txin.SignatureScript = sigStash[i] + } + if witStash[i] != nil { + txin.Witness = witStash[i] + txin.SignatureScript = nil + } + + } + + // fmt.Printf("tx: %s", TxToString(tx)) + // buf := bytes.NewBuffer(make([]byte, 0, tx.SerializeSize())) + + // send it out on the wire. hope it gets there. + // we should deal with rejects. Don't yet. + err = s.NewOutgoingTx(tx) + if err != nil { + return err + } + return nil +} + +// EstFee gives a fee estimate based on a tx and a sat/Byte target. +// The TX should have all outputs, including the change address already +// populated (with potentially 0 amount. Also it should have all inputs +// populated, but inputs don't need to have sigscripts or witnesses +// (it'll guess the sizes of sigs/wits that arent' filled in). +func EstFee(otx *wire.MsgTx, spB int64) int64 { + mtsig := make([]byte, 72) + mtpub := make([]byte, 33) + + tx := otx.Copy() + + // iterate through txins, replacing subscript sigscripts with noise + // sigs or witnesses + for _, txin := range tx.TxIn { + // check wpkh + if len(txin.SignatureScript) == 22 && + txin.SignatureScript[0] == 0x00 && txin.SignatureScript[1] == 0x14 { + txin.SignatureScript = nil + txin.Witness = make([][]byte, 2) + txin.Witness[0] = mtsig + txin.Witness[1] = mtpub + } else if len(txin.SignatureScript) == 34 && + txin.SignatureScript[0] == 0x00 && txin.SignatureScript[1] == 0x20 { + // p2wsh -- sig lenght is a total guess! + txin.SignatureScript = nil + txin.Witness = make([][]byte, 3) + // 3 sigs? totally guessing here + txin.Witness[0] = mtsig + txin.Witness[1] = mtsig + txin.Witness[2] = mtsig + } else { + // assume everything else is p2pkh. Even though it's not + txin.Witness = nil + txin.SignatureScript = make([]byte, 105) // len of p2pkh sigscript + } + } + fmt.Printf(TxToString(tx)) + size := int64(tx.VirtualSize()) + fmt.Printf("%d spB, est vsize %d, fee %d\n", spB, size, size*spB) + return size * spB +} + +// SignThis isn't used anymore... func (t *TxStore) SignThis(tx *wire.MsgTx) error { fmt.Printf("-= SignThis =-\n") diff --git a/uspv/txstore.go b/uspv/txstore.go index 17282bfb..acf74f9f 100644 --- a/uspv/txstore.go +++ b/uspv/txstore.go @@ -23,6 +23,8 @@ type TxStore struct { Adrs []MyAdr // endeavouring to acquire capital StateDB *bolt.DB // place to write all this down + localFilter *bloom.Filter // local bloom filter for hard mode + // Params live here, not SCon Param *chaincfg.Params // network parameters (testnet3, testnetL) @@ -38,7 +40,8 @@ type Utxo struct { // cash money. KeyIdx uint32 // index for private key needed to sign / spend Value int64 // higher is better - // IsCoinbase bool // can't spend for a while + // IsCoinbase bool // can't spend for a while + IsWit bool // true if p2wpkh output } // Stxo is a utxo that has moved on. @@ -78,7 +81,7 @@ func (t *TxStore) AddTxid(txid *wire.ShaHash, height int32) error { // ... or I'm gonna fade away func (t *TxStore) GimmeFilter() (*bloom.Filter, error) { if len(t.Adrs) == 0 { - return nil, fmt.Errorf("no addresses to filter for") + return nil, fmt.Errorf("no address to filter for") } // get all utxos to add outpoints to filter @@ -89,7 +92,10 @@ func (t *TxStore) GimmeFilter() (*bloom.Filter, error) { elem := uint32(len(t.Adrs) + len(allUtxos)) f := bloom.NewFilter(elem, 0, 0.000001, wire.BloomUpdateAll) - for _, a := range t.Adrs { + + // note there could be false positives since we're just looking + // for the 20 byte PKH without the opcodes. + for _, a := range t.Adrs { // add 20-byte pubkeyhash f.Add(a.PkhAdr.ScriptAddress()) } @@ -133,14 +139,19 @@ func CheckDoubleSpends( // TxToString prints out some info about a transaction. for testing / debugging func TxToString(tx *wire.MsgTx) string { - str := fmt.Sprintf("\t - Tx %s\n", tx.TxSha().String()) + str := fmt.Sprintf("size %d vsize %d wsize %d locktime %d flag %x txid %s\n", + tx.SerializeSize(), tx.VirtualSize(), tx.SerializeSizeWitness(), + tx.LockTime, tx.Flags, tx.TxSha().String()) for i, in := range tx.TxIn { - str += fmt.Sprintf("Input %d: %s\n", i, in.PreviousOutPoint.String()) - str += fmt.Sprintf("SigScript for input %d: %x\n", i, in.SignatureScript) + str += fmt.Sprintf("Input %d spends %s\n", i, in.PreviousOutPoint.String()) + str += fmt.Sprintf("\tSigScript: %x\n", in.SignatureScript) + for j, wit := range in.Witness { + str += fmt.Sprintf("\twitness %d: %x\n", j, wit) + } } for i, out := range tx.TxOut { if out != nil { - str += fmt.Sprintf("\toutput %d script: %x amt: %d\n", + str += fmt.Sprintf("output %d script: %x amt: %d\n", i, out.PkScript, out.Value) } else { str += fmt.Sprintf("output %d nil (WARNING)\n", i) @@ -175,6 +186,20 @@ func outPointToBytes(op *wire.OutPoint) ([]byte, error) { return buf.Bytes(), nil } +/*----- serialization for utxos ------- */ +/* Utxos serialization: +byte length desc at offset + +32 txid 0 +4 idx 32 +4 height 36 +4 keyidx 40 +8 amt 44 +1 flag 52 + +end len 53 +*/ + // ToBytes turns a Utxo into some bytes. // note that the txid is the first 36 bytes and in our use cases will be stripped // off, but is left here for other applications @@ -205,6 +230,16 @@ func (u *Utxo) ToBytes() ([]byte, error) { if err != nil { return nil, err } + // last byte indicates tx witness flags ( tx[5] from serialized tx) + // write a 1 at the end for p2wpkh (same as flags byte) + witByte := byte(0x00) + if u.IsWit { + witByte = 0x01 + } + err = buf.WriteByte(witByte) + if err != nil { + return nil, err + } return buf.Bytes(), nil } @@ -218,8 +253,8 @@ func UtxoFromBytes(b []byte) (Utxo, error) { return u, fmt.Errorf("nil input slice") } buf := bytes.NewBuffer(b) - if buf.Len() < 52 { // utxos are 52 bytes - return u, fmt.Errorf("Got %d bytes for utxo, expect 52", buf.Len()) + if buf.Len() < 53 { // utxos are 53 bytes + return u, fmt.Errorf("Got %d bytes for utxo, expect 53", buf.Len()) } // read 32 byte txid err := u.Op.Hash.SetBytes(buf.Next(32)) @@ -246,38 +281,44 @@ func UtxoFromBytes(b []byte) (Utxo, error) { if err != nil { return u, err } + // read 1 byte witness flags + witByte, err := buf.ReadByte() + if err != nil { + return u, err + } + if witByte != 0x00 { + u.IsWit = true + } + return u, nil } +/*----- serialization for stxos ------- */ +/* Stxo serialization: +byte length desc at offset + +53 utxo 0 +4 sheight 53 +32 stxid 57 + +end len 89 +*/ + // ToBytes turns an Stxo into some bytes. -// outpoint txid, outpoint idx, height, key idx, amt, spendheight, spendtxid +// prevUtxo serialization, then spendheight [4], spendtxid [32] func (s *Stxo) ToBytes() ([]byte, error) { var buf bytes.Buffer - // write 32 byte txid of the utxo - _, err := buf.Write(s.Op.Hash.Bytes()) + // first serialize the utxo part + uBytes, err := s.Utxo.ToBytes() if err != nil { return nil, err } - // write 4 byte outpoint index within the tx to spend - err = binary.Write(&buf, binary.BigEndian, s.Op.Index) - if err != nil { - return nil, err - } - // write 4 byte height of utxo - err = binary.Write(&buf, binary.BigEndian, s.AtHeight) - if err != nil { - return nil, err - } - // write 4 byte key index of utxo - err = binary.Write(&buf, binary.BigEndian, s.KeyIdx) - if err != nil { - return nil, err - } - // write 8 byte amount of money at the utxo - err = binary.Write(&buf, binary.BigEndian, s.Value) + // write that into the buffer first + _, err = buf.Write(uBytes) if err != nil { return nil, err } + // write 4 byte height where the txo was spent err = binary.Write(&buf, binary.BigEndian, s.SpendHeight) if err != nil { @@ -292,40 +333,21 @@ func (s *Stxo) ToBytes() ([]byte, error) { } // StxoFromBytes turns bytes into a Stxo. +// first take the first 53 bytes as a utxo, then the next 36 for how it's spent. func StxoFromBytes(b []byte) (Stxo, error) { var s Stxo - if b == nil { - return s, fmt.Errorf("nil input slice") + if len(b) < 89 { + return s, fmt.Errorf("Got %d bytes for stxo, expect 89", len(b)) } - buf := bytes.NewBuffer(b) - if buf.Len() < 88 { // stxos are 88 bytes - return s, fmt.Errorf("Got %d bytes for stxo, expect 88", buf.Len()) - } - // read 32 byte txid - err := s.Op.Hash.SetBytes(buf.Next(32)) - if err != nil { - return s, err - } - // read 4 byte outpoint index within the tx to spend - err = binary.Read(buf, binary.BigEndian, &s.Op.Index) - if err != nil { - return s, err - } - // read 4 byte height of utxo - err = binary.Read(buf, binary.BigEndian, &s.AtHeight) - if err != nil { - return s, err - } - // read 4 byte key index of utxo - err = binary.Read(buf, binary.BigEndian, &s.KeyIdx) - if err != nil { - return s, err - } - // read 8 byte amount of money at the utxo - err = binary.Read(buf, binary.BigEndian, &s.Value) + + u, err := UtxoFromBytes(b[:53]) if err != nil { return s, err } + s.Utxo = u // assign the utxo + + buf := bytes.NewBuffer(b[53:]) // make buffer for spend data + // read 4 byte spend height err = binary.Read(buf, binary.BigEndian, &s.SpendHeight) if err != nil { diff --git a/uspv/utxodb.go b/uspv/utxodb.go index 8550b50c..96f1115a 100644 --- a/uspv/utxodb.go +++ b/uspv/utxodb.go @@ -22,7 +22,7 @@ var ( BKTTxns = []byte("Txns") // all txs we care about, for replays BKTState = []byte("MiscState") // last state of DB // these are in the state bucket - KEYNumKeys = []byte("NumKeys") // number of keys used + KEYNumKeys = []byte("NumKeys") // number of p2pkh keys used KEYTipHeight = []byte("TipHeight") // height synced to ) @@ -82,17 +82,22 @@ func (ts *TxStore) OpenDB(filename string) error { // NewAdr creates a new, never before seen address, and increments the // DB counter as well as putting it in the ram Adrs store, and returns it -func (ts *TxStore) NewAdr() (*btcutil.AddressPubKeyHash, error) { +func (ts *TxStore) NewAdr() (btcutil.Address, error) { if ts.Param == nil { - return nil, fmt.Errorf("nil param") + return nil, fmt.Errorf("NewAdr error: nil param") } - n := uint32(len(ts.Adrs)) - priv, err := ts.rootPrivKey.Child(n + hdkeychain.HardenedKeyStart) + + priv := new(hdkeychain.ExtendedKey) + var err error + var n uint32 + var nAdr btcutil.Address + + n = uint32(len(ts.Adrs)) + priv, err = ts.rootPrivKey.Child(n + hdkeychain.HardenedKeyStart) if err != nil { return nil, err } - - newAdr, err := priv.Address(ts.Param) + nAdr, err = priv.Address(ts.Param) if err != nil { return nil, err } @@ -107,18 +112,21 @@ func (ts *TxStore) NewAdr() (*btcutil.AddressPubKeyHash, error) { // write to db file err = ts.StateDB.Update(func(btx *bolt.Tx) error { sta := btx.Bucket(BKTState) + return sta.Put(KEYNumKeys, buf.Bytes()) + }) if err != nil { return nil, err } // add in to ram. var ma MyAdr - ma.PkhAdr = newAdr + ma.PkhAdr = nAdr ma.KeyIdx = n - ts.Adrs = append(ts.Adrs, ma) - return newAdr, nil + ts.Adrs = append(ts.Adrs, ma) + ts.localFilter.Add(ma.PkhAdr.ScriptAddress()) + return nAdr, nil } // SetDBSyncHeight sets sync height of the db, indicated the latest block @@ -345,7 +353,6 @@ func (ts *TxStore) PopulateAdrs(lastKey uint32) error { func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) { var hits uint32 var err error - var spentOPs [][]byte var nUtxoBytes [][]byte // tx has been OK'd by SPV; check tx sanity @@ -358,30 +365,57 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) { // note that you can't check signatures; this is SPV. // 0 conf SPV means pretty much nothing. Anyone can say anything. + spentOPs := make([][]byte, len(tx.TxIn)) // before entering into db, serialize all inputs of the ingested tx - for _, txin := range tx.TxIn { - nOP, err := outPointToBytes(&txin.PreviousOutPoint) + for i, txin := range tx.TxIn { + spentOPs[i], err = outPointToBytes(&txin.PreviousOutPoint) if err != nil { return hits, err } - spentOPs = append(spentOPs, nOP) } - // also generate PKscripts for all addresses (maybe keep storing these?) - for _, adr := range ts.Adrs { + + // go through txouts, and then go through addresses to match + + // generate PKscripts for all addresses + wPKscripts := make([][]byte, len(ts.Adrs)) + aPKscripts := make([][]byte, len(ts.Adrs)) + + for i, _ := range ts.Adrs { // iterate through all our addresses - aPKscript, err := txscript.PayToAddrScript(adr.PkhAdr) + // convert regular address to witness address. (split adrs later) + wa, err := btcutil.NewAddressWitnessPubKeyHash( + ts.Adrs[i].PkhAdr.ScriptAddress(), ts.Param) if err != nil { return hits, err } - // iterate through all outputs of this tx - for i, out := range tx.TxOut { - if bytes.Equal(out.PkScript, aPKscript) { // new utxo for us - var newu Utxo + + wPKscripts[i], err = txscript.PayToAddrScript(wa) + if err != nil { + return hits, err + } + aPKscripts[i], err = txscript.PayToAddrScript(ts.Adrs[i].PkhAdr) + if err != nil { + return hits, err + } + } + + cachedSha := tx.TxSha() + // iterate through all outputs of this tx, see if we gain + for i, out := range tx.TxOut { + for j, ascr := range aPKscripts { + // detect p2wpkh + witBool := false + if bytes.Equal(out.PkScript, wPKscripts[j]) { + witBool = true + } + if bytes.Equal(out.PkScript, ascr) || witBool { // new utxo found + var newu Utxo // create new utxo and copy into it newu.AtHeight = height - newu.KeyIdx = adr.KeyIdx + newu.KeyIdx = ts.Adrs[j].KeyIdx newu.Value = out.Value + newu.IsWit = witBool // copy witness version from pkscript var newop wire.OutPoint - newop.Hash = tx.TxSha() + newop.Hash = cachedSha newop.Index = uint32(i) newu.Op = newop b, err := newu.ToBytes() @@ -390,7 +424,7 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) { } nUtxoBytes = append(nUtxoBytes, b) hits++ - break // only one match + break // txos can match only 1 script } } } @@ -405,53 +439,43 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) { return fmt.Errorf("error: db not initialized") } - // first see if we lose utxos // iterate through duffel bag and look for matches // this makes us lose money, which is regrettable, but we need to know. for _, nOP := range spentOPs { - duf.ForEach(func(k, v []byte) error { - if bytes.Equal(k, nOP) { // matched, we lost utxo - // do all this just to figure out value we lost - x := make([]byte, len(k)+len(v)) - copy(x, k) - copy(x[len(k):], v) - lostTxo, err := UtxoFromBytes(x) - if err != nil { - return err - } - hits++ - // then delete the utxo from duf, save to old - err = duf.Delete(k) - if err != nil { - return err - } - // after deletion, save stxo to old bucket - var st Stxo // generate spent txo - st.Utxo = lostTxo // assign outpoint - st.SpendHeight = height // spent at height - st.SpendTxid = tx.TxSha() // spent by txid - stxb, err := st.ToBytes() // serialize - if err != nil { - return err - } - err = old.Put(k, stxb) // write k:v outpoint:stxo bytes - if err != nil { - return err - } - // store this relevant tx - sha := tx.TxSha() - var buf bytes.Buffer - tx.Serialize(&buf) - err = txns.Put(sha.Bytes(), buf.Bytes()) - if err != nil { - return err - } - - return nil // matched utxo k, won't match another + v := duf.Get(nOP) + if v != nil { + hits++ + // do all this just to figure out value we lost + x := make([]byte, len(nOP)+len(v)) + copy(x, nOP) + copy(x[len(nOP):], v) + lostTxo, err := UtxoFromBytes(x) + if err != nil { + return err } - return nil // no match - }) - } // done losing utxos, next gain utxos + + // after marking for deletion, save stxo to old bucket + var st Stxo // generate spent txo + st.Utxo = lostTxo // assign outpoint + st.SpendHeight = height // spent at height + st.SpendTxid = cachedSha // spent by txid + stxb, err := st.ToBytes() // serialize + if err != nil { + return err + } + err = old.Put(nOP, stxb) // write nOP:v outpoint:stxo bytes + if err != nil { + return err + } + + err = duf.Delete(nOP) + if err != nil { + return err + } + } + } + + // done losing utxos, next gain utxos // next add all new utxos to db, this is quick as the work is above for _, ub := range nUtxoBytes { err = duf.Put(ub[:36], ub[36:]) @@ -459,6 +483,15 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) { return err } } + + // if hits is nonzero it's a relevant tx and we should store it + var buf bytes.Buffer + tx.SerializeWitness(&buf) // always store witness version + err = txns.Put(cachedSha.Bytes(), buf.Bytes()) + if err != nil { + return err + } + return nil }) return hits, err