Merge branch 'uspvdev'

.gitignore

@@ -33,6 +33,9 @@ cmd/cmd
 cmd/lncli/lncli
 cmd/lnshell/lnshell
 
+sighash143/sighash143
+sighash143/*.hex
+
 test_wal/*
 
 # vim

lnwallet/channel.go

@@ -86,7 +86,7 @@ func newLightningChannel(wallet *LightningWallet, events chainntnfs.ChainNotifie
 		return nil, err
 	}
 	_, multiSigIndex := findScriptOutputIndex(state.FundingTx, fundingPkScript)
-	lc.fundingTxIn = wire.NewTxIn(wire.NewOutPoint(&fundingTxId, multiSigIndex), nil)
+	lc.fundingTxIn = wire.NewTxIn(wire.NewOutPoint(&fundingTxId, multiSigIndex), nil, nil)
 	lc.fundingP2SH = fundingPkScript
 
 	return lc, nil

lnwallet/wallet.go

@@ -529,7 +529,7 @@ func (l *LightningWallet) handleFundingReserveRequest(req *initFundingReserveMsg
 		// Empty sig script, we'll actually sign if this reservation is
 		// queued up to be completed (the other side accepts).
 		outPoint := wire.NewOutPoint(coin.Hash(), coin.Index())
-		ourContribution.Inputs[i] = wire.NewTxIn(outPoint, nil)
+		ourContribution.Inputs[i] = wire.NewTxIn(outPoint, nil, nil)
 	}
 
 	l.coinSelectMtx.Unlock()
@@ -794,7 +794,7 @@ func (l *LightningWallet) handleContributionMsg(req *addContributionMsg) {
 	// since the outputs are cannonically sorted.
 	fundingNTxid := fundingTx.TxSha() // NOTE: assumes testnet-L
 	_, multiSigIndex := findScriptOutputIndex(fundingTx, multiSigOut.PkScript)
-	fundingTxIn := wire.NewTxIn(wire.NewOutPoint(&fundingNTxid, multiSigIndex), nil)
+	fundingTxIn := wire.NewTxIn(wire.NewOutPoint(&fundingNTxid, multiSigIndex), nil, nil)
 
 	// With the funding tx complete, create both commitment transactions.
 	initialBalance := ourContribution.FundingAmount

lnwire/lnwire.go

@@ -4,11 +4,12 @@ import (
 	"bytes"
 	"encoding/binary"
 	"fmt"
+	"io"
+	"io/ioutil"
+
 	"github.com/btcsuite/btcd/btcec"
 	"github.com/btcsuite/btcd/wire"
 	"github.com/btcsuite/btcutil"
-	"io"
-	"io/ioutil"
 )
 
 var MAX_SLICE_LENGTH = 65535
@@ -557,7 +558,7 @@ func readElement(r io.Reader, element interface{}) error {
 		var txins []*wire.TxIn
 		for i := uint8(0); i < numScripts; i++ {
 			outpoint := new(wire.OutPoint)
-			txin := wire.NewTxIn(outpoint, nil)
+			txin := wire.NewTxIn(outpoint, nil, nil)
 			err = readElement(r, &txin)
 			if err != nil {
 				return err

shell.go

@@ -2,16 +2,13 @@ package main
 
 import (
 	"bufio"
-	"bytes"
 	"fmt"
 	"log"
 	"os"
+	"sort"
 	"strconv"
 	"strings"
 
-	"github.com/btcsuite/btcd/txscript"
-	"github.com/btcsuite/btcd/wire"
-
 	"github.com/btcsuite/btcd/chaincfg"
 	"github.com/btcsuite/btcutil"
 
@@ -25,15 +22,15 @@ testing.  It can send and receive coins.
 const (
 	keyFileName    = "testkey.hex"
 	headerFileName = "headers.bin"
-	dbFileName     = "/dev/shm/utxo.db"
+	dbFileName     = "utxo.db"
 	// this is my local testnet node, replace it with your own close by.
 	// Random internet testnet nodes usually work but sometimes don't, so
 	// maybe I should test against different versions out there.
-	SPVHostAdr = "127.0.0.1:18333"
+	SPVHostAdr = "127.0.0.1:28333"
 )
 
 var (
-	Params = &chaincfg.TestNet3Params
+	Params = &chaincfg.SegNetParams
 	SCon   uspv.SPVCon // global here for now
 )
 
@@ -51,7 +48,7 @@ func shell() {
 	// setup spvCon
 
 	SCon, err = uspv.OpenSPV(
-		SPVHostAdr, headerFileName, dbFileName, &Store, false, false, Params)
+		SPVHostAdr, headerFileName, dbFileName, &Store, true, false, Params)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -61,14 +58,14 @@ func shell() {
 		log.Fatal(err)
 	}
 	if tip == 0 { // DB has never been used, set to birthday
-		tip = 675000 // hardcoded; later base on keyfile date?
+		tip = 21900 // hardcoded; later base on keyfile date?
 		err = SCon.TS.SetDBSyncHeight(tip)
 		if err != nil {
 			log.Fatal(err)
 		}
 	}
 
-	// once we're connected, initiate headers sync
+	//	 once we're connected, initiate headers sync
 	err = SCon.AskForHeaders()
 	if err != nil {
 		log.Fatal(err)
@@ -144,6 +141,20 @@ func Shellparse(cmdslice []string) error {
 		}
 		return nil
 	}
+	if cmd == "fan" {
+		err = Fan(args)
+		if err != nil {
+			fmt.Printf("fan error: %s\n", err)
+		}
+		return nil
+	}
+	if cmd == "sweep" {
+		err = Sweep(args)
+		if err != nil {
+			fmt.Printf("sweep error: %s\n", err)
+		}
+		return nil
+	}
 	if cmd == "txs" {
 		err = Txs(args)
 		if err != nil {
@@ -199,39 +210,164 @@ func Bal(args []string) error {
 		return fmt.Errorf("Can't get balance, spv connection broken")
 	}
 	fmt.Printf(" ----- Account Balance ----- \n")
-	allUtxos, err := SCon.TS.GetAllUtxos()
+	rawUtxos, err := SCon.TS.GetAllUtxos()
 	if err != nil {
 		return err
 	}
-	var score int64
+	var allUtxos uspv.SortableUtxoSlice
+	for _, utxo := range rawUtxos {
+		allUtxos = append(allUtxos, *utxo)
+	}
+	// smallest and unconfirmed last (because it's reversed)
+	sort.Sort(sort.Reverse(allUtxos))
+
+	var score, confScore int64
 	for i, u := range allUtxos {
-		fmt.Printf("\tutxo %d height %d %s key: %d amt %d\n",
+		fmt.Printf("\tutxo %d height %d %s key:%d amt %d",
 			i, u.AtHeight, u.Op.String(), u.KeyIdx, u.Value)
+		if u.IsWit {
+			fmt.Printf(" WIT")
+		}
+		fmt.Printf("\n")
 		score += u.Value
+		if u.AtHeight != 0 {
+			confScore += u.Value
+		}
 	}
 	height, _ := SCon.TS.GetDBSyncHeight()
 
+	atx, err := SCon.TS.GetAllTxs()
+
+	stxos, err := SCon.TS.GetAllStxos()
+
 	for i, a := range SCon.TS.Adrs {
-		fmt.Printf("address %d %s\n", i, a.PkhAdr.String())
+		wa, err := btcutil.NewAddressWitnessPubKeyHash(
+			a.PkhAdr.ScriptAddress(), Params)
+		if err != nil {
+			return err
+		}
+		fmt.Printf("address %d %s OR %s\n", i, a.PkhAdr.String(), wa.String())
 	}
-	fmt.Printf("Total known utxos: %d\n", len(allUtxos))
+	fmt.Printf("Total known txs: %d\n", len(atx))
-	fmt.Printf("Total spendable coin: %d\n", score)
+	fmt.Printf("Known utxos: %d\tPreviously spent txos: %d\n",
+		len(allUtxos), len(stxos))
+	fmt.Printf("Total coin: %d confirmed: %d\n", score, confScore)
 	fmt.Printf("DB sync height: %d\n", height)
 	return nil
 }
 
 // Adr makes a new address.
 func Adr(args []string) error {
+
+	// if there's an arg, make 10 adrs
+	if len(args) > 0 {
+		for i := 0; i < 10; i++ {
+			_, err := SCon.TS.NewAdr()
+			if err != nil {
+				return err
+			}
+		}
+	}
+	if len(args) > 1 {
+		for i := 0; i < 1000; i++ {
+			_, err := SCon.TS.NewAdr()
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	// always make one
 	a, err := SCon.TS.NewAdr()
 	if err != nil {
 		return err
 	}
-	fmt.Printf("made new address %s, %d addresses total\n",
+	fmt.Printf("made new address %s\n",
-		a.String(), len(SCon.TS.Adrs))
+		a.String())
 
 	return nil
 }
 
+// Sweep sends every confirmed uxto in your wallet to an address.
+// it does them all individually to there are a lot of txs generated.
+// syntax: sweep adr
+func Sweep(args []string) error {
+	if len(args) < 2 {
+		return fmt.Errorf("sweep syntax: sweep adr")
+	}
+
+	adr, err := btcutil.DecodeAddress(args[0], SCon.TS.Param)
+	if err != nil {
+		fmt.Printf("error parsing %s as address\t", args[0])
+		return err
+	}
+	numTxs, err := strconv.ParseInt(args[1], 10, 64)
+	if err != nil {
+		return err
+	}
+	if numTxs < 1 {
+		return fmt.Errorf("can't send %d txs", numTxs)
+	}
+
+	rawUtxos, err := SCon.TS.GetAllUtxos()
+	if err != nil {
+		return err
+	}
+	var allUtxos uspv.SortableUtxoSlice
+	for _, utxo := range rawUtxos {
+		allUtxos = append(allUtxos, *utxo)
+	}
+	// smallest and unconfirmed last (because it's reversed)
+	sort.Sort(sort.Reverse(allUtxos))
+
+	for i, u := range allUtxos {
+		if u.AtHeight != 0 {
+			err = SCon.SendOne(allUtxos[i], adr)
+			if err != nil {
+				return err
+			}
+			numTxs--
+			if numTxs == 0 {
+				return nil
+			}
+		}
+	}
+
+	fmt.Printf("spent all confirmed utxos; not enough by %d\n", numTxs)
+
+	return nil
+}
+
+// Fan generates a bunch of fanout.  Only for testing, can be expensive.
+// syntax: fan adr numOutputs valOutputs witty
+func Fan(args []string) error {
+	if len(args) < 3 {
+		return fmt.Errorf("fan syntax: fan adr numOutputs valOutputs")
+	}
+	adr, err := btcutil.DecodeAddress(args[0], SCon.TS.Param)
+	if err != nil {
+		fmt.Printf("error parsing %s as address\t", args[0])
+		return err
+	}
+	numOutputs, err := strconv.ParseInt(args[1], 10, 64)
+	if err != nil {
+		return err
+	}
+	valOutputs, err := strconv.ParseInt(args[2], 10, 64)
+	if err != nil {
+		return err
+	}
+
+	adrs := make([]btcutil.Address, numOutputs)
+	amts := make([]int64, numOutputs)
+
+	for i := int64(0); i < numOutputs; i++ {
+		adrs[i] = adr
+		amts[i] = valOutputs + i
+	}
+	return SCon.SendCoins(adrs, amts)
+}
+
 // Send sends coins.
 func Send(args []string) error {
 	if SCon.RBytes == 0 {
@@ -254,102 +390,30 @@ func Send(args []string) error {
 	}
 	// need args, fail
 	if len(args) < 2 {
-		return fmt.Errorf("need args: ssend amount(satoshis) address")
+		return fmt.Errorf("need args: ssend address amount(satoshis) wit?")
 	}
-	amt, err := strconv.ParseInt(args[0], 10, 64)
+	adr, err := btcutil.DecodeAddress(args[0], SCon.TS.Param)
+	if err != nil {
+		fmt.Printf("error parsing %s as address\t", args[0])
+		return err
+	}
+	amt, err := strconv.ParseInt(args[1], 10, 64)
 	if err != nil {
 		return err
 	}
 	if amt < 1000 {
 		return fmt.Errorf("can't send %d, too small", amt)
 	}
-	adr, err := btcutil.DecodeAddress(args[1], SCon.TS.Param)
+
-	if err != nil {
-		fmt.Printf("error parsing %s as address\t", args[1])
-		return err
-	}
 	fmt.Printf("send %d to address: %s \n",
 		amt, adr.String())
-	err = SendCoins(SCon, adr, amt)
-	if err != nil {
-		return err
-	}
-	return nil
-}
 
-// SendCoins does send coins, but it's very rudimentary
+	var adrs []btcutil.Address
-func SendCoins(s uspv.SPVCon, adr btcutil.Address, sendAmt int64) error {
+	var amts []int64
-	var err error
-	var score int64
-	allUtxos, err := s.TS.GetAllUtxos()
-	if err != nil {
-		return err
-	}
 
-	for _, utxo := range allUtxos {
+	adrs = append(adrs, adr)
-		score += utxo.Value
+	amts = append(amts, amt)
-	}
+	err = SCon.SendCoins(adrs, amts)
-	// important rule in bitcoin, output total > input total is invalid.
-	if sendAmt > score {
-		return fmt.Errorf("trying to send %d but %d available.",
-			sendAmt, score)
-	}
-
-	tx := wire.NewMsgTx() // make new tx
-	// make address script 76a914...88ac
-	adrScript, err := txscript.PayToAddrScript(adr)
-	if err != nil {
-		return err
-	}
-	// make user specified txout and add to tx
-	txout := wire.NewTxOut(sendAmt, adrScript)
-	tx.AddTxOut(txout)
-
-	nokori := sendAmt // nokori is how much is needed on input side
-	for _, utxo := range allUtxos {
-		// generate pkscript to sign
-		prevPKscript, err := txscript.PayToAddrScript(
-			s.TS.Adrs[utxo.KeyIdx].PkhAdr)
-		if err != nil {
-			return err
-		}
-		// make new input from this utxo
-		thisInput := wire.NewTxIn(&utxo.Op, prevPKscript)
-		tx.AddTxIn(thisInput)
-		nokori -= utxo.Value
-		if nokori < -10000 { // minimum overage / fee is 1K now
-			break
-		}
-	}
-	// there's enough left to make a change output
-	if nokori < -200000 {
-		change, err := s.TS.NewAdr()
-		if err != nil {
-			return err
-		}
-
-		changeScript, err := txscript.PayToAddrScript(change)
-		if err != nil {
-			return err
-		}
-		changeOut := wire.NewTxOut((-100000)-nokori, changeScript)
-		tx.AddTxOut(changeOut)
-	}
-
-	// use txstore method to sign
-	err = s.TS.SignThis(tx)
-	if err != nil {
-		return err
-	}
-
-	fmt.Printf("tx: %s", uspv.TxToString(tx))
-	buf := bytes.NewBuffer(make([]byte, 0, tx.SerializeSize()))
-	tx.Serialize(buf)
-	fmt.Printf("tx: %x\n", buf.Bytes())
-
-	// send it out on the wire.  hope it gets there.
-	// we should deal with rejects.  Don't yet.
-	err = s.NewOutgoingTx(tx)
 	if err != nil {
 		return err
 	}

sighash143/sighash143.go

@@ -0,0 +1,317 @@
+package main
+
+import (
+	"bytes"
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"math"
+	"strings"
+
+	"github.com/btcsuite/btcd/btcec"
+	"github.com/btcsuite/btcd/txscript"
+	"github.com/btcsuite/btcd/wire"
+	"github.com/lightningnetwork/lnd/uspv"
+)
+
+const (
+	inspk0 = "2103c9f4836b9a4f77fc0d81f7bcb01b7f1b35916864b9476c241ce9fc198bd25432ac"
+	inamt0 = int64(625000000)
+
+	inspk1 = "00141d0f172a0ecb48aee1be1f2687d2963ae33f71a1"
+	inamt1 = int64(600000000)
+
+	xpecthash = "c37af31116d1b27caf68aae9e3ac82f1477929014d5b917657d0eb49478cb670"
+)
+
+// calcWitnessSignatureHash is the witnessified version of calcSignatureHash
+// put your pkscripts in the sigscript slot before handing the tx to this
+// function.  Also you're clearly supposed to cache the 3 sub-hashes generated
+// here, because they apply to the tx, not a txin.  But this doesn't yet.
+func calcWitnessSignatureHash(sigscript []byte,
+	hashType txscript.SigHashType, tx *wire.MsgTx, idx int, amt int64) []byte {
+	// in the script.go calcSignatureHash(), idx is assumed safe, so I guess
+	// that's OK here too...?  Nah I'm gonna check
+	if idx > len(tx.TxIn)-1 {
+		fmt.Printf("calcWitnessSignatureHash error: idx %d but %d txins",
+			idx, len(tx.TxIn))
+		return nil
+	}
+
+	// first get hashPrevOuts, hashSequence, and hashOutputs
+	hashPrevOuts := calcHashPrevOuts(tx, hashType)
+	hashSequence := calcHashSequence(tx, hashType)
+	hashOutputs := calcHashOutputs(tx, idx, hashType)
+
+	var buf4 [4]byte // buffer for 4-byte stuff
+	var buf8 [8]byte // buffer for 8-byte stuff
+	var pre []byte   // the pre-image we're generating
+
+	binary.LittleEndian.PutUint32(buf4[:], uint32(tx.Version))
+	pre = append(pre, buf4[:]...)
+
+	pre = append(pre, hashPrevOuts.Bytes()...)
+	pre = append(pre, hashSequence.Bytes()...)
+
+	// outpoint being spent
+	pre = append(pre, tx.TxIn[idx].PreviousOutPoint.Hash.Bytes()...)
+	binary.LittleEndian.PutUint32(buf4[:], tx.TxIn[idx].PreviousOutPoint.Index)
+	pre = append(pre, buf4[:]...)
+
+	// scriptCode which is some new thing
+
+	// detect wpkh mode
+	fmt.Printf("txin %d sscript len %d\n", idx, len(tx.TxIn[idx].SignatureScript))
+	if len(sigscript) == 22 && sigscript[0] == 0x00 && sigscript[1] == 0x14 {
+		// wpkh mode .... recreate op_dup codes here
+		sCode := []byte{0x19, 0x76, 0xa9, 0x14}
+		sCode = append(sCode, sigscript[2:22]...)
+		sCode = append(sCode, []byte{0x88, 0xac}...)
+		pre = append(pre, sCode...)
+	} else if len(sigscript) == 34 &&
+		sigscript[0] == 0x00 && sigscript[1] == 0x20 {
+		// whs mode- need to remove codeseparators.  this doesn't yet.
+		var buf bytes.Buffer
+		writeVarBytes(&buf, 0, sigscript)
+		pre = append(pre, buf.Bytes()...)
+	} else {
+		// ?? this is not witness tx! fail
+		fmt.Printf("Non witness error ")
+		return nil
+	}
+
+	// amount being signed off
+	binary.LittleEndian.PutUint64(buf8[:], uint64(amt))
+	pre = append(pre, buf8[:]...)
+
+	// nsequence of input
+	binary.LittleEndian.PutUint32(buf4[:], tx.TxIn[idx].Sequence)
+	pre = append(pre, buf4[:]...)
+
+	pre = append(pre, hashOutputs.Bytes()...)
+
+	// locktime
+	binary.LittleEndian.PutUint32(buf4[:], tx.LockTime)
+	pre = append(pre, buf4[:]...)
+
+	// hashType... in 4 bytes, instead of 1, because reasons.
+	binary.LittleEndian.PutUint32(buf4[:], uint32(hashType))
+	pre = append(pre, buf4[:]...)
+
+	fmt.Printf("calcWitnessSignatureHash pre: %x\n", pre)
+	hsh := wire.DoubleSha256SH(pre)
+	return hsh.Bytes()
+}
+
+// calcHashPrevOuts makes a single hash of all previous outputs in the tx
+func calcHashPrevOuts(tx *wire.MsgTx, hType txscript.SigHashType) wire.ShaHash {
+	// skip this (0x00) for anyonecanpay
+	if hType == txscript.SigHashAnyOneCanPay {
+		var empty [32]byte
+		return empty
+	}
+
+	var pre []byte
+	for _, in := range tx.TxIn {
+		// first append 32 byte hash
+		pre = append(pre, in.PreviousOutPoint.Hash.Bytes()...)
+		// then make a buffer, put 4 byte index in lil' endian and append that
+		var buf [4]byte
+		binary.LittleEndian.PutUint32(buf[:], in.PreviousOutPoint.Index)
+		pre = append(pre, buf[:]...)
+	}
+	fmt.Printf("calcHashPrevOuts pre: %x\n", pre)
+	return wire.DoubleSha256SH(pre)
+}
+
+// calcHashSequence is hash of txins' seq numbers, lil' endian, stuck together
+func calcHashSequence(tx *wire.MsgTx, hType txscript.SigHashType) wire.ShaHash {
+	// skip (0x00) for single, none, anyonecanpay
+	if hType == txscript.SigHashSingle || hType == txscript.SigHashNone ||
+		hType == txscript.SigHashAnyOneCanPay {
+		var empty [32]byte
+		return empty
+	}
+
+	var pre []byte
+	for _, in := range tx.TxIn {
+		var buf [4]byte
+		binary.LittleEndian.PutUint32(buf[:], in.Sequence)
+		pre = append(pre, buf[:]...)
+	}
+	fmt.Printf("calcHashSequence pre: %x\n", pre)
+	return wire.DoubleSha256SH(pre)
+}
+
+// calcHashOutputs also wants a input index, which it only uses for
+// sighash single.  If it's not sighash single, just put a 0 or whatever.
+func calcHashOutputs(
+	tx *wire.MsgTx, inIndex int, hType txscript.SigHashType) wire.ShaHash {
+	if hType == txscript.SigHashNone ||
+		(hType == txscript.SigHashSingle && inIndex <= len(tx.TxOut)) {
+		var empty [32]byte
+		return empty
+	}
+	if hType == txscript.SigHashSingle {
+		var buf bytes.Buffer
+		writeTxOut(&buf, 0, 0, tx.TxOut[inIndex])
+		return wire.DoubleSha256SH(buf.Bytes())
+	}
+
+	var pre []byte
+	for _, out := range tx.TxOut {
+		var buf bytes.Buffer
+		writeTxOut(&buf, 0, 0, out)
+		pre = append(pre, buf.Bytes()...)
+	}
+	fmt.Printf("calcHashOutputs pre: %x\n", pre)
+	return wire.DoubleSha256SH(pre)
+}
+
+func main() {
+	fmt.Printf("sighash 143\n")
+
+	// get previous pkscripts for inputs 0 and 1 from hex
+	in0spk, err := hex.DecodeString(string(inspk0))
+	if err != nil {
+		log.Fatal(err)
+	}
+	in1spk, err := hex.DecodeString(string(inspk1))
+	if err != nil {
+		log.Fatal(err)
+	}
+	xpkt, err := hex.DecodeString(string(xpecthash))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// load tx skeleton from local file
+	fmt.Printf("loading tx from file tx.hex\n")
+	txhex, err := ioutil.ReadFile("tx.hex")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	txhex = []byte(strings.TrimSpace(string(txhex)))
+	txbytes, err := hex.DecodeString(string(txhex))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Printf("loaded %d byte tx %x\n", len(txbytes), txbytes)
+
+	// 	make tx
+	ttx := wire.NewMsgTx()
+	// deserialize into tx
+	buf := bytes.NewBuffer(txbytes)
+	ttx.Deserialize(buf)
+
+	fmt.Printf("=====tx locktime: %x\n", ttx.LockTime)
+	ttx.TxIn[0].SignatureScript = in0spk
+	//	ttx.TxIn[1].SignatureScript = in1spk
+
+	fmt.Printf(uspv.TxToString(ttx))
+
+	priv, err := btcec.NewPrivateKey(btcec.S256())
+	if err != nil {
+		log.Fatal(err)
+	}
+	// assert flag before writing witness
+	ttx.Flags = 0x01
+	ttx.TxIn[1].Witness, err = txscript.WitnessScript(ttx, 1, inamt1, in1spk,
+		txscript.SigHashAll, priv, true)
+	//	ttx.TxIn[1].SignatureScript = nil
+
+	hxh := calcWitnessSignatureHash(in1spk, txscript.SigHashAll, ttx, 1, inamt1)
+
+	fmt.Printf("got sigHash %x NON LIB\n", hxh)
+	fmt.Printf("expect hash %x\n ", xpkt)
+	fmt.Printf("\n%s\n", uspv.TxToString(ttx))
+
+	fmt.Printf("loading tx from file xtx.hex\n")
+	txhex, err = ioutil.ReadFile("xtx.hex")
+	if err != nil {
+		log.Fatal(err)
+	}
+	txhex = []byte(strings.TrimSpace(string(txhex)))
+	txbytes, err = hex.DecodeString(string(txhex))
+	if err != nil {
+		log.Fatal(err)
+	}
+	// 	make tx
+	xtx := wire.NewMsgTx()
+	// deserialize into tx
+	buf = bytes.NewBuffer(txbytes)
+	xtx.Deserialize(buf)
+
+	//	ttx.Flags = 0x01
+	fmt.Printf("\n%s\n", uspv.TxToString(xtx))
+}
+
+// pver can be 0, doesn't do anything in these.  Same for msg.Version
+// writeVarInt serializes val to w using a variable number of bytes depending
+// on its value.
+func writeVarInt(w io.Writer, pver uint32, val uint64) error {
+	if val < 0xfd {
+		_, err := w.Write([]byte{uint8(val)})
+		return err
+	}
+
+	if val <= math.MaxUint16 {
+		var buf [3]byte
+		buf[0] = 0xfd
+		binary.LittleEndian.PutUint16(buf[1:], uint16(val))
+		_, err := w.Write(buf[:])
+		return err
+	}
+
+	if val <= math.MaxUint32 {
+		var buf [5]byte
+		buf[0] = 0xfe
+		binary.LittleEndian.PutUint32(buf[1:], uint32(val))
+		_, err := w.Write(buf[:])
+		return err
+	}
+
+	var buf [9]byte
+	buf[0] = 0xff
+	binary.LittleEndian.PutUint64(buf[1:], val)
+	_, err := w.Write(buf[:])
+	return err
+}
+
+// writeVarBytes serializes a variable length byte array to w as a varInt
+// containing the number of bytes, followed by the bytes themselves.
+func writeVarBytes(w io.Writer, pver uint32, bytes []byte) error {
+	slen := uint64(len(bytes))
+	err := writeVarInt(w, pver, slen)
+	if err != nil {
+		return err
+	}
+
+	_, err = w.Write(bytes)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// writeTxOut encodes to into the bitcoin protocol encoding for a transaction
+// output (TxOut) to w.
+func writeTxOut(w io.Writer, pver uint32, version int32, to *wire.TxOut) error {
+	var buf [8]byte
+	binary.LittleEndian.PutUint64(buf[:], uint64(to.Value))
+	_, err := w.Write(buf[:])
+	if err != nil {
+		return err
+	}
+
+	err = writeVarBytes(w, pver, to.PkScript)
+	if err != nil {
+		return err
+	}
+	return nil
+}

uspv/eight333.go

@@ -15,7 +15,8 @@ const (
 	headerFileName = "headers.bin"
 
 	// version hardcoded for now, probably ok...?
-	VERSION = 70011
+	// 70012 is for segnet... make this a init var?
+	VERSION = 70012
 )
 
 type SPVCon struct {
@@ -115,7 +116,10 @@ func (s *SPVCon) IngestMerkleBlock(m *wire.MsgMerkleBlock) {
 	// into our SPV header file
 	newMerkBlockSha := m.Header.BlockSha()
 	if !hah.blockhash.IsEqual(&newMerkBlockSha) {
-		log.Printf("merkle block out of order error")
+		log.Printf("merkle block out of order got %s expect %s",
+			m.Header.BlockSha().String(), hah.blockhash.String())
+		log.Printf("has %d hashes %d txs flags: %x",
+			len(m.Hashes), m.Transactions, m.Flags)
 		return
 	}
 
@@ -280,6 +284,40 @@ func (s *SPVCon) AskForHeaders() error {
 	return nil
 }
 
+// AskForOneBlock is for testing only, so you can ask for a specific block height
+// and see what goes wrong
+func (s *SPVCon) AskForOneBlock(h int32) error {
+	var hdr wire.BlockHeader
+	var err error
+
+	dbTip := int32(h)
+	s.headerMutex.Lock() // seek to header we need
+	_, err = s.headerFile.Seek(int64((dbTip)*80), os.SEEK_SET)
+	if err != nil {
+		return err
+	}
+	err = hdr.Deserialize(s.headerFile) // read header, done w/ file for now
+	s.headerMutex.Unlock()              // unlock after reading 1 header
+	if err != nil {
+		log.Printf("header deserialize error!\n")
+		return err
+	}
+
+	bHash := hdr.BlockSha()
+	// create inventory we're asking for
+	iv1 := wire.NewInvVect(wire.InvTypeWitnessBlock, &bHash)
+	gdataMsg := wire.NewMsgGetData()
+	// add inventory
+	err = gdataMsg.AddInvVect(iv1)
+	if err != nil {
+		return err
+	}
+	hah := NewRootAndHeight(bHash, h)
+	s.outMsgQueue <- gdataMsg
+	s.blockQueue <- hah // push height and mroot of requested block on queue
+	return nil
+}
+
 // AskForMerkBlocks requests blocks from current to last
 // right now this asks for 1 block per getData message.
 // Maybe it's faster to ask for many in a each message?
@@ -311,7 +349,7 @@ func (s *SPVCon) AskForBlocks() error {
 		return nil
 	}
 
-	fmt.Printf("will request merkleblocks %d to %d\n", dbTip, headerTip)
+	fmt.Printf("will request blocks %d to %d\n", dbTip+1, headerTip)
 
 	if !s.HardMode { // don't send this in hardmode! that's the whole point
 		// create initial filter
@@ -324,11 +362,12 @@ func (s *SPVCon) AskForBlocks() error {
 		fmt.Printf("sent filter %x\n", filt.MsgFilterLoad().Filter)
 	}
 	// loop through all heights where we want merkleblocks.
-	for dbTip <= headerTip {
+	for dbTip < headerTip {
-		// load header from file
+		dbTip++ // we're requesting the next header
 
+		// load header from file
 		s.headerMutex.Lock() // seek to header we need
-		_, err = s.headerFile.Seek(int64((dbTip-1)*80), os.SEEK_SET)
+		_, err = s.headerFile.Seek(int64((dbTip)*80), os.SEEK_SET)
 		if err != nil {
 			return err
 		}
@@ -344,9 +383,9 @@ func (s *SPVCon) AskForBlocks() error {
 		iv1 := new(wire.InvVect)
 		// if hardmode, ask for legit blocks, none of this ralphy stuff
 		if s.HardMode {
-			iv1 = wire.NewInvVect(wire.InvTypeBlock, &bHash)
+			iv1 = wire.NewInvVect(wire.InvTypeWitnessBlock, &bHash)
 		} else { // ah well
-			iv1 = wire.NewInvVect(wire.InvTypeFilteredBlock, &bHash)
+			iv1 = wire.NewInvVect(wire.InvTypeFilteredWitnessBlock, &bHash)
 		}
 		gdataMsg := wire.NewMsgGetData()
 		// add inventory
@@ -354,14 +393,14 @@ func (s *SPVCon) AskForBlocks() error {
 		if err != nil {
 			return err
 		}
+
 		hah := NewRootAndHeight(hdr.BlockSha(), dbTip)
 		if dbTip == headerTip { // if this is the last block, indicate finality
 			hah.final = true
 		}
-		s.outMsgQueue <- gdataMsg
 		// waits here most of the time for the queue to empty out
 		s.blockQueue <- hah // push height and mroot of requested block on queue
-		dbTip++
+		s.outMsgQueue <- gdataMsg
 	}
 	return nil
 }

uspv/hardmode.go

@@ -1,38 +1,156 @@
 package uspv
 
 import (
+	"bytes"
 	"fmt"
 	"log"
-	"os"
-	"sync"
 
 	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
+	"github.com/btcsuite/btcutil/bloom"
 )
 
-// BlockRootOK checks that all the txs in the block match the merkle root.
+var (
-// Only checks merkle root; it doesn't look at txs themselves.
+	WitMagicBytes = []byte{0x6a, 0x24, 0xaa, 0x21, 0xa9, 0xed}
-func BlockRootOK(blk wire.MsgBlock) bool {
+)
-	var shas []*wire.ShaHash
+
-	for _, tx := range blk.Transactions { // make slice of txids
+// BlockRootOK checks for block self-consistency.
-		nSha := tx.TxSha()
+// If the block has no wintess txs, and no coinbase witness commitment,
-		shas = append(shas, &nSha)
+// it only checks the tx merkle root.  If either a witness commitment or
+// any witnesses are detected, it also checks that as well.
+// Returns false if anything goes wrong, true if everything is fine.
+func BlockOK(blk wire.MsgBlock) bool {
+	var txids, wtxids []*wire.ShaHash // txids and wtxids
+	// witMode true if any tx has a wintess OR coinbase has wit commit
+	var witMode bool
+
+	for _, tx := range blk.Transactions { // make slice of (w)/txids
+		txid := tx.TxSha()
+		wtxid := tx.WTxSha()
+		if !witMode && !txid.IsEqual(&wtxid) {
+			witMode = true
+		}
+		txids = append(txids, &txid)
+		wtxids = append(wtxids, &wtxid)
 	}
-	neededLen := int(nextPowerOfTwo(uint32(len(shas)))) // kindof ugly
+
-	for len(shas) < neededLen {
+	var commitBytes []byte
-		shas = append(shas, nil) // pad out tx slice to get the full tree base
+	// try to extract coinbase witness commitment (even if !witMode)
+	cb := blk.Transactions[0]                 // get coinbase tx
+	for i := len(cb.TxOut) - 1; i >= 0; i-- { // start at the last txout
+		if bytes.HasPrefix(cb.TxOut[i].PkScript, WitMagicBytes) &&
+			len(cb.TxOut[i].PkScript) > 37 {
+			// 38 bytes or more, and starts with WitMagicBytes is a hit
+			commitBytes = cb.TxOut[i].PkScript[6:38]
+			witMode = true // it there is a wit commit it must be valid
+		}
 	}
-	for len(shas) > 1 { // calculate merkle root. Terse, eh?
+
-		shas = append(shas[2:], MakeMerkleParent(shas[0], shas[1]))
+	if witMode { // witmode, so check witness tree
-	} // auto recognizes coinbase-only blocks
+		// first find ways witMode can be disqualified
-	return blk.Header.MerkleRoot.IsEqual(shas[0])
+		if len(commitBytes) != 32 {
+			// witness in block but didn't find a wintess commitment; fail
+			log.Printf("block %s has witness but no witcommit",
+				blk.BlockSha().String())
+			return false
+		}
+		if len(cb.TxIn) != 1 {
+			log.Printf("block %s coinbase tx has %d txins (must be 1)",
+				blk.BlockSha().String(), len(cb.TxIn))
+			return false
+		}
+		if len(cb.TxIn[0].Witness) != 1 {
+			log.Printf("block %s coinbase has %d witnesses (must be 1)",
+				blk.BlockSha().String(), len(cb.TxIn[0].Witness))
+			return false
+		}
+
+		if len(cb.TxIn[0].Witness[0]) != 32 {
+			log.Printf("block %s coinbase has %d byte witness nonce (not 32)",
+				blk.BlockSha().String(), len(cb.TxIn[0].Witness[0]))
+			return false
+		}
+		// witness nonce is the cb's witness, subject to above constraints
+		witNonce, err := wire.NewShaHash(cb.TxIn[0].Witness[0])
+		if err != nil {
+			log.Printf("Witness nonce error: %s", err.Error())
+			return false // not sure why that'd happen but fail
+		}
+
+		var empty [32]byte
+		wtxids[0].SetBytes(empty[:]) // coinbase wtxid is 0x00...00
+
+		// witness root calculated from wtixds
+		witRoot := calcRoot(wtxids)
+
+		calcWitCommit := wire.DoubleSha256SH(
+			append(witRoot.Bytes(), witNonce.Bytes()...))
+
+		// witness root given in coinbase op_return
+		givenWitCommit, err := wire.NewShaHash(commitBytes)
+		if err != nil {
+			log.Printf("Witness root error: %s", err.Error())
+			return false // not sure why that'd happen but fail
+		}
+		// they should be the same.  If not, fail.
+		if !calcWitCommit.IsEqual(givenWitCommit) {
+			log.Printf("Block %s witRoot error: calc %s given %s",
+				blk.BlockSha().String(),
+				calcWitCommit.String(), givenWitCommit.String())
+			return false
+		}
+	}
+
+	// got through witMode check so that should be OK;
+	// check regular txid merkleroot.  Which is, like, trivial.
+	return blk.Header.MerkleRoot.IsEqual(calcRoot(txids))
+}
+
+// calcRoot calculates the merkle root of a slice of hashes.
+func calcRoot(hashes []*wire.ShaHash) *wire.ShaHash {
+	for len(hashes) < int(nextPowerOfTwo(uint32(len(hashes)))) {
+		hashes = append(hashes, nil) // pad out hash slice to get the full base
+	}
+	for len(hashes) > 1 { // calculate merkle root. Terse, eh?
+		hashes = append(hashes[2:], MakeMerkleParent(hashes[0], hashes[1]))
+	}
+	return hashes[0]
+}
+
+func (ts *TxStore) Refilter() error {
+	allUtxos, err := ts.GetAllUtxos()
+	if err != nil {
+		return err
+	}
+	filterElements := uint32(len(allUtxos) + len(ts.Adrs))
+
+	ts.localFilter = bloom.NewFilter(filterElements, 0, 0, wire.BloomUpdateAll)
+
+	for _, u := range allUtxos {
+		ts.localFilter.AddOutPoint(&u.Op)
+	}
+	for _, a := range ts.Adrs {
+		ts.localFilter.Add(a.PkhAdr.ScriptAddress())
+	}
+
+	msg := ts.localFilter.MsgFilterLoad()
+	fmt.Printf("made %d element filter: %x\n", filterElements, msg.Filter)
+	return nil
 }
 
 // IngestBlock is like IngestMerkleBlock but aralphic
 // different enough that it's better to have 2 separate functions
 func (s *SPVCon) IngestBlock(m *wire.MsgBlock) {
 	var err error
-
+	//	var buf bytes.Buffer
-	ok := BlockRootOK(*m) // check block self-consistency
+	//	m.SerializeWitness(&buf)
+	//	fmt.Printf("block hex %x\n", buf.Bytes())
+	//	for _, tx := range m.Transactions {
+	//		fmt.Printf("wtxid: %s\n", tx.WTxSha())
+	//		fmt.Printf(" txid: %s\n", tx.TxSha())
+	//		fmt.Printf("%d %s", i, TxToString(tx))
+	//	}
+	ok := BlockOK(*m) // check block self-consistency
 	if !ok {
 		fmt.Printf("block %s not OK!!11\n", m.BlockSha().String())
 		return
@@ -53,28 +171,39 @@ func (s *SPVCon) IngestBlock(m *wire.MsgBlock) {
 		return
 	}
 
-	// iterate through all txs in the block, looking for matches.
+	fPositive := 0 // local filter false positives
-	// this is slow and can be sped up by doing in-ram filters client side.
+	reFilter := 10 // after that many false positives, regenerate filter.
-	// kindof a pain to implement though and it's fast enough for now.
+	// 10?  Making it up.  False positives have disk i/o cost, and regenning
-	var wg sync.WaitGroup
+	// the filter also has costs.  With a large local filter, false positives
-	wg.Add(len(m.Transactions))
+	// should be rare.
-	for i, tx := range m.Transactions {
 
-		go func() {
+	// iterate through all txs in the block, looking for matches.
+	// use a local bloom filter to ignore txs that don't affect us
+	for _, tx := range m.Transactions {
+		utilTx := btcutil.NewTx(tx)
+		if s.TS.localFilter.MatchTxAndUpdate(utilTx) {
 			hits, err := s.TS.Ingest(tx, hah.height)
 			if err != nil {
 				log.Printf("Incoming Tx error: %s\n", err.Error())
 				return
 			}
 			if hits > 0 {
-				log.Printf("block %d tx %d %s ingested and matches %d utxo/adrs.",
+				// log.Printf("block %d tx %d %s ingested and matches %d utxo/adrs.",
-					hah.height, i, tx.TxSha().String(), hits)
+				//	hah.height, i, tx.TxSha().String(), hits)
+			} else {
+				fPositive++ // matched filter but no hits
 			}
-			wg.Done()
+		}
-		}()
 	}
-	wg.Wait()
 
+	if fPositive > reFilter {
+		fmt.Printf("%d filter false positives in this block\n", fPositive)
+		err = s.TS.Refilter()
+		if err != nil {
+			log.Printf("Refilter error: %s\n", err.Error())
+			return
+		}
+	}
 	// write to db that we've sync'd to the height indicated in the
 	// merkle block.  This isn't QUITE true since we haven't actually gotten
 	// the txs yet but if there are problems with the txs we should backtrack.
@@ -83,6 +212,7 @@ func (s *SPVCon) IngestBlock(m *wire.MsgBlock) {
 		log.Printf("full block sync error: %s\n", err.Error())
 		return
 	}
+
 	fmt.Printf("ingested full block %s height %d OK\n",
 		m.Header.BlockSha().String(), hah.height)
 
@@ -99,35 +229,3 @@ func (s *SPVCon) IngestBlock(m *wire.MsgBlock) {
 	}
 	return
 }
-
-func (s *SPVCon) AskForOneBlock(h int32) error {
-	var hdr wire.BlockHeader
-	var err error
-
-	dbTip := int32(h)
-	s.headerMutex.Lock() // seek to header we need
-	_, err = s.headerFile.Seek(int64((dbTip)*80), os.SEEK_SET)
-	if err != nil {
-		return err
-	}
-	err = hdr.Deserialize(s.headerFile) // read header, done w/ file for now
-	s.headerMutex.Unlock()              // unlock after reading 1 header
-	if err != nil {
-		log.Printf("header deserialize error!\n")
-		return err
-	}
-
-	bHash := hdr.BlockSha()
-	// create inventory we're asking for
-	iv1 := wire.NewInvVect(wire.InvTypeBlock, &bHash)
-	gdataMsg := wire.NewMsgGetData()
-	// add inventory
-	err = gdataMsg.AddInvVect(iv1)
-	if err != nil {
-		return err
-	}
-
-	s.outMsgQueue <- gdataMsg
-
-	return nil
-}

uspv/init.go

@@ -49,6 +49,8 @@ func OpenSPV(remoteNode string, hfn, dbfn string,
 	}
 	// must set this to enable SPV stuff
 	myMsgVer.AddService(wire.SFNodeBloom)
+	// set this to enable segWit
+	myMsgVer.AddService(wire.SFNodeWitness)
 	// this actually sends
 	n, err := wire.WriteMessageN(s.con, myMsgVer, s.localVersion, s.TS.Param.Net)
 	if err != nil {
@@ -89,6 +91,13 @@ func OpenSPV(remoteNode string, hfn, dbfn string,
 	s.inWaitState = make(chan bool, 1)
 	go s.fPositiveHandler()
 
+	if hard {
+		err = s.TS.Refilter()
+		if err != nil {
+			return s, err
+		}
+	}
+
 	return s, nil
 }
 

uspv/msghandler.go

@@ -5,6 +5,7 @@ import (
 	"log"
 
 	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
 )
 
 func (s *SPVCon) incomingMessageHandler() {
@@ -88,10 +89,18 @@ func (s *SPVCon) fPositiveHandler() {
 			// send filter
 			s.SendFilter(filt)
 			fmt.Printf("sent filter %x\n", filt.MsgFilterLoad().Filter)
+
 			// clear the channel
-			for len(s.fPositives) != 0 {
+		finClear:
-				fpAccumulator += <-s.fPositives
+			for {
+				select {
+				case x := <-s.fPositives:
+					fpAccumulator += x
+				default:
+					break finClear
+				}
 			}
+
 			fmt.Printf("reset %d false positives\n", fpAccumulator)
 			// reset accumulator
 			fpAccumulator = 0
@@ -128,40 +137,43 @@ func (s *SPVCon) TxHandler(m *wire.MsgTx) {
 	height, ok := s.TS.OKTxids[m.TxSha()]
 	s.TS.OKMutex.Unlock()
 	if !ok {
-		log.Printf("Tx %s unknown, will not ingest\n")
+		log.Printf("Tx %s unknown, will not ingest\n", m.TxSha().String())
 		return
 	}
 
 	// check for double spends
-	allTxs, err := s.TS.GetAllTxs()
+	//	allTxs, err := s.TS.GetAllTxs()
-	if err != nil {
+	//	if err != nil {
-		log.Printf("Can't get txs from db: %s", err.Error())
+	//		log.Printf("Can't get txs from db: %s", err.Error())
-		return
+	//		return
-	}
+	//	}
-	dubs, err := CheckDoubleSpends(m, allTxs)
+	//	dubs, err := CheckDoubleSpends(m, allTxs)
-	if err != nil {
+	//	if err != nil {
-		log.Printf("CheckDoubleSpends error: %s", err.Error())
+	//		log.Printf("CheckDoubleSpends error: %s", err.Error())
-		return
+	//		return
-	}
+	//	}
-	if len(dubs) > 0 {
+	//	if len(dubs) > 0 {
-		for i, dub := range dubs {
+	//		for i, dub := range dubs {
-			fmt.Printf("dub %d known tx %s and new tx %s are exclusive!!!\n",
+	//			fmt.Printf("dub %d known tx %s and new tx %s are exclusive!!!\n",
-				i, dub.String(), m.TxSha().String())
+	//				i, dub.String(), m.TxSha().String())
+	//		}
+	//	}
+	utilTx := btcutil.NewTx(m)
+	if !s.HardMode || s.TS.localFilter.MatchTxAndUpdate(utilTx) {
+		hits, err := s.TS.Ingest(m, height)
+		if err != nil {
+			log.Printf("Incoming Tx error: %s\n", err.Error())
+			return
 		}
+		if hits == 0 && !s.HardMode {
+			log.Printf("tx %s had no hits, filter false positive.",
+				m.TxSha().String())
+			s.fPositives <- 1 // add one false positive to chan
+			return
+		}
+		log.Printf("tx %s ingested and matches %d utxo/adrs.",
+			m.TxSha().String(), hits)
 	}
-	hits, err := s.TS.Ingest(m, height)
-	if err != nil {
-		log.Printf("Incoming Tx error: %s\n", err.Error())
-		return
-	}
-	if hits == 0 && !s.HardMode {
-		log.Printf("tx %s had no hits, filter false positive.",
-			m.TxSha().String())
-		s.fPositives <- 1 // add one false positive to chan
-		return
-	}
-	log.Printf("tx %s ingested and matches %d utxo/adrs.",
-		m.TxSha().String(), hits)
 }
 
 // GetDataHandler responds to requests for tx data, which happen after
@@ -172,17 +184,33 @@ func (s *SPVCon) GetDataHandler(m *wire.MsgGetData) {
 	for i, thing := range m.InvList {
 		log.Printf("\t%d)%s : %s",
 			i, thing.Type.String(), thing.Hash.String())
-		if thing.Type != wire.InvTypeTx { // refuse non-tx reqs
+
-			log.Printf("We only respond to tx requests, ignoring")
+		// separate wittx and tx.  needed / combine?
+		// does the same thing right now
+		if thing.Type == wire.InvTypeWitnessTx {
+			tx, err := s.TS.GetTx(&thing.Hash)
+			if err != nil {
+				log.Printf("error getting tx %s: %s",
+					thing.Hash.String(), err.Error())
+			}
+			s.outMsgQueue <- tx
+			sent++
 			continue
 		}
-		tx, err := s.TS.GetTx(&thing.Hash)
+		if thing.Type == wire.InvTypeTx {
-		if err != nil {
+			tx, err := s.TS.GetTx(&thing.Hash)
-			log.Printf("error getting tx %s: %s",
+			if err != nil {
-				thing.Hash.String(), err.Error())
+				log.Printf("error getting tx %s: %s",
+					thing.Hash.String(), err.Error())
+			}
+			tx.Flags = 0x00 // dewitnessify
+			s.outMsgQueue <- tx
+			sent++
+			continue
 		}
-		s.outMsgQueue <- tx
+		// didn't match, so it's not something we're responding to
-		sent++
+		log.Printf("We only respond to tx requests, ignoring")
+
 	}
 	log.Printf("sent %d of %d requested items", sent, len(m.InvList))
 }

uspv/sortsignsend.go

@@ -4,9 +4,11 @@ import (
 	"bytes"
 	"fmt"
 	"log"
+	"sort"
 
 	"github.com/btcsuite/btcd/txscript"
 	"github.com/btcsuite/btcd/wire"
+	"github.com/btcsuite/btcutil"
 	"github.com/btcsuite/btcutil/bloom"
 	"github.com/btcsuite/btcutil/hdkeychain"
 	"github.com/btcsuite/btcutil/txsort"
@@ -41,6 +43,48 @@ func (s *SPVCon) Rebroadcast() {
 	return
 }
 
+// make utxo slices sortable -- same as txsort
+type utxoSlice []Utxo
+
+// Sort utxos just like txins -- Len, Less, Swap
+func (s utxoSlice) Len() int      { return len(s) }
+func (s utxoSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+
+// outpoint sort; First input hash (reversed / rpc-style), then index.
+func (s utxoSlice) Less(i, j int) bool {
+	// Input hashes are the same, so compare the index.
+	ihash := s[i].Op.Hash
+	jhash := s[j].Op.Hash
+	if ihash == jhash {
+		return s[i].Op.Index < s[j].Op.Index
+	}
+	// At this point, the hashes are not equal, so reverse them to
+	// big-endian and return the result of the comparison.
+	const hashSize = wire.HashSize
+	for b := 0; b < hashSize/2; b++ {
+		ihash[b], ihash[hashSize-1-b] = ihash[hashSize-1-b], ihash[b]
+		jhash[b], jhash[hashSize-1-b] = jhash[hashSize-1-b], jhash[b]
+	}
+	return bytes.Compare(ihash[:], jhash[:]) == -1
+}
+
+type SortableUtxoSlice []Utxo
+
+// utxoByAmts get sorted by utxo value. also put unconfirmed last
+func (s SortableUtxoSlice) Len() int      { return len(s) }
+func (s SortableUtxoSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+
+// height 0 means your lesser
+func (s SortableUtxoSlice) Less(i, j int) bool {
+	if s[i].AtHeight == 0 && s[j].AtHeight > 0 {
+		return true
+	}
+	if s[j].AtHeight == 0 && s[i].AtHeight > 0 {
+		return false
+	}
+	return s[i].Value < s[j].Value
+}
+
 func (s *SPVCon) NewOutgoingTx(tx *wire.MsgTx) error {
 	txid := tx.TxSha()
 	// assign height of zero for txs we create
@@ -53,7 +97,7 @@ func (s *SPVCon) NewOutgoingTx(tx *wire.MsgTx) error {
 		return err
 	}
 	// make an inv message instead of a tx message to be polite
-	iv1 := wire.NewInvVect(wire.InvTypeTx, &txid)
+	iv1 := wire.NewInvVect(wire.InvTypeWitnessTx, &txid)
 	invMsg := wire.NewMsgInv()
 	err = invMsg.AddInvVect(iv1)
 	if err != nil {
@@ -63,6 +107,313 @@ func (s *SPVCon) NewOutgoingTx(tx *wire.MsgTx) error {
 	return nil
 }
 
+func (s *SPVCon) SendOne(u Utxo, adr btcutil.Address) error {
+	// fixed fee
+	fee := int64(5000)
+
+	sendAmt := u.Value - fee
+	tx := wire.NewMsgTx() // make new tx
+	// add single output
+	outAdrScript, err := txscript.PayToAddrScript(adr)
+	if err != nil {
+		return err
+	}
+	// make user specified txout and add to tx
+	txout := wire.NewTxOut(sendAmt, outAdrScript)
+	tx.AddTxOut(txout)
+
+	var prevPKs []byte
+	if u.IsWit {
+		tx.Flags = 0x01
+		wa, err := btcutil.NewAddressWitnessPubKeyHash(
+			s.TS.Adrs[u.KeyIdx].PkhAdr.ScriptAddress(), s.TS.Param)
+		prevPKs, err = txscript.PayToAddrScript(wa)
+		if err != nil {
+			return err
+		}
+	} else { // otherwise generate directly
+		prevPKs, err = txscript.PayToAddrScript(
+			s.TS.Adrs[u.KeyIdx].PkhAdr)
+		if err != nil {
+			return err
+		}
+	}
+
+	tx.AddTxIn(wire.NewTxIn(&u.Op, prevPKs, nil))
+
+	var sig []byte
+	var wit [][]byte
+	hCache := txscript.CalcHashCache(tx, 0, txscript.SigHashAll)
+
+	child, err := s.TS.rootPrivKey.Child(u.KeyIdx + hdkeychain.HardenedKeyStart)
+
+	if err != nil {
+		return err
+	}
+	priv, err := child.ECPrivKey()
+	if err != nil {
+		return err
+	}
+
+	// This is where witness based sighash types need to happen
+	// sign into stash
+	if u.IsWit {
+		wit, err = txscript.WitnessScript(
+			tx, hCache, 0, u.Value, tx.TxIn[0].SignatureScript,
+			txscript.SigHashAll, priv, true)
+		if err != nil {
+			return err
+		}
+	} else {
+		sig, err = txscript.SignatureScript(
+			tx, 0, tx.TxIn[0].SignatureScript,
+			txscript.SigHashAll, priv, true)
+		if err != nil {
+			return err
+		}
+	}
+
+	// swap sigs into sigScripts in txins
+
+	if sig != nil {
+		tx.TxIn[0].SignatureScript = sig
+	}
+	if wit != nil {
+		tx.TxIn[0].Witness = wit
+		tx.TxIn[0].SignatureScript = nil
+	}
+	return s.NewOutgoingTx(tx)
+}
+
+// SendCoins does send coins, but it's very rudimentary
+// wit makes it into p2wpkh.  Which is not yet spendable.
+func (s *SPVCon) SendCoins(adrs []btcutil.Address, sendAmts []int64) error {
+	if len(adrs) != len(sendAmts) {
+		return fmt.Errorf("%d addresses and %d amounts", len(adrs), len(sendAmts))
+	}
+	var err error
+	var score, totalSend, fee int64
+	dustCutoff := int64(20000) // below this amount, just give to miners
+	satPerByte := int64(80)    // satoshis per byte fee; have as arg later
+	rawUtxos, err := s.TS.GetAllUtxos()
+	if err != nil {
+		return err
+	}
+	var allUtxos SortableUtxoSlice
+	// start with utxos sorted by value.
+
+	for _, utxo := range rawUtxos {
+		score += utxo.Value
+		allUtxos = append(allUtxos, *utxo)
+	}
+	// smallest and unconfirmed last (because it's reversed)
+	sort.Sort(sort.Reverse(allUtxos))
+
+	//	sort.Reverse(allUtxos)
+	for _, amt := range sendAmts {
+		totalSend += amt
+	}
+	// important rule in bitcoin, output total > input total is invalid.
+	if totalSend > score {
+		return fmt.Errorf("trying to send %d but %d available.",
+			totalSend, score)
+	}
+
+	tx := wire.NewMsgTx() // make new tx
+	// add non-change (arg) outputs
+	for i, adr := range adrs {
+		// make address script 76a914...88ac or 0014...
+		outAdrScript, err := txscript.PayToAddrScript(adr)
+		if err != nil {
+			return err
+		}
+		// make user specified txout and add to tx
+		txout := wire.NewTxOut(sendAmts[i], outAdrScript)
+		tx.AddTxOut(txout)
+	}
+
+	// generate a utxo slice for your inputs
+	var ins utxoSlice
+
+	// add utxos until we've had enough
+	nokori := totalSend // nokori is how much is needed on input side
+	for _, utxo := range allUtxos {
+		// skip unconfirmed.  Or de-prioritize?
+		//		if utxo.AtHeight == 0 {
+		//			continue
+		//		}
+
+		// yeah, lets add this utxo!
+		ins = append(ins, utxo)
+		// as we add utxos, fill in sigscripts
+		// generate previous pkscripts (subscritpt?) for all utxos
+		// then make txins with the utxo and prevpk, and insert them into the tx
+		// these are all zeroed out during signing but it's an easy way to keep track
+		var prevPKs []byte
+		if utxo.IsWit {
+			tx.Flags = 0x01
+			wa, err := btcutil.NewAddressWitnessPubKeyHash(
+				s.TS.Adrs[utxo.KeyIdx].PkhAdr.ScriptAddress(), s.TS.Param)
+			prevPKs, err = txscript.PayToAddrScript(wa)
+			if err != nil {
+				return err
+			}
+		} else { // otherwise generate directly
+			prevPKs, err = txscript.PayToAddrScript(
+				s.TS.Adrs[utxo.KeyIdx].PkhAdr)
+			if err != nil {
+				return err
+			}
+		}
+		tx.AddTxIn(wire.NewTxIn(&utxo.Op, prevPKs, nil))
+		nokori -= utxo.Value
+		// if nokori is positive, don't bother checking fee yet.
+		if nokori < 0 {
+			fee = EstFee(tx, satPerByte)
+			if nokori < -fee { // done adding utxos: nokori below negative est. fee
+				break
+			}
+		}
+	}
+
+	// see if there's enough left to also add a change output
+
+	changeOld, err := s.TS.NewAdr() // change is witnessy
+	if err != nil {
+		return err
+	}
+	changeAdr, err := btcutil.NewAddressWitnessPubKeyHash(
+		changeOld.ScriptAddress(), s.TS.Param)
+	if err != nil {
+		return err
+	}
+
+	changeScript, err := txscript.PayToAddrScript(changeAdr)
+	if err != nil {
+		return err
+	}
+
+	changeOut := wire.NewTxOut(0, changeScript)
+	tx.AddTxOut(changeOut)
+	fee = EstFee(tx, satPerByte)
+	changeOut.Value = -(nokori + fee)
+	if changeOut.Value < dustCutoff {
+		// remove last output (change) : not worth it
+		tx.TxOut = tx.TxOut[:len(tx.TxOut)-1]
+	}
+
+	// sort utxos on the input side.  use this instead of txsort
+	// because we want to remember which keys are associated with which inputs
+	sort.Sort(ins)
+
+	// sort tx -- this only will change txouts since inputs are already sorted
+	txsort.InPlaceSort(tx)
+
+	// tx is ready for signing,
+	sigStash := make([][]byte, len(ins))
+	witStash := make([][][]byte, len(ins))
+
+	// generate tx-wide hashCache for segwit stuff
+	// middle index number doesn't matter for sighashAll.
+	hCache := txscript.CalcHashCache(tx, 0, txscript.SigHashAll)
+
+	for i, txin := range tx.TxIn {
+		// pick key
+		child, err := s.TS.rootPrivKey.Child(
+			ins[i].KeyIdx + hdkeychain.HardenedKeyStart)
+		if err != nil {
+			return err
+		}
+		priv, err := child.ECPrivKey()
+		if err != nil {
+			return err
+		}
+
+		// This is where witness based sighash types need to happen
+		// sign into stash
+		if ins[i].IsWit {
+			witStash[i], err = txscript.WitnessScript(
+				tx, hCache, i, ins[i].Value, txin.SignatureScript,
+				txscript.SigHashAll, priv, true)
+			if err != nil {
+				return err
+			}
+		} else {
+			sigStash[i], err = txscript.SignatureScript(
+				tx, i, txin.SignatureScript,
+				txscript.SigHashAll, priv, true)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	// swap sigs into sigScripts in txins
+	for i, txin := range tx.TxIn {
+		if sigStash[i] != nil {
+			txin.SignatureScript = sigStash[i]
+		}
+		if witStash[i] != nil {
+			txin.Witness = witStash[i]
+			txin.SignatureScript = nil
+		}
+
+	}
+
+	//	fmt.Printf("tx: %s", TxToString(tx))
+	//	buf := bytes.NewBuffer(make([]byte, 0, tx.SerializeSize()))
+
+	// send it out on the wire.  hope it gets there.
+	// we should deal with rejects.  Don't yet.
+	err = s.NewOutgoingTx(tx)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// EstFee gives a fee estimate based on a tx and a sat/Byte target.
+// The TX should have all outputs, including the change address already
+// populated (with potentially 0 amount.  Also it should have all inputs
+// populated, but inputs don't need to have sigscripts or witnesses
+// (it'll guess the sizes of sigs/wits that arent' filled in).
+func EstFee(otx *wire.MsgTx, spB int64) int64 {
+	mtsig := make([]byte, 72)
+	mtpub := make([]byte, 33)
+
+	tx := otx.Copy()
+
+	// iterate through txins, replacing subscript sigscripts with noise
+	// sigs or witnesses
+	for _, txin := range tx.TxIn {
+		// check wpkh
+		if len(txin.SignatureScript) == 22 &&
+			txin.SignatureScript[0] == 0x00 && txin.SignatureScript[1] == 0x14 {
+			txin.SignatureScript = nil
+			txin.Witness = make([][]byte, 2)
+			txin.Witness[0] = mtsig
+			txin.Witness[1] = mtpub
+		} else if len(txin.SignatureScript) == 34 &&
+			txin.SignatureScript[0] == 0x00 && txin.SignatureScript[1] == 0x20 {
+			// p2wsh -- sig lenght is a total guess!
+			txin.SignatureScript = nil
+			txin.Witness = make([][]byte, 3)
+			// 3 sigs? totally guessing here
+			txin.Witness[0] = mtsig
+			txin.Witness[1] = mtsig
+			txin.Witness[2] = mtsig
+		} else {
+			// assume everything else is p2pkh.  Even though it's not
+			txin.Witness = nil
+			txin.SignatureScript = make([]byte, 105) // len of p2pkh sigscript
+		}
+	}
+	fmt.Printf(TxToString(tx))
+	size := int64(tx.VirtualSize())
+	fmt.Printf("%d spB, est vsize %d, fee %d\n", spB, size, size*spB)
+	return size * spB
+}
+
+// SignThis isn't used anymore...
 func (t *TxStore) SignThis(tx *wire.MsgTx) error {
 	fmt.Printf("-= SignThis =-\n")
 

uspv/txstore.go

@@ -23,6 +23,8 @@ type TxStore struct {
 	Adrs    []MyAdr  // endeavouring to acquire capital
 	StateDB *bolt.DB // place to write all this down
 
+	localFilter *bloom.Filter // local bloom filter for hard mode
+
 	// Params live here, not SCon
 	Param *chaincfg.Params // network parameters (testnet3, testnetL)
 
@@ -38,7 +40,8 @@ type Utxo struct { // cash money.
 	KeyIdx   uint32 // index for private key needed to sign / spend
 	Value    int64  // higher is better
 
-	//	IsCoinbase bool          // can't spend for a while
+	//		IsCoinbase bool          // can't spend for a while
+	IsWit bool // true if p2wpkh output
 }
 
 // Stxo is a utxo that has moved on.
@@ -78,7 +81,7 @@ func (t *TxStore) AddTxid(txid *wire.ShaHash, height int32) error {
 // ... or I'm gonna fade away
 func (t *TxStore) GimmeFilter() (*bloom.Filter, error) {
 	if len(t.Adrs) == 0 {
-		return nil, fmt.Errorf("no addresses to filter for")
+		return nil, fmt.Errorf("no address to filter for")
 	}
 
 	// get all utxos to add outpoints to filter
@@ -89,7 +92,10 @@ func (t *TxStore) GimmeFilter() (*bloom.Filter, error) {
 
 	elem := uint32(len(t.Adrs) + len(allUtxos))
 	f := bloom.NewFilter(elem, 0, 0.000001, wire.BloomUpdateAll)
-	for _, a := range t.Adrs {
+
+	// note there could be false positives since we're just looking
+	// for the 20 byte PKH without the opcodes.
+	for _, a := range t.Adrs { // add 20-byte pubkeyhash
 		f.Add(a.PkhAdr.ScriptAddress())
 	}
 
@@ -133,14 +139,19 @@ func CheckDoubleSpends(
 
 // TxToString prints out some info about a transaction. for testing / debugging
 func TxToString(tx *wire.MsgTx) string {
-	str := fmt.Sprintf("\t - Tx %s\n", tx.TxSha().String())
+	str := fmt.Sprintf("size %d vsize %d wsize %d locktime %d flag %x txid %s\n",
+		tx.SerializeSize(), tx.VirtualSize(), tx.SerializeSizeWitness(),
+		tx.LockTime, tx.Flags, tx.TxSha().String())
 	for i, in := range tx.TxIn {
-		str += fmt.Sprintf("Input %d: %s\n", i, in.PreviousOutPoint.String())
+		str += fmt.Sprintf("Input %d spends %s\n", i, in.PreviousOutPoint.String())
-		str += fmt.Sprintf("SigScript for input %d: %x\n", i, in.SignatureScript)
+		str += fmt.Sprintf("\tSigScript: %x\n", in.SignatureScript)
+		for j, wit := range in.Witness {
+			str += fmt.Sprintf("\twitness %d: %x\n", j, wit)
+		}
 	}
 	for i, out := range tx.TxOut {
 		if out != nil {
-			str += fmt.Sprintf("\toutput %d script: %x amt: %d\n",
+			str += fmt.Sprintf("output %d script: %x amt: %d\n",
 				i, out.PkScript, out.Value)
 		} else {
 			str += fmt.Sprintf("output %d nil (WARNING)\n", i)
@@ -175,6 +186,20 @@ func outPointToBytes(op *wire.OutPoint) ([]byte, error) {
 	return buf.Bytes(), nil
 }
 
+/*----- serialization for utxos ------- */
+/* Utxos serialization:
+byte length   desc   at offset
+
+32	txid		0
+4	idx		32
+4	height	36
+4	keyidx	40
+8	amt		44
+1	flag		52
+
+end len 	53
+*/
+
 // ToBytes turns a Utxo into some bytes.
 // note that the txid is the first 36 bytes and in our use cases will be stripped
 // off, but is left here for other applications
@@ -205,6 +230,16 @@ func (u *Utxo) ToBytes() ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
+	// last byte indicates tx witness flags ( tx[5] from serialized tx)
+	// write a 1 at the end for p2wpkh (same as flags byte)
+	witByte := byte(0x00)
+	if u.IsWit {
+		witByte = 0x01
+	}
+	err = buf.WriteByte(witByte)
+	if err != nil {
+		return nil, err
+	}
 
 	return buf.Bytes(), nil
 }
@@ -218,8 +253,8 @@ func UtxoFromBytes(b []byte) (Utxo, error) {
 		return u, fmt.Errorf("nil input slice")
 	}
 	buf := bytes.NewBuffer(b)
-	if buf.Len() < 52 { // utxos are 52 bytes
+	if buf.Len() < 53 { // utxos are 53 bytes
-		return u, fmt.Errorf("Got %d bytes for utxo, expect 52", buf.Len())
+		return u, fmt.Errorf("Got %d bytes for utxo, expect 53", buf.Len())
 	}
 	// read 32 byte txid
 	err := u.Op.Hash.SetBytes(buf.Next(32))
@@ -246,38 +281,44 @@ func UtxoFromBytes(b []byte) (Utxo, error) {
 	if err != nil {
 		return u, err
 	}
+	// read 1 byte witness flags
+	witByte, err := buf.ReadByte()
+	if err != nil {
+		return u, err
+	}
+	if witByte != 0x00 {
+		u.IsWit = true
+	}
+
 	return u, nil
 }
 
+/*----- serialization for stxos ------- */
+/* Stxo serialization:
+byte length   desc   at offset
+
+53	utxo		0
+4	sheight	53
+32	stxid	57
+
+end len 	89
+*/
+
 // ToBytes turns an Stxo into some bytes.
-// outpoint txid, outpoint idx, height, key idx, amt, spendheight, spendtxid
+// prevUtxo serialization, then spendheight [4], spendtxid [32]
 func (s *Stxo) ToBytes() ([]byte, error) {
 	var buf bytes.Buffer
-	// write 32 byte txid of the utxo
+	// first serialize the utxo part
-	_, err := buf.Write(s.Op.Hash.Bytes())
+	uBytes, err := s.Utxo.ToBytes()
 	if err != nil {
 		return nil, err
 	}
-	// write 4 byte outpoint index within the tx to spend
+	// write that into the buffer first
-	err = binary.Write(&buf, binary.BigEndian, s.Op.Index)
+	_, err = buf.Write(uBytes)
-	if err != nil {
-		return nil, err
-	}
-	// write 4 byte height of utxo
-	err = binary.Write(&buf, binary.BigEndian, s.AtHeight)
-	if err != nil {
-		return nil, err
-	}
-	// write 4 byte key index of utxo
-	err = binary.Write(&buf, binary.BigEndian, s.KeyIdx)
-	if err != nil {
-		return nil, err
-	}
-	// write 8 byte amount of money at the utxo
-	err = binary.Write(&buf, binary.BigEndian, s.Value)
 	if err != nil {
 		return nil, err
 	}
+
 	// write 4 byte height where the txo was spent
 	err = binary.Write(&buf, binary.BigEndian, s.SpendHeight)
 	if err != nil {
@@ -292,40 +333,21 @@ func (s *Stxo) ToBytes() ([]byte, error) {
 }
 
 // StxoFromBytes turns bytes into a Stxo.
+// first take the first 53 bytes as a utxo, then the next 36 for how it's spent.
 func StxoFromBytes(b []byte) (Stxo, error) {
 	var s Stxo
-	if b == nil {
+	if len(b) < 89 {
-		return s, fmt.Errorf("nil input slice")
+		return s, fmt.Errorf("Got %d bytes for stxo, expect 89", len(b))
 	}
-	buf := bytes.NewBuffer(b)
+
-	if buf.Len() < 88 { // stxos are 88 bytes
+	u, err := UtxoFromBytes(b[:53])
-		return s, fmt.Errorf("Got %d bytes for stxo, expect 88", buf.Len())
-	}
-	// read 32 byte txid
-	err := s.Op.Hash.SetBytes(buf.Next(32))
-	if err != nil {
-		return s, err
-	}
-	// read 4 byte outpoint index within the tx to spend
-	err = binary.Read(buf, binary.BigEndian, &s.Op.Index)
-	if err != nil {
-		return s, err
-	}
-	// read 4 byte height of utxo
-	err = binary.Read(buf, binary.BigEndian, &s.AtHeight)
-	if err != nil {
-		return s, err
-	}
-	// read 4 byte key index of utxo
-	err = binary.Read(buf, binary.BigEndian, &s.KeyIdx)
-	if err != nil {
-		return s, err
-	}
-	// read 8 byte amount of money at the utxo
-	err = binary.Read(buf, binary.BigEndian, &s.Value)
 	if err != nil {
 		return s, err
 	}
+	s.Utxo = u // assign the utxo
+
+	buf := bytes.NewBuffer(b[53:]) // make buffer for spend data
+
 	// read 4 byte spend height
 	err = binary.Read(buf, binary.BigEndian, &s.SpendHeight)
 	if err != nil {

uspv/utxodb.go

@@ -22,7 +22,7 @@ var (
 	BKTTxns  = []byte("Txns")      // all txs we care about, for replays
 	BKTState = []byte("MiscState") // last state of DB
 	// these are in the state bucket
-	KEYNumKeys   = []byte("NumKeys")   // number of keys used
+	KEYNumKeys   = []byte("NumKeys")   // number of p2pkh keys used
 	KEYTipHeight = []byte("TipHeight") // height synced to
 )
 
@@ -82,17 +82,22 @@ func (ts *TxStore) OpenDB(filename string) error {
 
 // NewAdr creates a new, never before seen address, and increments the
 // DB counter as well as putting it in the ram Adrs store, and returns it
-func (ts *TxStore) NewAdr() (*btcutil.AddressPubKeyHash, error) {
+func (ts *TxStore) NewAdr() (btcutil.Address, error) {
 	if ts.Param == nil {
-		return nil, fmt.Errorf("nil param")
+		return nil, fmt.Errorf("NewAdr error: nil param")
 	}
-	n := uint32(len(ts.Adrs))
+
-	priv, err := ts.rootPrivKey.Child(n + hdkeychain.HardenedKeyStart)
+	priv := new(hdkeychain.ExtendedKey)
+	var err error
+	var n uint32
+	var nAdr btcutil.Address
+
+	n = uint32(len(ts.Adrs))
+	priv, err = ts.rootPrivKey.Child(n + hdkeychain.HardenedKeyStart)
 	if err != nil {
 		return nil, err
 	}
-
+	nAdr, err = priv.Address(ts.Param)
-	newAdr, err := priv.Address(ts.Param)
 	if err != nil {
 		return nil, err
 	}
@@ -107,18 +112,21 @@ func (ts *TxStore) NewAdr() (*btcutil.AddressPubKeyHash, error) {
 	// write to db file
 	err = ts.StateDB.Update(func(btx *bolt.Tx) error {
 		sta := btx.Bucket(BKTState)
+
 		return sta.Put(KEYNumKeys, buf.Bytes())
+
 	})
 	if err != nil {
 		return nil, err
 	}
 	// add in to ram.
 	var ma MyAdr
-	ma.PkhAdr = newAdr
+	ma.PkhAdr = nAdr
 	ma.KeyIdx = n
-	ts.Adrs = append(ts.Adrs, ma)
 
-	return newAdr, nil
+	ts.Adrs = append(ts.Adrs, ma)
+	ts.localFilter.Add(ma.PkhAdr.ScriptAddress())
+	return nAdr, nil
 }
 
 // SetDBSyncHeight sets sync height of the db, indicated the latest block
@@ -345,7 +353,6 @@ func (ts *TxStore) PopulateAdrs(lastKey uint32) error {
 func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 	var hits uint32
 	var err error
-	var spentOPs [][]byte
 	var nUtxoBytes [][]byte
 
 	// tx has been OK'd by SPV; check tx sanity
@@ -358,30 +365,57 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 	// note that you can't check signatures; this is SPV.
 	// 0 conf SPV means pretty much nothing.  Anyone can say anything.
 
+	spentOPs := make([][]byte, len(tx.TxIn))
 	// before entering into db, serialize all inputs of the ingested tx
-	for _, txin := range tx.TxIn {
+	for i, txin := range tx.TxIn {
-		nOP, err := outPointToBytes(&txin.PreviousOutPoint)
+		spentOPs[i], err = outPointToBytes(&txin.PreviousOutPoint)
 		if err != nil {
 			return hits, err
 		}
-		spentOPs = append(spentOPs, nOP)
 	}
-	// also generate PKscripts for all addresses (maybe keep storing these?)
+
-	for _, adr := range ts.Adrs {
+	// go through txouts, and then go through addresses to match
+
+	// generate PKscripts for all addresses
+	wPKscripts := make([][]byte, len(ts.Adrs))
+	aPKscripts := make([][]byte, len(ts.Adrs))
+
+	for i, _ := range ts.Adrs {
 		// iterate through all our addresses
-		aPKscript, err := txscript.PayToAddrScript(adr.PkhAdr)
+		// convert regular address to witness address.  (split adrs later)
+		wa, err := btcutil.NewAddressWitnessPubKeyHash(
+			ts.Adrs[i].PkhAdr.ScriptAddress(), ts.Param)
 		if err != nil {
 			return hits, err
 		}
-		// iterate through all outputs of this tx
+
-		for i, out := range tx.TxOut {
+		wPKscripts[i], err = txscript.PayToAddrScript(wa)
-			if bytes.Equal(out.PkScript, aPKscript) { // new utxo for us
+		if err != nil {
-				var newu Utxo
+			return hits, err
+		}
+		aPKscripts[i], err = txscript.PayToAddrScript(ts.Adrs[i].PkhAdr)
+		if err != nil {
+			return hits, err
+		}
+	}
+
+	cachedSha := tx.TxSha()
+	// iterate through all outputs of this tx, see if we gain
+	for i, out := range tx.TxOut {
+		for j, ascr := range aPKscripts {
+			// detect p2wpkh
+			witBool := false
+			if bytes.Equal(out.PkScript, wPKscripts[j]) {
+				witBool = true
+			}
+			if bytes.Equal(out.PkScript, ascr) || witBool { // new utxo found
+				var newu Utxo // create new utxo and copy into it
 				newu.AtHeight = height
-				newu.KeyIdx = adr.KeyIdx
+				newu.KeyIdx = ts.Adrs[j].KeyIdx
 				newu.Value = out.Value
+				newu.IsWit = witBool // copy witness version from pkscript
 				var newop wire.OutPoint
-				newop.Hash = tx.TxSha()
+				newop.Hash = cachedSha
 				newop.Index = uint32(i)
 				newu.Op = newop
 				b, err := newu.ToBytes()
@@ -390,7 +424,7 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 				}
 				nUtxoBytes = append(nUtxoBytes, b)
 				hits++
-				break // only one match
+				break // txos can match only 1 script
 			}
 		}
 	}
@@ -405,53 +439,43 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 			return fmt.Errorf("error: db not initialized")
 		}
 
-		// first see if we lose utxos
 		// iterate through duffel bag and look for matches
 		// this makes us lose money, which is regrettable, but we need to know.
 		for _, nOP := range spentOPs {
-			duf.ForEach(func(k, v []byte) error {
+			v := duf.Get(nOP)
-				if bytes.Equal(k, nOP) { // matched, we lost utxo
+			if v != nil {
-					// do all this just to figure out value we lost
+				hits++
-					x := make([]byte, len(k)+len(v))
+				// do all this just to figure out value we lost
-					copy(x, k)
+				x := make([]byte, len(nOP)+len(v))
-					copy(x[len(k):], v)
+				copy(x, nOP)
-					lostTxo, err := UtxoFromBytes(x)
+				copy(x[len(nOP):], v)
-					if err != nil {
+				lostTxo, err := UtxoFromBytes(x)
-						return err
+				if err != nil {
-					}
+					return err
-					hits++
-					// then delete the utxo from duf, save to old
-					err = duf.Delete(k)
-					if err != nil {
-						return err
-					}
-					// after deletion, save stxo to old bucket
-					var st Stxo               // generate spent txo
-					st.Utxo = lostTxo         // assign outpoint
-					st.SpendHeight = height   // spent at height
-					st.SpendTxid = tx.TxSha() // spent by txid
-					stxb, err := st.ToBytes() // serialize
-					if err != nil {
-						return err
-					}
-					err = old.Put(k, stxb) // write k:v outpoint:stxo bytes
-					if err != nil {
-						return err
-					}
-					// store this relevant tx
-					sha := tx.TxSha()
-					var buf bytes.Buffer
-					tx.Serialize(&buf)
-					err = txns.Put(sha.Bytes(), buf.Bytes())
-					if err != nil {
-						return err
-					}
-
-					return nil // matched utxo k, won't match another
 				}
-				return nil // no match
+
-			})
+				// after marking for deletion, save stxo to old bucket
-		} // done losing utxos, next gain utxos
+				var st Stxo               // generate spent txo
+				st.Utxo = lostTxo         // assign outpoint
+				st.SpendHeight = height   // spent at height
+				st.SpendTxid = cachedSha  // spent by txid
+				stxb, err := st.ToBytes() // serialize
+				if err != nil {
+					return err
+				}
+				err = old.Put(nOP, stxb) // write nOP:v outpoint:stxo bytes
+				if err != nil {
+					return err
+				}
+
+				err = duf.Delete(nOP)
+				if err != nil {
+					return err
+				}
+			}
+		}
+
+		// done losing utxos, next gain utxos
 		// next add all new utxos to db, this is quick as the work is above
 		for _, ub := range nUtxoBytes {
 			err = duf.Put(ub[:36], ub[36:])
@@ -459,6 +483,15 @@ func (ts *TxStore) Ingest(tx *wire.MsgTx, height int32) (uint32, error) {
 				return err
 			}
 		}
+
+		// if hits is nonzero it's a relevant tx and we should store it
+		var buf bytes.Buffer
+		tx.SerializeWitness(&buf) // always store witness version
+		err = txns.Put(cachedSha.Bytes(), buf.Bytes())
+		if err != nil {
+			return err
+		}
+
 		return nil
 	})
 	return hits, err