From 43c2031fa8416ee1ff41e81d73f11051a4bafba1 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Fri, 9 Oct 2020 13:35:00 +0200 Subject: [PATCH 1/4] lntest: extract common bitcoind code into own file To prepare for running multiple tests on bitcoind with different options each time, we extract the common code into its own file. --- lntest/bitcoind.go | 175 +---------------------------------- lntest/bitcoind_common.go | 189 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 193 insertions(+), 171 deletions(-) create mode 100644 lntest/bitcoind_common.go diff --git a/lntest/bitcoind.go b/lntest/bitcoind.go index 56670892..3eb07a5e 100644 --- a/lntest/bitcoind.go +++ b/lntest/bitcoind.go @@ -3,187 +3,20 @@ package lntest import ( - "errors" - "fmt" - "io/ioutil" - "math/rand" - "os" - "os/exec" - "path/filepath" - "time" - "github.com/btcsuite/btcd/chaincfg" - "github.com/btcsuite/btcd/rpcclient" ) -// logDir is the name of the temporary log directory. -const logDir = "./.backendlogs" - -// BitcoindBackendConfig is an implementation of the BackendConfig interface -// backed by a Bitcoind node. -type BitcoindBackendConfig struct { - rpcHost string - rpcUser string - rpcPass string - zmqBlockPath string - zmqTxPath string - p2pPort int - rpcClient *rpcclient.Client - - // minerAddr is the p2p address of the miner to connect to. - minerAddr string -} - -// A compile time assertion to ensure BitcoindBackendConfig meets the -// BackendConfig interface. -var _ BackendConfig = (*BitcoindBackendConfig)(nil) - -// GenArgs returns the arguments needed to be passed to LND at startup for -// using this node as a chain backend. -func (b BitcoindBackendConfig) GenArgs() []string { - var args []string - args = append(args, "--bitcoin.node=bitcoind") - args = append(args, fmt.Sprintf("--bitcoind.rpchost=%v", b.rpcHost)) - args = append(args, fmt.Sprintf("--bitcoind.rpcuser=%v", b.rpcUser)) - args = append(args, fmt.Sprintf("--bitcoind.rpcpass=%v", b.rpcPass)) - args = append(args, fmt.Sprintf("--bitcoind.zmqpubrawblock=%v", - b.zmqBlockPath)) - args = append(args, fmt.Sprintf("--bitcoind.zmqpubrawtx=%v", - b.zmqTxPath)) - - return args -} - -// ConnectMiner is called to establish a connection to the test miner. -func (b BitcoindBackendConfig) ConnectMiner() error { - return b.rpcClient.AddNode(b.minerAddr, rpcclient.ANAdd) -} - -// DisconnectMiner is called to disconnect the miner. -func (b BitcoindBackendConfig) DisconnectMiner() error { - return b.rpcClient.AddNode(b.minerAddr, rpcclient.ANRemove) -} - -// Name returns the name of the backend type. -func (b BitcoindBackendConfig) Name() string { - return "bitcoind" -} - -// NewBackend starts a bitcoind node and returns a BitoindBackendConfig for -// that node. +// NewBackend starts a bitcoind node with the txindex enabled and returns a +// BitcoindBackendConfig for that node. func NewBackend(miner string, netParams *chaincfg.Params) ( *BitcoindBackendConfig, func() error, error) { - if netParams != &chaincfg.RegressionNetParams { - return nil, nil, fmt.Errorf("only regtest supported") - } - - if err := os.MkdirAll(logDir, 0700); err != nil { - return nil, nil, err - } - - logFile, err := filepath.Abs(logDir + "/bitcoind.log") - if err != nil { - return nil, nil, err - } - - tempBitcoindDir, err := ioutil.TempDir("", "bitcoind") - if err != nil { - return nil, nil, - fmt.Errorf("unable to create temp directory: %v", err) - } - - zmqBlockPath := "ipc:///" + tempBitcoindDir + "/blocks.socket" - zmqTxPath := "ipc:///" + tempBitcoindDir + "/txs.socket" - rpcPort := rand.Int()%(65536-1024) + 1024 - p2pPort := rand.Int()%(65536-1024) + 1024 - - bitcoind := exec.Command( - "bitcoind", - "-datadir="+tempBitcoindDir, + extraArgs := []string{ "-debug", "-regtest", "-txindex", - "-whitelist=127.0.0.1", // whitelist localhost to speed up relay - "-rpcauth=weks:469e9bb14ab2360f8e226efed5ca6f"+ - "d$507c670e800a95284294edb5773b05544b"+ - "220110063096c221be9933c82d38e1", - fmt.Sprintf("-rpcport=%d", rpcPort), - fmt.Sprintf("-port=%d", p2pPort), "-disablewallet", - "-zmqpubrawblock="+zmqBlockPath, - "-zmqpubrawtx="+zmqTxPath, - "-debuglogfile="+logFile, - ) - - err = bitcoind.Start() - if err != nil { - os.RemoveAll(tempBitcoindDir) - return nil, nil, fmt.Errorf("couldn't start bitcoind: %v", err) } - cleanUp := func() error { - bitcoind.Process.Kill() - bitcoind.Wait() - - var errStr string - // After shutting down the chain backend, we'll make a copy of - // the log file before deleting the temporary log dir. - err := CopyFile("./output_bitcoind_chainbackend.log", logFile) - if err != nil { - errStr += fmt.Sprintf("unable to copy file: %v\n", err) - } - if err = os.RemoveAll(logDir); err != nil { - errStr += fmt.Sprintf( - "cannot remove dir %s: %v\n", logDir, err, - ) - } - if err := os.RemoveAll(tempBitcoindDir); err != nil { - errStr += fmt.Sprintf( - "cannot remove dir %s: %v\n", - tempBitcoindDir, err, - ) - } - if errStr != "" { - return errors.New(errStr) - } - return nil - } - - // Allow process to start. - time.Sleep(1 * time.Second) - - rpcHost := fmt.Sprintf("127.0.0.1:%d", rpcPort) - rpcUser := "weks" - rpcPass := "weks" - - rpcCfg := rpcclient.ConnConfig{ - Host: rpcHost, - User: rpcUser, - Pass: rpcPass, - DisableConnectOnNew: true, - DisableAutoReconnect: false, - DisableTLS: true, - HTTPPostMode: true, - } - - client, err := rpcclient.New(&rpcCfg, nil) - if err != nil { - cleanUp() - return nil, nil, fmt.Errorf("unable to create rpc client: %v", - err) - } - - bd := BitcoindBackendConfig{ - rpcHost: rpcHost, - rpcUser: rpcUser, - rpcPass: rpcPass, - zmqBlockPath: zmqBlockPath, - zmqTxPath: zmqTxPath, - p2pPort: p2pPort, - rpcClient: client, - minerAddr: miner, - } - - return &bd, cleanUp, nil + return newBackend(miner, netParams, extraArgs) } diff --git a/lntest/bitcoind_common.go b/lntest/bitcoind_common.go new file mode 100644 index 00000000..51bc2ce1 --- /dev/null +++ b/lntest/bitcoind_common.go @@ -0,0 +1,189 @@ +// +build bitcoind + +package lntest + +import ( + "errors" + "fmt" + "io/ioutil" + "math/rand" + "os" + "os/exec" + "path/filepath" + "time" + + "github.com/btcsuite/btcd/chaincfg" + "github.com/btcsuite/btcd/rpcclient" +) + +// logDir is the name of the temporary log directory. +const logDir = "./.backendlogs" + +// BitcoindBackendConfig is an implementation of the BackendConfig interface +// backed by a Bitcoind node. +type BitcoindBackendConfig struct { + rpcHost string + rpcUser string + rpcPass string + zmqBlockPath string + zmqTxPath string + p2pPort int + rpcClient *rpcclient.Client + + // minerAddr is the p2p address of the miner to connect to. + minerAddr string +} + +// A compile time assertion to ensure BitcoindBackendConfig meets the +// BackendConfig interface. +var _ BackendConfig = (*BitcoindBackendConfig)(nil) + +// GenArgs returns the arguments needed to be passed to LND at startup for +// using this node as a chain backend. +func (b BitcoindBackendConfig) GenArgs() []string { + var args []string + args = append(args, "--bitcoin.node=bitcoind") + args = append(args, fmt.Sprintf("--bitcoind.rpchost=%v", b.rpcHost)) + args = append(args, fmt.Sprintf("--bitcoind.rpcuser=%v", b.rpcUser)) + args = append(args, fmt.Sprintf("--bitcoind.rpcpass=%v", b.rpcPass)) + args = append(args, fmt.Sprintf("--bitcoind.zmqpubrawblock=%v", + b.zmqBlockPath)) + args = append(args, fmt.Sprintf("--bitcoind.zmqpubrawtx=%v", + b.zmqTxPath)) + + return args +} + +// ConnectMiner is called to establish a connection to the test miner. +func (b BitcoindBackendConfig) ConnectMiner() error { + return b.rpcClient.AddNode(b.minerAddr, rpcclient.ANAdd) +} + +// DisconnectMiner is called to disconnect the miner. +func (b BitcoindBackendConfig) DisconnectMiner() error { + return b.rpcClient.AddNode(b.minerAddr, rpcclient.ANRemove) +} + +// Name returns the name of the backend type. +func (b BitcoindBackendConfig) Name() string { + return "bitcoind" +} + +// newBackend starts a bitcoind node with the given extra parameters and returns +// a BitcoindBackendConfig for that node. +func newBackend(miner string, netParams *chaincfg.Params, _ []string) ( + *BitcoindBackendConfig, func() error, error) { + + if netParams != &chaincfg.RegressionNetParams { + return nil, nil, fmt.Errorf("only regtest supported") + } + + if err := os.MkdirAll(logDir, 0700); err != nil { + return nil, nil, err + } + + logFile, err := filepath.Abs(logDir + "/bitcoind.log") + if err != nil { + return nil, nil, err + } + + tempBitcoindDir, err := ioutil.TempDir("", "bitcoind") + if err != nil { + return nil, nil, + fmt.Errorf("unable to create temp directory: %v", err) + } + + zmqBlockPath := "ipc:///" + tempBitcoindDir + "/blocks.socket" + zmqTxPath := "ipc:///" + tempBitcoindDir + "/txs.socket" + rpcPort := rand.Int()%(65536-1024) + 1024 + p2pPort := rand.Int()%(65536-1024) + 1024 + + bitcoind := exec.Command( + "bitcoind", + "-datadir="+tempBitcoindDir, + "-debug", + "-regtest", + "-txindex", + "-whitelist=127.0.0.1", // whitelist localhost to speed up relay + "-rpcauth=weks:469e9bb14ab2360f8e226efed5ca6f"+ + "d$507c670e800a95284294edb5773b05544b"+ + "220110063096c221be9933c82d38e1", + fmt.Sprintf("-rpcport=%d", rpcPort), + fmt.Sprintf("-port=%d", p2pPort), + "-disablewallet", + "-zmqpubrawblock="+zmqBlockPath, + "-zmqpubrawtx="+zmqTxPath, + "-debuglogfile="+logFile, + ) + + err = bitcoind.Start() + if err != nil { + os.RemoveAll(tempBitcoindDir) + return nil, nil, fmt.Errorf("couldn't start bitcoind: %v", err) + } + + cleanUp := func() error { + bitcoind.Process.Kill() + bitcoind.Wait() + + var errStr string + // After shutting down the chain backend, we'll make a copy of + // the log file before deleting the temporary log dir. + err := CopyFile("./output_bitcoind_chainbackend.log", logFile) + if err != nil { + errStr += fmt.Sprintf("unable to copy file: %v\n", err) + } + if err = os.RemoveAll(logDir); err != nil { + errStr += fmt.Sprintf( + "cannot remove dir %s: %v\n", logDir, err, + ) + } + if err := os.RemoveAll(tempBitcoindDir); err != nil { + errStr += fmt.Sprintf( + "cannot remove dir %s: %v\n", + tempBitcoindDir, err, + ) + } + if errStr != "" { + return errors.New(errStr) + } + return nil + } + + // Allow process to start. + time.Sleep(1 * time.Second) + + rpcHost := fmt.Sprintf("127.0.0.1:%d", rpcPort) + rpcUser := "weks" + rpcPass := "weks" + + rpcCfg := rpcclient.ConnConfig{ + Host: rpcHost, + User: rpcUser, + Pass: rpcPass, + DisableConnectOnNew: true, + DisableAutoReconnect: false, + DisableTLS: true, + HTTPPostMode: true, + } + + client, err := rpcclient.New(&rpcCfg, nil) + if err != nil { + cleanUp() + return nil, nil, fmt.Errorf("unable to create rpc client: %v", + err) + } + + bd := BitcoindBackendConfig{ + rpcHost: rpcHost, + rpcUser: rpcUser, + rpcPass: rpcPass, + zmqBlockPath: zmqBlockPath, + zmqTxPath: zmqTxPath, + p2pPort: p2pPort, + rpcClient: client, + minerAddr: miner, + } + + return &bd, cleanUp, nil +} From b7fc7c3daef83cf8704e20142f9f4fb3e024cc30 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Fri, 9 Oct 2020 13:35:02 +0200 Subject: [PATCH 2/4] lntest: use extraArgs, fix linter issues --- lntest/bitcoind_common.go | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/lntest/bitcoind_common.go b/lntest/bitcoind_common.go index 51bc2ce1..b59fdac8 100644 --- a/lntest/bitcoind_common.go +++ b/lntest/bitcoind_common.go @@ -71,7 +71,7 @@ func (b BitcoindBackendConfig) Name() string { // newBackend starts a bitcoind node with the given extra parameters and returns // a BitcoindBackendConfig for that node. -func newBackend(miner string, netParams *chaincfg.Params, _ []string) ( +func newBackend(miner string, netParams *chaincfg.Params, extraArgs []string) ( *BitcoindBackendConfig, func() error, error) { if netParams != &chaincfg.RegressionNetParams { @@ -98,33 +98,33 @@ func newBackend(miner string, netParams *chaincfg.Params, _ []string) ( rpcPort := rand.Int()%(65536-1024) + 1024 p2pPort := rand.Int()%(65536-1024) + 1024 - bitcoind := exec.Command( - "bitcoind", - "-datadir="+tempBitcoindDir, - "-debug", - "-regtest", - "-txindex", + cmdArgs := []string{ + "-datadir=" + tempBitcoindDir, "-whitelist=127.0.0.1", // whitelist localhost to speed up relay - "-rpcauth=weks:469e9bb14ab2360f8e226efed5ca6f"+ - "d$507c670e800a95284294edb5773b05544b"+ + "-rpcauth=weks:469e9bb14ab2360f8e226efed5ca6f" + + "d$507c670e800a95284294edb5773b05544b" + "220110063096c221be9933c82d38e1", fmt.Sprintf("-rpcport=%d", rpcPort), fmt.Sprintf("-port=%d", p2pPort), - "-disablewallet", - "-zmqpubrawblock="+zmqBlockPath, - "-zmqpubrawtx="+zmqTxPath, - "-debuglogfile="+logFile, - ) + "-zmqpubrawblock=" + zmqBlockPath, + "-zmqpubrawtx=" + zmqTxPath, + "-debuglogfile=" + logFile, + } + cmdArgs = append(cmdArgs, extraArgs...) + bitcoind := exec.Command("bitcoind", cmdArgs...) err = bitcoind.Start() if err != nil { - os.RemoveAll(tempBitcoindDir) + if err := os.RemoveAll(tempBitcoindDir); err != nil { + fmt.Printf("unable to remote temp dir %v: %v", + tempBitcoindDir, err) + } return nil, nil, fmt.Errorf("couldn't start bitcoind: %v", err) } cleanUp := func() error { - bitcoind.Process.Kill() - bitcoind.Wait() + _ = bitcoind.Process.Kill() + _ = bitcoind.Wait() var errStr string // After shutting down the chain backend, we'll make a copy of @@ -169,7 +169,7 @@ func newBackend(miner string, netParams *chaincfg.Params, _ []string) ( client, err := rpcclient.New(&rpcCfg, nil) if err != nil { - cleanUp() + _ = cleanUp() return nil, nil, fmt.Errorf("unable to create rpc client: %v", err) } From 3e3618ae9a4f9c6361c7a0541fe2dfd0dcd8c5b3 Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Fri, 9 Oct 2020 13:35:04 +0200 Subject: [PATCH 3/4] lntest: add build flag for disabling txindex on bitcoind We create a new build flag for running the bitcoind tests without the txindex enabled. We don't want this to be the default so we use a negated build flag. --- lntest/bitcoind.go | 1 + lntest/bitcoind_notxindex.go | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+) create mode 100644 lntest/bitcoind_notxindex.go diff --git a/lntest/bitcoind.go b/lntest/bitcoind.go index 3eb07a5e..e730b707 100644 --- a/lntest/bitcoind.go +++ b/lntest/bitcoind.go @@ -1,4 +1,5 @@ // +build bitcoind +// +build !notxindex package lntest diff --git a/lntest/bitcoind_notxindex.go b/lntest/bitcoind_notxindex.go new file mode 100644 index 00000000..1d565d8c --- /dev/null +++ b/lntest/bitcoind_notxindex.go @@ -0,0 +1,22 @@ +// +build bitcoind +// +build notxindex + +package lntest + +import ( + "github.com/btcsuite/btcd/chaincfg" +) + +// NewBackend starts a bitcoind node without the txindex enabled and returns a +// BitoindBackendConfig for that node. +func NewBackend(miner string, netParams *chaincfg.Params) ( + *BitcoindBackendConfig, func() error, error) { + + extraArgs := []string{ + "-debug", + "-regtest", + "-disablewallet", + } + + return newBackend(miner, netParams, extraArgs) +} From 4dcdfd98bb9b9b62e74dc1204bf301b36fc8ab0b Mon Sep 17 00:00:00 2001 From: Oliver Gugger Date: Fri, 9 Oct 2020 13:35:06 +0200 Subject: [PATCH 4/4] travis: add bitcoind test without txindex --- .travis.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 5479a832..feed17cc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,11 +52,16 @@ jobs: script: - make itest - - name: Bitcoind Integration + - name: Bitcoind Integration (txindex enabled) script: - bash ./scripts/install_bitcoind.sh - make itest backend=bitcoind + - name: Bitcoind Integration (txindex disabled) + script: + - bash ./scripts/install_bitcoind.sh + - make itest backend="bitcoind notxindex" + - name: Neutrino Integration script: - make itest backend=neutrino