From a50d337e4212b6960f8ad9660b81f63647e9e962 Mon Sep 17 00:00:00 2001
From: Oliver Gugger <gugger@gmail.com>
Date: Wed, 4 Nov 2020 11:03:25 +0100
Subject: [PATCH 01/10] lntest: lower initial port, add ApplyPortOffset
 function

To allow running multiple test tranches in parallel, we need a way to
make sure the TCP ports don't collide. We'll work with offsets for the
ports, using a different offset for each tranche.
---
 lntest/node.go | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/lntest/node.go b/lntest/node.go
index cdf0be03..8c72a1ed 100644
--- a/lntest/node.go
+++ b/lntest/node.go
@@ -43,7 +43,7 @@ const (
 	// defaultNodePort is the start of the range for listening ports of
 	// harness nodes. Ports are monotonically increasing starting from this
 	// number and are determined by the results of nextAvailablePort().
-	defaultNodePort = 19555
+	defaultNodePort = 5555
 
 	// logPubKeyBytes is the number of bytes of the node's PubKey that will
 	// be appended to the log file name. The whole PubKey is too long and
@@ -104,6 +104,12 @@ func nextAvailablePort() int {
 	panic("no ports available for listening")
 }
 
+// ApplyPortOffset adds the given offset to the lastPort variable, making it
+// possible to run the tests in parallel without colliding on the same ports.
+func ApplyPortOffset(offset uint32) {
+	_ = atomic.AddUint32(&lastPort, offset)
+}
+
 // generateListeningPorts returns four ints representing ports to listen on
 // designated for the current lightning network test. This returns the next
 // available ports for the p2p, rpc, rest and profiling services.

From 5c04449dfdf53c4ef31dc4f58312172d4f469b7f Mon Sep 17 00:00:00 2001
From: Oliver Gugger <gugger@gmail.com>
Date: Wed, 4 Nov 2020 11:03:26 +0100
Subject: [PATCH 02/10] lntest: use nextAvailablePort for fee service

---
 lntest/fee_service.go | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/lntest/fee_service.go b/lntest/fee_service.go
index 68e7d435..d71dae7d 100644
--- a/lntest/fee_service.go
+++ b/lntest/fee_service.go
@@ -16,9 +16,6 @@ const (
 	// is returned. Requests for higher confirmation targets will fall back
 	// to this.
 	feeServiceTarget = 2
-
-	// feeServicePort is the tcp port on which the service runs.
-	feeServicePort = 16534
 )
 
 // feeService runs a web service that provides fee estimation information.
@@ -40,16 +37,15 @@ type feeEstimates struct {
 
 // startFeeService spins up a go-routine to serve fee estimates.
 func startFeeService() *feeService {
+	port := nextAvailablePort()
 	f := feeService{
-		url: fmt.Sprintf(
-			"http://localhost:%v/fee-estimates.json", feeServicePort,
-		),
+		url: fmt.Sprintf("http://localhost:%v/fee-estimates.json", port),
 	}
 
 	// Initialize default fee estimate.
 	f.Fees = map[uint32]uint32{feeServiceTarget: 50000}
 
-	listenAddr := fmt.Sprintf(":%v", feeServicePort)
+	listenAddr := fmt.Sprintf(":%v", port)
 	f.srv = &http.Server{
 		Addr: listenAddr,
 	}

From 891eb0d63d273b976f0c3f72b35fbb9f0a1643ea Mon Sep 17 00:00:00 2001
From: Oliver Gugger <gugger@gmail.com>
Date: Wed, 4 Nov 2020 11:03:27 +0100
Subject: [PATCH 03/10] lntest: use nextAvailablePort for bitcoind

---
 lntest/bitcoind_common.go | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/lntest/bitcoind_common.go b/lntest/bitcoind_common.go
index b59fdac8..019ac268 100644
--- a/lntest/bitcoind_common.go
+++ b/lntest/bitcoind_common.go
@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"math/rand"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -93,10 +92,10 @@ func newBackend(miner string, netParams *chaincfg.Params, extraArgs []string) (
 			fmt.Errorf("unable to create temp directory: %v", err)
 	}
 
-	zmqBlockPath := "ipc:///" + tempBitcoindDir + "/blocks.socket"
-	zmqTxPath := "ipc:///" + tempBitcoindDir + "/txs.socket"
-	rpcPort := rand.Int()%(65536-1024) + 1024
-	p2pPort := rand.Int()%(65536-1024) + 1024
+	zmqBlockAddr := fmt.Sprintf("tcp://127.0.0.1:%d", nextAvailablePort())
+	zmqTxAddr := fmt.Sprintf("tcp://127.0.0.1:%d", nextAvailablePort())
+	rpcPort := nextAvailablePort()
+	p2pPort := nextAvailablePort()
 
 	cmdArgs := []string{
 		"-datadir=" + tempBitcoindDir,
@@ -106,8 +105,8 @@ func newBackend(miner string, netParams *chaincfg.Params, extraArgs []string) (
 			"220110063096c221be9933c82d38e1",
 		fmt.Sprintf("-rpcport=%d", rpcPort),
 		fmt.Sprintf("-port=%d", p2pPort),
-		"-zmqpubrawblock=" + zmqBlockPath,
-		"-zmqpubrawtx=" + zmqTxPath,
+		"-zmqpubrawblock=" + zmqBlockAddr,
+		"-zmqpubrawtx=" + zmqTxAddr,
 		"-debuglogfile=" + logFile,
 	}
 	cmdArgs = append(cmdArgs, extraArgs...)
@@ -178,8 +177,8 @@ func newBackend(miner string, netParams *chaincfg.Params, extraArgs []string) (
 		rpcHost:      rpcHost,
 		rpcUser:      rpcUser,
 		rpcPass:      rpcPass,
-		zmqBlockPath: zmqBlockPath,
-		zmqTxPath:    zmqTxPath,
+		zmqBlockPath: zmqBlockAddr,
+		zmqTxPath:    zmqTxAddr,
 		p2pPort:      p2pPort,
 		rpcClient:    client,
 		minerAddr:    miner,

From 9bbf134237a11f740cd73b81803be6e96cfb4500 Mon Sep 17 00:00:00 2001
From: Oliver Gugger <gugger@gmail.com>
Date: Wed, 4 Nov 2020 11:03:29 +0100
Subject: [PATCH 04/10] itest: split tests into dynamic tranches

---
 lntest/itest/lnd_test.go               | 82 +++++++++++++++++++++++---
 lntest/itest/lnd_test_list_off_test.go |  2 +-
 lntest/itest/lnd_test_list_on_test.go  |  2 +-
 3 files changed, 75 insertions(+), 11 deletions(-)

diff --git a/lntest/itest/lnd_test.go b/lntest/itest/lnd_test.go
index 5dc089cb..1b1533fb 100644
--- a/lntest/itest/lnd_test.go
+++ b/lntest/itest/lnd_test.go
@@ -6,6 +6,7 @@ import (
 	"crypto/rand"
 	"crypto/sha256"
 	"encoding/hex"
+	"flag"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -53,6 +54,60 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+const (
+	// defaultSplitTranches is the default number of tranches we split the
+	// test cases into.
+	defaultSplitTranches uint = 1
+
+	// defaultRunTranche is the default index of the test cases tranche that
+	// we run.
+	defaultRunTranche uint = 0
+)
+
+var (
+	// testCasesSplitParts is the number of tranches the test cases should
+	// be split into. By default this is set to 1, so no splitting happens.
+	// If this value is increased, then the -runtranche flag must be
+	// specified as well to indicate which part should be run in the current
+	// invocation.
+	testCasesSplitTranches = flag.Uint(
+		"splittranches", defaultSplitTranches, "split the test cases "+
+			"in this many tranches and run the tranche at "+
+			"0-based index specified by the -runtranche flag",
+	)
+
+	// testCasesRunTranche is the 0-based index of the split test cases
+	// tranche to run in the current invocation.
+	testCasesRunTranche = flag.Uint(
+		"runtranche", defaultRunTranche, "run the tranche of the "+
+			"split test cases with the given (0-based) index",
+	)
+)
+
+// getTestCaseSplitTranche returns the sub slice of the test cases that should
+// be run as the current split tranche as well as the index and slice offset of
+// the tranche.
+func getTestCaseSplitTranche() ([]*testCase, uint, uint) {
+	numTranches := defaultSplitTranches
+	if testCasesSplitTranches != nil {
+		numTranches = *testCasesSplitTranches
+	}
+	runTranche := defaultRunTranche
+	if testCasesRunTranche != nil {
+		runTranche = *testCasesRunTranche
+	}
+
+	numCases := uint(len(allTestCases))
+	testsPerTranche := numCases / numTranches
+	trancheOffset := runTranche * testsPerTranche
+	trancheEnd := trancheOffset + testsPerTranche
+	if trancheEnd > numCases || runTranche == numTranches-1 {
+		trancheEnd = numCases
+	}
+
+	return allTestCases[trancheOffset:trancheEnd], runTranche, trancheOffset
+}
+
 func rpcPointToWirePoint(t *harnessTest, chanPoint *lnrpc.ChannelPoint) wire.OutPoint {
 	txid, err := lnd.GetChanPointFundingTxid(chanPoint)
 	if err != nil {
@@ -14098,10 +14153,14 @@ func getPaymentResult(stream routerrpc.Router_SendPaymentV2Client) (
 // programmatically driven network of lnd nodes.
 func TestLightningNetworkDaemon(t *testing.T) {
 	// If no tests are registered, then we can exit early.
-	if len(testsCases) == 0 {
+	if len(allTestCases) == 0 {
 		t.Skip("integration tests not selected with flag 'rpctest'")
 	}
 
+	// Parse testing flags that influence our test execution.
+	testCases, trancheIndex, trancheOffset := getTestCaseSplitTranche()
+	lntest.ApplyPortOffset(uint32(trancheIndex) * 1000)
+
 	ht := newHarnessTest(t, nil)
 
 	// Declare the network harness here to gain access to its
@@ -14149,8 +14208,7 @@ func TestLightningNetworkDaemon(t *testing.T) {
 
 	// Connect chainbackend to miner.
 	require.NoError(
-		t, chainBackend.ConnectMiner(),
-		"failed to connect to miner",
+		t, chainBackend.ConnectMiner(), "failed to connect to miner",
 	)
 
 	binary := itestLndBinary
@@ -14187,7 +14245,8 @@ func TestLightningNetworkDaemon(t *testing.T) {
 				if !more {
 					return
 				}
-				ht.Logf("lnd finished with error (stderr):\n%v", err)
+				ht.Logf("lnd finished with error (stderr):\n%v",
+					err)
 			}
 		}
 	}()
@@ -14210,8 +14269,9 @@ func TestLightningNetworkDaemon(t *testing.T) {
 		ht.Fatalf("unable to set up test lightning network: %v", err)
 	}
 
-	t.Logf("Running %v integration tests", len(testsCases))
-	for _, testCase := range testsCases {
+	// Run the subset of the test cases selected in this tranche.
+	for idx, testCase := range testCases {
+		testCase := testCase
 		logLine := fmt.Sprintf("STARTING ============ %v ============\n",
 			testCase.name)
 
@@ -14232,7 +14292,10 @@ func TestLightningNetworkDaemon(t *testing.T) {
 		// Start every test with the default static fee estimate.
 		lndHarness.SetFeeEstimate(12500)
 
-		success := t.Run(testCase.name, func(t1 *testing.T) {
+		name := fmt.Sprintf("%02d-of-%d/%s/%s",
+			trancheOffset+uint(idx)+1, len(allTestCases),
+			chainBackend.Name(), testCase.name)
+		success := t.Run(name, func(t1 *testing.T) {
 			ht := newHarnessTest(t1, lndHarness)
 			ht.RunTestCase(testCase)
 		})
@@ -14242,8 +14305,9 @@ func TestLightningNetworkDaemon(t *testing.T) {
 		if !success {
 			// Log failure time to help relate the lnd logs to the
 			// failure.
-			t.Logf("Failure time: %v",
-				time.Now().Format("2006-01-02 15:04:05.000"))
+			t.Logf("Failure time: %v", time.Now().Format(
+				"2006-01-02 15:04:05.000",
+			))
 			break
 		}
 	}
diff --git a/lntest/itest/lnd_test_list_off_test.go b/lntest/itest/lnd_test_list_off_test.go
index ae18d5e0..59795f1d 100644
--- a/lntest/itest/lnd_test_list_off_test.go
+++ b/lntest/itest/lnd_test_list_off_test.go
@@ -2,4 +2,4 @@
 
 package itest
 
-var testsCases = []*testCase{}
+var allTestCases = []*testCase{}
diff --git a/lntest/itest/lnd_test_list_on_test.go b/lntest/itest/lnd_test_list_on_test.go
index 98910d22..420331c7 100644
--- a/lntest/itest/lnd_test_list_on_test.go
+++ b/lntest/itest/lnd_test_list_on_test.go
@@ -2,7 +2,7 @@
 
 package itest
 
-var testsCases = []*testCase{
+var allTestCases = []*testCase{
 	{
 		name: "sweep coins",
 		test: testSweepAllCoins,

From f358a4474d2f4ec1901113a33acc8276603b4e87 Mon Sep 17 00:00:00 2001
From: Oliver Gugger <gugger@gmail.com>
Date: Wed, 4 Nov 2020 11:03:30 +0100
Subject: [PATCH 05/10] lntest: add log dir flag

---
 lntest/bitcoind_common.go | 18 +++++++++++-------
 lntest/btcd.go            | 18 +++++++++++-------
 lntest/itest/lnd_test.go  |  6 ++++--
 lntest/node.go            | 37 ++++++++++++++++++++++++++-----------
 4 files changed, 52 insertions(+), 27 deletions(-)

diff --git a/lntest/bitcoind_common.go b/lntest/bitcoind_common.go
index 019ac268..f673400a 100644
--- a/lntest/bitcoind_common.go
+++ b/lntest/bitcoind_common.go
@@ -15,8 +15,8 @@ import (
 	"github.com/btcsuite/btcd/rpcclient"
 )
 
-// logDir is the name of the temporary log directory.
-const logDir = "./.backendlogs"
+// logDirPattern is the pattern of the name of the temporary log directory.
+const logDirPattern = "%s/.backendlogs"
 
 // BitcoindBackendConfig is an implementation of the BackendConfig interface
 // backed by a Bitcoind node.
@@ -73,15 +73,16 @@ func (b BitcoindBackendConfig) Name() string {
 func newBackend(miner string, netParams *chaincfg.Params, extraArgs []string) (
 	*BitcoindBackendConfig, func() error, error) {
 
+	baseLogDir := fmt.Sprintf(logDirPattern, GetLogDir())
 	if netParams != &chaincfg.RegressionNetParams {
 		return nil, nil, fmt.Errorf("only regtest supported")
 	}
 
-	if err := os.MkdirAll(logDir, 0700); err != nil {
+	if err := os.MkdirAll(baseLogDir, 0700); err != nil {
 		return nil, nil, err
 	}
 
-	logFile, err := filepath.Abs(logDir + "/bitcoind.log")
+	logFile, err := filepath.Abs(baseLogDir + "/bitcoind.log")
 	if err != nil {
 		return nil, nil, err
 	}
@@ -128,13 +129,16 @@ func newBackend(miner string, netParams *chaincfg.Params, extraArgs []string) (
 		var errStr string
 		// After shutting down the chain backend, we'll make a copy of
 		// the log file before deleting the temporary log dir.
-		err := CopyFile("./output_bitcoind_chainbackend.log", logFile)
+		logDestination := fmt.Sprintf(
+			"%s/output_bitcoind_chainbackend.log", GetLogDir(),
+		)
+		err := CopyFile(logDestination, logFile)
 		if err != nil {
 			errStr += fmt.Sprintf("unable to copy file: %v\n", err)
 		}
-		if err = os.RemoveAll(logDir); err != nil {
+		if err = os.RemoveAll(baseLogDir); err != nil {
 			errStr += fmt.Sprintf(
-				"cannot remove dir %s: %v\n", logDir, err,
+				"cannot remove dir %s: %v\n", baseLogDir, err,
 			)
 		}
 		if err := os.RemoveAll(tempBitcoindDir); err != nil {
diff --git a/lntest/btcd.go b/lntest/btcd.go
index 11e19d3f..e8b8cac4 100644
--- a/lntest/btcd.go
+++ b/lntest/btcd.go
@@ -14,8 +14,8 @@ import (
 	"github.com/btcsuite/btcd/rpcclient"
 )
 
-// logDir is the name of the temporary log directory.
-const logDir = "./.backendlogs"
+// logDirPattern is the pattern of the name of the temporary log directory.
+const logDirPattern = "%s/.backendlogs"
 
 // temp is used to signal we want to establish a temporary connection using the
 // btcd Node API.
@@ -75,12 +75,13 @@ func (b BtcdBackendConfig) Name() string {
 func NewBackend(miner string, netParams *chaincfg.Params) (
 	*BtcdBackendConfig, func() error, error) {
 
+	baseLogDir := fmt.Sprintf(logDirPattern, GetLogDir())
 	args := []string{
 		"--rejectnonstd",
 		"--txindex",
 		"--trickleinterval=100ms",
 		"--debuglevel=debug",
-		"--logdir=" + logDir,
+		"--logdir=" + baseLogDir,
 		"--nowinservice",
 		// The miner will get banned and disconnected from the node if
 		// its requested data are not found. We add a nobanning flag to
@@ -110,14 +111,17 @@ func NewBackend(miner string, netParams *chaincfg.Params) (
 
 		// After shutting down the chain backend, we'll make a copy of
 		// the log file before deleting the temporary log dir.
-		logFile := logDir + "/" + netParams.Name + "/btcd.log"
-		err := CopyFile("./output_btcd_chainbackend.log", logFile)
+		logFile := baseLogDir + "/" + netParams.Name + "/btcd.log"
+		logDestination := fmt.Sprintf(
+			"%s/output_btcd_chainbackend.log", GetLogDir(),
+		)
+		err := CopyFile(logDestination, logFile)
 		if err != nil {
 			errStr += fmt.Sprintf("unable to copy file: %v\n", err)
 		}
-		if err = os.RemoveAll(logDir); err != nil {
+		if err = os.RemoveAll(baseLogDir); err != nil {
 			errStr += fmt.Sprintf(
-				"cannot remove dir %s: %v\n", logDir, err,
+				"cannot remove dir %s: %v\n", baseLogDir, err,
 			)
 		}
 		if errStr != "" {
diff --git a/lntest/itest/lnd_test.go b/lntest/itest/lnd_test.go
index 1b1533fb..c2ceac58 100644
--- a/lntest/itest/lnd_test.go
+++ b/lntest/itest/lnd_test.go
@@ -2435,7 +2435,7 @@ func testOpenChannelAfterReorg(net *lntest.NetworkHarness, t *harnessTest) {
 	)
 
 	// Set up a new miner that we can use to cause a reorg.
-	tempLogDir := "./.tempminerlogs"
+	tempLogDir := fmt.Sprintf("%s/.tempminerlogs", lntest.GetLogDir())
 	logFilename := "output-open_channel_reorg-temp_miner.log"
 	tempMiner, tempMinerCleanUp, err := lntest.NewMiner(
 		tempLogDir, logFilename,
@@ -14158,6 +14158,8 @@ func TestLightningNetworkDaemon(t *testing.T) {
 	}
 
 	// Parse testing flags that influence our test execution.
+	logDir := lntest.GetLogDir()
+	require.NoError(t, os.MkdirAll(logDir, 0700))
 	testCases, trancheIndex, trancheOffset := getTestCaseSplitTranche()
 	lntest.ApplyPortOffset(uint32(trancheIndex) * 1000)
 
@@ -14176,7 +14178,7 @@ func TestLightningNetworkDaemon(t *testing.T) {
 	// guarantees of getting included in to blocks.
 	//
 	// We will also connect it to our chain backend.
-	minerLogDir := "./.minerlogs"
+	minerLogDir := fmt.Sprintf("%s/.minerlogs", logDir)
 	miner, minerCleanUp, err := lntest.NewMiner(
 		minerLogDir, "output_btcd_miner.log",
 		harnessNetParams, &rpcclient.NotificationHandlers{},
diff --git a/lntest/node.go b/lntest/node.go
index 8c72a1ed..cb368e75 100644
--- a/lntest/node.go
+++ b/lntest/node.go
@@ -70,6 +70,10 @@ var (
 	logOutput = flag.Bool("logoutput", false,
 		"log output from node n to file output-n.log")
 
+	// logSubDir is the default directory where the logs are written to if
+	// logOutput is true.
+	logSubDir = flag.String("logdir", ".", "default dir to write logs to")
+
 	// goroutineDump is a flag that can be set to dump the active
 	// goroutines of test nodes on failure.
 	goroutineDump = flag.Bool("goroutinedump", false,
@@ -110,6 +114,15 @@ func ApplyPortOffset(offset uint32) {
 	_ = atomic.AddUint32(&lastPort, offset)
 }
 
+// GetLogDir returns the passed --logdir flag or the default value if it wasn't
+// set.
+func GetLogDir() string {
+	if logSubDir != nil && *logSubDir != "" {
+		return *logSubDir
+	}
+	return "."
+}
+
 // generateListeningPorts returns four ints representing ports to listen on
 // designated for the current lightning network test. This returns the next
 // available ports for the p2p, rpc, rest and profiling services.
@@ -392,11 +405,9 @@ func NewMiner(logDir, logFilename string, netParams *chaincfg.Params,
 
 		// After shutting down the miner, we'll make a copy of the log
 		// file before deleting the temporary log dir.
-		logFile := fmt.Sprintf(
-			"%s/%s/btcd.log", logDir, netParams.Name,
-		)
-		copyPath := fmt.Sprintf("./%s", logFilename)
-		err := CopyFile(copyPath, logFile)
+		logFile := fmt.Sprintf("%s/%s/btcd.log", logDir, netParams.Name)
+		copyPath := fmt.Sprintf("%s/../%s", logDir, logFilename)
+		err := CopyFile(filepath.Clean(copyPath), logFile)
 		if err != nil {
 			return fmt.Errorf("unable to copy file: %v", err)
 		}
@@ -481,24 +492,28 @@ func (hn *HarnessNode) start(lndBinary string, lndError chan<- error) error {
 	// If the logoutput flag is passed, redirect output from the nodes to
 	// log files.
 	if *logOutput {
-		fileName := fmt.Sprintf("output-%d-%s-%s.log", hn.NodeID,
+		dir := GetLogDir()
+		fileName := fmt.Sprintf("%s/output-%d-%s-%s.log", dir, hn.NodeID,
 			hn.Cfg.Name, hex.EncodeToString(hn.PubKey[:logPubKeyBytes]))
 
 		// If the node's PubKey is not yet initialized, create a temporary
 		// file name. Later, after the PubKey has been initialized, the
 		// file can be moved to its final name with the PubKey included.
 		if bytes.Equal(hn.PubKey[:4], []byte{0, 0, 0, 0}) {
-			fileName = fmt.Sprintf("output-%d-%s-tmp__.log", hn.NodeID,
-				hn.Cfg.Name)
+			fileName = fmt.Sprintf("%s/output-%d-%s-tmp__.log",
+				dir, hn.NodeID, hn.Cfg.Name)
 
 			// Once the node has done its work, the log file can be renamed.
 			finalizeLogfile = func() {
 				if hn.logFile != nil {
 					hn.logFile.Close()
 
-					newFileName := fmt.Sprintf("output-%d-%s-%s.log",
-						hn.NodeID, hn.Cfg.Name,
-						hex.EncodeToString(hn.PubKey[:logPubKeyBytes]))
+					pubKeyHex := hex.EncodeToString(
+						hn.PubKey[:logPubKeyBytes],
+					)
+					newFileName := fmt.Sprintf("%s/output"+
+						"-%d-%s-%s.log", dir, hn.NodeID,
+						hn.Cfg.Name, pubKeyHex)
 					err := os.Rename(fileName, newFileName)
 					if err != nil {
 						fmt.Printf("could not rename "+

From ca7564e4b4504cacfe63ab098bef516d5bbeb84c Mon Sep 17 00:00:00 2001
From: Oliver Gugger <gugger@gmail.com>
Date: Wed, 4 Nov 2020 11:03:31 +0100
Subject: [PATCH 06/10] itest: add flags for lnd executable

---
 lntest/itest/lnd_test.go     | 17 +----------------
 lntest/itest/test_harness.go | 34 ++++++++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 16 deletions(-)

diff --git a/lntest/itest/lnd_test.go b/lntest/itest/lnd_test.go
index c2ceac58..e49c7090 100644
--- a/lntest/itest/lnd_test.go
+++ b/lntest/itest/lnd_test.go
@@ -14,7 +14,6 @@ import (
 	"os"
 	"path/filepath"
 	"reflect"
-	"runtime"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -14213,23 +14212,9 @@ func TestLightningNetworkDaemon(t *testing.T) {
 		t, chainBackend.ConnectMiner(), "failed to connect to miner",
 	)
 
-	binary := itestLndBinary
-	if runtime.GOOS == "windows" {
-		// Windows (even in a bash like environment like git bash as on
-		// Travis) doesn't seem to like relative paths to exe files...
-		currentDir, err := os.Getwd()
-		if err != nil {
-			ht.Fatalf("unable to get working directory: %v", err)
-		}
-		targetPath := filepath.Join(currentDir, "../../lnd-itest.exe")
-		binary, err = filepath.Abs(targetPath)
-		if err != nil {
-			ht.Fatalf("unable to get absolute path: %v", err)
-		}
-	}
-
 	// Now we can set up our test harness (LND instance), with the chain
 	// backend we just created.
+	binary := ht.getLndBinary()
 	lndHarness, err = lntest.NewNetworkHarness(miner, chainBackend, binary)
 	if err != nil {
 		ht.Fatalf("unable to create lightning network harness: %v", err)
diff --git a/lntest/itest/test_harness.go b/lntest/itest/test_harness.go
index a3c47528..45248f60 100644
--- a/lntest/itest/test_harness.go
+++ b/lntest/itest/test_harness.go
@@ -3,8 +3,12 @@ package itest
 import (
 	"bytes"
 	"context"
+	"flag"
 	"fmt"
 	"math"
+	"os"
+	"path/filepath"
+	"runtime"
 	"testing"
 	"time"
 
@@ -20,6 +24,11 @@ import (
 
 var (
 	harnessNetParams = &chaincfg.RegressionNetParams
+
+	// lndExecutable is the full path to the lnd binary.
+	lndExecutable = flag.String(
+		"lndexec", itestLndBinary, "full path to lnd binary",
+	)
 )
 
 const (
@@ -111,6 +120,31 @@ func (h *harnessTest) Log(args ...interface{}) {
 	h.t.Log(args...)
 }
 
+func (h *harnessTest) getLndBinary() string {
+	binary := itestLndBinary
+	lndExec := ""
+	if lndExecutable != nil && *lndExecutable != "" {
+		lndExec = *lndExecutable
+	}
+	if lndExec == "" && runtime.GOOS == "windows" {
+		// Windows (even in a bash like environment like git bash as on
+		// Travis) doesn't seem to like relative paths to exe files...
+		currentDir, err := os.Getwd()
+		if err != nil {
+			h.Fatalf("unable to get working directory: %v", err)
+		}
+		targetPath := filepath.Join(currentDir, "../../lnd-itest.exe")
+		binary, err = filepath.Abs(targetPath)
+		if err != nil {
+			h.Fatalf("unable to get absolute path: %v", err)
+		}
+	} else if lndExec != "" {
+		binary = lndExec
+	}
+
+	return binary
+}
+
 type testCase struct {
 	name string
 	test func(net *lntest.NetworkHarness, t *harnessTest)

From d4068e989282c47665fc15d90e3825dc1a00cab4 Mon Sep 17 00:00:00 2001
From: Oliver Gugger <gugger@gmail.com>
Date: Wed, 4 Nov 2020 11:03:33 +0100
Subject: [PATCH 07/10] travis+make: execute test groups in parallel

---
 .gitignore            |  2 ++
 .travis.yml           | 21 ++++++++++-----------
 Makefile              | 21 +++++++++++++++++++++
 make/testing_flags.mk |  8 +++++++-
 scripts/itest_part.sh | 23 +++++++++++++++++++++++
 5 files changed, 63 insertions(+), 12 deletions(-)
 create mode 100755 scripts/itest_part.sh

diff --git a/.gitignore b/.gitignore
index f23749bd..371b57b6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -36,6 +36,8 @@ lntest/itest/output*.log
 lntest/itest/pprof*.log
 lntest/itest/.backendlogs
 lntest/itest/.minerlogs
+lntest/itest/lnd-itest
+lntest/itest/.logs-*
 
 cmd/cmd
 *.key
diff --git a/.travis.yml b/.travis.yml
index feed17cc..51cf9402 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -50,32 +50,30 @@ jobs:
     - stage: Integration Test
       name: Btcd Integration
       script:
-        - make itest
+        - make itest-parallel
 
     - name: Bitcoind Integration (txindex enabled)
       script:
         - bash ./scripts/install_bitcoind.sh
-        - make itest backend=bitcoind
+        - make itest-parallel backend=bitcoind
 
     - name: Bitcoind Integration (txindex disabled)
       script:
         - bash ./scripts/install_bitcoind.sh
-        - make itest backend="bitcoind notxindex"
+        - make itest-parallel backend="bitcoind notxindex"
 
     - name: Neutrino Integration
       script:
-        - make itest backend=neutrino
+        - make itest-parallel backend=neutrino
 
     - name: Btcd Integration ARM
       script:
-        - GOARM=7 GOARCH=arm GOOS=linux CGO_ENABLED=0 make btcd build-itest
-        - file lnd-itest
-        - GOARM=7 GOARCH=arm GOOS=linux CGO_ENABLED=0 make itest-only
+        - GOARM=7 GOARCH=arm GOOS=linux make itest-parallel
       arch: arm64
 
     - name: Btcd Integration Windows
       script:
-        - make itest-windows
+        - make itest-parallel-windows
       os: windows
       before_install:
         - choco upgrade --no-progress -y make netcat curl findutils
@@ -85,7 +83,8 @@ jobs:
           case $TRAVIS_OS_NAME in
             windows)
               echo "Uploading to termbin.com..."
-              for f in ./lntest/itest/*.log; do cat $f | nc termbin.com 9999 | xargs -r0 printf "$f"' uploaded to %s'; done
+              LOG_FILES=$(find ./lntest/itest -name '*.log')
+              for f in $LOG_FILES; do echo -n $f; cat $f | nc termbin.com 9999 | xargs -r0 printf ' uploaded to %s'; done
               ;;
           esac
 
@@ -97,8 +96,8 @@ after_failure:
         ;;
 
       *)
-        LOG_FILES=./lntest/itest/*.log
-        echo "Uploading to termbin.com..." && find $LOG_FILES | xargs -I{} sh -c "cat {} | nc termbin.com 9999 | xargs -r0 printf '{} uploaded to %s'"
+        LOG_FILES=$(find ./lntest/itest -name '*.log')
+        echo "Uploading to termbin.com..." && for f in $LOG_FILES; do echo -n $f; cat $f | nc termbin.com 9999 | xargs -r0 printf ' uploaded to %s'; done
         echo "Uploading to file.io..." && tar -zcvO $LOG_FILES | curl -s -F 'file=@-;filename=logs.tar.gz' https://file.io | xargs -r0 printf 'logs.tar.gz uploaded to %s\n'
         ;;
     esac
diff --git a/Makefile b/Makefile
index 5f55f0bc..ec0e3227 100644
--- a/Makefile
+++ b/Makefile
@@ -175,6 +175,27 @@ itest-only:
 
 itest: btcd build-itest itest-only
 
+itest-parallel: btcd
+	@$(call print, "Building lnd binary")
+	CGO_ENABLED=0 $(GOBUILD) -tags="$(ITEST_TAGS)" -o lntest/itest/lnd-itest $(ITEST_LDFLAGS) $(PKG)/cmd/lnd
+
+	@$(call print, "Building itest binary for $(backend) backend")
+	CGO_ENABLED=0 $(GOTEST) -v ./lntest/itest -tags="$(DEV_TAGS) $(RPC_TAGS) rpctest $(backend)" -logoutput -goroutinedump -c -o lntest/itest/itest.test
+
+	@$(call print, "Running tests")
+	rm -rf lntest/itest/*.log lntest/itest/.logs-*
+	echo -n "$$(seq 0 $$(expr $(NUM_ITEST_TRANCHES) - 1))" | xargs -P $(NUM_ITEST_TRANCHES) -n 1 -I {} scripts/itest_part.sh {} $(NUM_ITEST_TRANCHES) $(TEST_FLAGS)
+
+itest-parallel-windows: btcd
+	@$(call print, "Building lnd binary")
+	CGO_ENABLED=0 $(GOBUILD) -tags="$(ITEST_TAGS)" -o lntest/itest/lnd-itest.exe $(ITEST_LDFLAGS) $(PKG)/cmd/lnd
+
+	@$(call print, "Building itest binary for $(backend) backend")
+	CGO_ENABLED=0 $(GOTEST) -v ./lntest/itest -tags="$(DEV_TAGS) $(RPC_TAGS) rpctest $(backend)" -logoutput -goroutinedump -c -o lntest/itest/itest.test.exe
+
+	@$(call print, "Running tests")
+	EXEC_SUFFIX=".exe" echo -n "$$(seq 0 $$(expr $(NUM_ITEST_TRANCHES) - 1))" | xargs -P $(NUM_ITEST_TRANCHES) -n 1 -I {} scripts/itest_part.sh {} $(NUM_ITEST_TRANCHES) $(TEST_FLAGS)
+
 itest-windows: btcd build-itest-windows itest-only
 
 unit: btcd
diff --git a/make/testing_flags.mk b/make/testing_flags.mk
index 1443ab5b..f64d859c 100644
--- a/make/testing_flags.mk
+++ b/make/testing_flags.mk
@@ -3,12 +3,18 @@ RPC_TAGS = autopilotrpc chainrpc invoicesrpc routerrpc signrpc verrpc walletrpc
 LOG_TAGS =
 TEST_FLAGS =
 COVER_PKG = $$(go list -deps ./... | grep '$(PKG)' | grep -v lnrpc)
+NUM_ITEST_TRANCHES = 6
 
 # If rpc option is set also add all extra RPC tags to DEV_TAGS
 ifneq ($(with-rpc),)
 DEV_TAGS += $(RPC_TAGS)
 endif
 
+# Scale the number of parallel running itest tranches.
+ifneq ($(tranches),)
+NUM_ITEST_TRANCHES = $(tranches)
+endif
+
 # If specific package is being unit tested, construct the full name of the
 # subpackage.
 ifneq ($(pkg),)
@@ -25,7 +31,7 @@ endif
 
 # Define the integration test.run filter if the icase argument was provided.
 ifneq ($(icase),)
-TEST_FLAGS += -test.run=TestLightningNetworkDaemon/$(icase)
+TEST_FLAGS += -test.run="TestLightningNetworkDaemon/.*-of-.*/.*/$(icase)"
 endif
 
 ifneq ($(tags),)
diff --git a/scripts/itest_part.sh b/scripts/itest_part.sh
new file mode 100755
index 00000000..52c3481c
--- /dev/null
+++ b/scripts/itest_part.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Let's work with absolute paths only, we run in the itest directory itself.
+WORKDIR=$(pwd)/lntest/itest
+
+TRANCHE=$1
+NUM_TRANCHES=$2
+
+# Shift the passed parameters by two, giving us all remaining testing flags in
+# the $@ special variable.
+shift
+shift
+
+# Windows insists on having the .exe suffix for an executable, we need to add
+# that here if necessary.
+EXEC="$WORKDIR"/itest.test"$EXEC_SUFFIX"
+LND_EXEC="$WORKDIR"/lnd-itest"$EXEC_SUFFIX"
+echo $EXEC -test.v "$@" -logoutput -goroutinedump -logdir=.logs-tranche$TRANCHE -lndexec=$LND_EXEC -splittranches=$NUM_TRANCHES -runtranche=$TRANCHE
+
+# Exit code 255 causes the parallel jobs to abort, so if one part fails the
+# other is aborted too.
+cd "$WORKDIR" || exit 255
+$EXEC -test.v "$@" -logoutput -goroutinedump -logdir=.logs-tranche$TRANCHE -lndexec=$LND_EXEC -splittranches=$NUM_TRANCHES -runtranche=$TRANCHE || exit 255

From ea4bb5dc5ce1a028811617ecd74197f296890c32 Mon Sep 17 00:00:00 2001
From: Oliver Gugger <gugger@gmail.com>
Date: Wed, 4 Nov 2020 11:03:34 +0100
Subject: [PATCH 08/10] itest: fix chanbackup restore flake

Updating the fee of the mock estimator _after_ starting carol turned out
to be flaky and could lead to the new fee not being picked up in time
for the force close. That lead to carol not cpfp'ing the force closed
transaction.
---
 lntest/itest/lnd_channel_backup_test.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lntest/itest/lnd_channel_backup_test.go b/lntest/itest/lnd_channel_backup_test.go
index 5a8bf87d..2a7e0eb6 100644
--- a/lntest/itest/lnd_channel_backup_test.go
+++ b/lntest/itest/lnd_channel_backup_test.go
@@ -1012,6 +1012,10 @@ func testChanRestoreScenario(t *harnessTest, net *lntest.NetworkHarness,
 	require.Contains(t.t, err.Error(), "cannot close channel with state: ")
 	require.Contains(t.t, err.Error(), "ChanStatusRestored")
 
+	// Increase the fee estimate so that the following force close tx will
+	// be cpfp'ed in case of anchor commitments.
+	net.SetFeeEstimate(30000)
+
 	// Now that we have ensured that the channels restored by the backup are
 	// in the correct state even without the remote peer telling us so,
 	// let's start up Carol again.

From e6c47294fb770157d4650cff8040ddb4dd048b19 Mon Sep 17 00:00:00 2001
From: Oliver Gugger <gugger@gmail.com>
Date: Wed, 4 Nov 2020 11:03:36 +0100
Subject: [PATCH 09/10] itest: fix typo and formatting

---
 .../lnd_multi-hop_htlc_remote_chain_claim_test.go      | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/lntest/itest/lnd_multi-hop_htlc_remote_chain_claim_test.go b/lntest/itest/lnd_multi-hop_htlc_remote_chain_claim_test.go
index 1aeff01e..72a3c63c 100644
--- a/lntest/itest/lnd_multi-hop_htlc_remote_chain_claim_test.go
+++ b/lntest/itest/lnd_multi-hop_htlc_remote_chain_claim_test.go
@@ -101,15 +101,17 @@ func testMultiHopHtlcRemoteChainClaim(net *lntest.NetworkHarness, t *harnessTest
 	// bob will attempt to redeem his anchor commitment (if the channel
 	// type is of that type).
 	if c == commitTypeAnchors {
-		_, err = waitForNTxsInMempool(net.Miner.Node, 1, minerMempoolTimeout)
+		_, err = waitForNTxsInMempool(
+			net.Miner.Node, 1, minerMempoolTimeout,
+		)
 		if err != nil {
-			t.Fatalf("unable to find bob's anchor commit sweep: %v", err)
-
+			t.Fatalf("unable to find bob's anchor commit sweep: %v",
+				err)
 		}
 	}
 
 	// Mine enough blocks for Alice to sweep her funds from the force
-	// closed channel. closeCHannelAndAssertType() already mined a block
+	// closed channel. closeChannelAndAssertType() already mined a block
 	// containing the commitment tx and the commit sweep tx will be
 	// broadcast immediately before it can be included in a block, so mine
 	// one less than defaultCSV in order to perform mempool assertions.

From a606f462bc1748b03d1faf7d23f72e0bdebc0a54 Mon Sep 17 00:00:00 2001
From: Oliver Gugger <gugger@gmail.com>
Date: Wed, 4 Nov 2020 11:03:37 +0100
Subject: [PATCH 10/10] itest: move longest test to beginning

To make sure the test that takes the longest overall time is always
started first, independent of the number of test tranches we run, we
move it to the beginning of the list. Because that test involves a lot
of waiting, it allows us to play around with the number of tranches more
efficiently.
---
 lntest/itest/lnd_test_list_on_test.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lntest/itest/lnd_test_list_on_test.go b/lntest/itest/lnd_test_list_on_test.go
index 420331c7..3575213c 100644
--- a/lntest/itest/lnd_test_list_on_test.go
+++ b/lntest/itest/lnd_test_list_on_test.go
@@ -3,6 +3,10 @@
 package itest
 
 var allTestCases = []*testCase{
+	{
+		name: "test multi-hop htlc",
+		test: testMultiHopHtlcClaims,
+	},
 	{
 		name: "sweep coins",
 		test: testSweepAllCoins,
@@ -144,10 +148,6 @@ var allTestCases = []*testCase{
 		name: "async bidirectional payments",
 		test: testBidirectionalAsyncPayments,
 	},
-	{
-		name: "test multi-hop htlc",
-		test: testMultiHopHtlcClaims,
-	},
 	{
 		name: "switch circuit persistence",
 		test: testSwitchCircuitPersistence,