Merge pull request #4756 from guggero/itest-flakehunt

itest: add parallel flakehunter

Makefile

@@ -184,7 +184,7 @@ itest-parallel: btcd
 
 	@$(call print, "Running tests")
 	rm -rf lntest/itest/*.log lntest/itest/.logs-*
-	echo -n "$$(seq 0 $$(expr $(NUM_ITEST_TRANCHES) - 1))" | xargs -P $(NUM_ITEST_TRANCHES) -n 1 -I {} scripts/itest_part.sh {} $(NUM_ITEST_TRANCHES) $(TEST_FLAGS)
+	echo "$$(seq 0 $$(expr $(ITEST_PARALLELISM) - 1))" | xargs -P $(ITEST_PARALLELISM) -n 1 -I {} scripts/itest_part.sh {} $(NUM_ITEST_TRANCHES) $(TEST_FLAGS)
 
 itest-parallel-windows: btcd
 	@$(call print, "Building lnd binary")
@@ -194,7 +194,7 @@ itest-parallel-windows: btcd
 	CGO_ENABLED=0 $(GOTEST) -v ./lntest/itest -tags="$(DEV_TAGS) $(RPC_TAGS) rpctest $(backend)" -logoutput -goroutinedump -c -o lntest/itest/itest.test.exe
 
 	@$(call print, "Running tests")
-	EXEC_SUFFIX=".exe" echo -n "$$(seq 0 $$(expr $(NUM_ITEST_TRANCHES) - 1))" | xargs -P $(NUM_ITEST_TRANCHES) -n 1 -I {} scripts/itest_part.sh {} $(NUM_ITEST_TRANCHES) $(TEST_FLAGS)
+	EXEC_SUFFIX=".exe" echo "$$(seq 0 $$(expr $(ITEST_PARALLELISM) - 1))" | xargs -P $(ITEST_PARALLELISM) -n 1 -I {} scripts/itest_part.sh {} $(NUM_ITEST_TRANCHES) $(TEST_FLAGS)
 
 itest-windows: btcd build-itest-windows itest-only
 
@@ -232,6 +232,10 @@ flake-unit:
 	@$(call print, "Flake hunting unit tests.")
 	while [ $$? -eq 0 ]; do GOTRACEBACK=all $(UNIT) -count=1; done
 
+flakehunter-parallel:
+	@$(call print, "Flake hunting ${backend} integration tests in parallel.")
+	while [ $$? -eq 0 ]; do make itest-parallel tranches=1 parallel=${ITEST_PARALLELISM} icase='${icase}' backend='${backend}'; done
+
 # =============
 # FUZZING
 # =============

docs/MAKEFILE.md

@@ -121,6 +121,32 @@ Arguments:
 - `icase=<itestcase>` (the snake_case version of the testcase name field in the testCases slice (i.e. sweep_coins), not the test func name)
 - `timeout=<timeout>`
 
+`itest-parallel`
+------
+Does the same as `itest` but splits the total set of tests into
+`NUM_ITEST_TRANCHES` tranches (currently set to 6 by default, can be overwritten
+by setting `tranches=Y`) and runs them in parallel.
+
+Arguments:
+- `icase=<itestcase>`: The snake_case version of the testcase name field in the
+  testCases slice (i.e. `sweep_coins`, not the test func name) or any regular
+  expression describing a set of tests.
+- `timeout=<timeout>`
+- `tranches=<number_of_tranches>`: The number of parts/tranches to split the
+  total set of tests into.
+- `parallel=<number_of_threads>`: The number of threads to run in parallel. Must
+  be greater or equal to `tranches`, otherwise undefined behavior is expected.
+
+`flakehunter-parallel`
+------
+Runs the test specified by `icase` simultaneously `parallel` (default=6) times
+until an error occurs. Useful for hunting flakes.
+
+Example:
+```shell
+$ make flakehunter-parallel icase='(data_loss_protection|channel_backup)' backend=neutrino
+```
+
 `lint`
 ------
 Ensures that [`gopkg.in/alecthomas/gometalinter.v1`][gometalinter] is

lntest/itest/lnd_test.go

@@ -96,6 +96,15 @@ func getTestCaseSplitTranche() ([]*testCase, uint, uint) {
 		runTranche = *testCasesRunTranche
 	}
 
+	// There's a special flake-hunt mode where we run the same test multiple
+	// times in parallel. In that case the tranche index is equal to the
+	// thread ID, but we need to actually run all tests for the regex
+	// selection to work.
+	threadID := runTranche
+	if numTranches == 1 {
+		runTranche = 0
+	}
+
 	numCases := uint(len(allTestCases))
 	testsPerTranche := numCases / numTranches
 	trancheOffset := runTranche * testsPerTranche
@@ -104,7 +113,7 @@ func getTestCaseSplitTranche() ([]*testCase, uint, uint) {
 		trancheEnd = numCases
 	}
 
-	return allTestCases[trancheOffset:trancheEnd], runTranche, trancheOffset
+	return allTestCases[trancheOffset:trancheEnd], threadID, trancheOffset
 }
 
 func rpcPointToWirePoint(t *harnessTest, chanPoint *lnrpc.ChannelPoint) wire.OutPoint {

make/testing_flags.mk

@@ -4,6 +4,7 @@ LOG_TAGS =
 TEST_FLAGS =
 COVER_PKG = $$(go list -deps ./... | grep '$(PKG)' | grep -v lnrpc)
 NUM_ITEST_TRANCHES = 6
+ITEST_PARALLELISM = $(NUM_ITEST_TRANCHES)
 
 # If rpc option is set also add all extra RPC tags to DEV_TAGS
 ifneq ($(with-rpc),)
@@ -13,6 +14,12 @@ endif
 # Scale the number of parallel running itest tranches.
 ifneq ($(tranches),)
 NUM_ITEST_TRANCHES = $(tranches)
+ITEST_PARALLELISM = $(NUM_ITEST_TRANCHES)
+endif
+
+# Give the ability to run the same tranche multiple times at the same time.
+ifneq ($(parallel),)
+ITEST_PARALLELISM = $(parallel)
 endif
 
 # If specific package is being unit tested, construct the full name of the