travis+make: execute test groups in parallel

.gitignore

@@ -36,6 +36,8 @@ lntest/itest/output*.log
 lntest/itest/pprof*.log
 lntest/itest/.backendlogs
 lntest/itest/.minerlogs
+lntest/itest/lnd-itest
+lntest/itest/.logs-*
 
 cmd/cmd
 *.key

.travis.yml

@@ -50,32 +50,30 @@ jobs:
     - stage: Integration Test
       name: Btcd Integration
       script:
-        - make itest
+        - make itest-parallel
 
     - name: Bitcoind Integration (txindex enabled)
       script:
         - bash ./scripts/install_bitcoind.sh
-        - make itest backend=bitcoind
+        - make itest-parallel backend=bitcoind
 
     - name: Bitcoind Integration (txindex disabled)
       script:
         - bash ./scripts/install_bitcoind.sh
-        - make itest backend="bitcoind notxindex"
+        - make itest-parallel backend="bitcoind notxindex"
 
     - name: Neutrino Integration
       script:
-        - make itest backend=neutrino
+        - make itest-parallel backend=neutrino
 
     - name: Btcd Integration ARM
       script:
-        - GOARM=7 GOARCH=arm GOOS=linux CGO_ENABLED=0 make btcd build-itest
+        - GOARM=7 GOARCH=arm GOOS=linux make itest-parallel
-        - file lnd-itest
-        - GOARM=7 GOARCH=arm GOOS=linux CGO_ENABLED=0 make itest-only
       arch: arm64
 
     - name: Btcd Integration Windows
       script:
-        - make itest-windows
+        - make itest-parallel-windows
       os: windows
       before_install:
         - choco upgrade --no-progress -y make netcat curl findutils
@@ -85,7 +83,8 @@ jobs:
           case $TRAVIS_OS_NAME in
             windows)
               echo "Uploading to termbin.com..."
-              for f in ./lntest/itest/*.log; do cat $f | nc termbin.com 9999 | xargs -r0 printf "$f"' uploaded to %s'; done
+              LOG_FILES=$(find ./lntest/itest -name '*.log')
+              for f in $LOG_FILES; do echo -n $f; cat $f | nc termbin.com 9999 | xargs -r0 printf ' uploaded to %s'; done
               ;;
           esac
 
@@ -97,8 +96,8 @@ after_failure:
         ;;
 
       *)
-        LOG_FILES=./lntest/itest/*.log
+        LOG_FILES=$(find ./lntest/itest -name '*.log')
-        echo "Uploading to termbin.com..." && find $LOG_FILES | xargs -I{} sh -c "cat {} | nc termbin.com 9999 | xargs -r0 printf '{} uploaded to %s'"
+        echo "Uploading to termbin.com..." && for f in $LOG_FILES; do echo -n $f; cat $f | nc termbin.com 9999 | xargs -r0 printf ' uploaded to %s'; done
         echo "Uploading to file.io..." && tar -zcvO $LOG_FILES | curl -s -F 'file=@-;filename=logs.tar.gz' https://file.io | xargs -r0 printf 'logs.tar.gz uploaded to %s\n'
         ;;
     esac

Makefile

@@ -175,6 +175,27 @@ itest-only:
 
 itest: btcd build-itest itest-only
 
+itest-parallel: btcd
+	@$(call print, "Building lnd binary")
+	CGO_ENABLED=0 $(GOBUILD) -tags="$(ITEST_TAGS)" -o lntest/itest/lnd-itest $(ITEST_LDFLAGS) $(PKG)/cmd/lnd
+
+	@$(call print, "Building itest binary for $(backend) backend")
+	CGO_ENABLED=0 $(GOTEST) -v ./lntest/itest -tags="$(DEV_TAGS) $(RPC_TAGS) rpctest $(backend)" -logoutput -goroutinedump -c -o lntest/itest/itest.test
+
+	@$(call print, "Running tests")
+	rm -rf lntest/itest/*.log lntest/itest/.logs-*
+	echo -n "$$(seq 0 $$(expr $(NUM_ITEST_TRANCHES) - 1))" | xargs -P $(NUM_ITEST_TRANCHES) -n 1 -I {} scripts/itest_part.sh {} $(NUM_ITEST_TRANCHES) $(TEST_FLAGS)
+
+itest-parallel-windows: btcd
+	@$(call print, "Building lnd binary")
+	CGO_ENABLED=0 $(GOBUILD) -tags="$(ITEST_TAGS)" -o lntest/itest/lnd-itest.exe $(ITEST_LDFLAGS) $(PKG)/cmd/lnd
+
+	@$(call print, "Building itest binary for $(backend) backend")
+	CGO_ENABLED=0 $(GOTEST) -v ./lntest/itest -tags="$(DEV_TAGS) $(RPC_TAGS) rpctest $(backend)" -logoutput -goroutinedump -c -o lntest/itest/itest.test.exe
+
+	@$(call print, "Running tests")
+	EXEC_SUFFIX=".exe" echo -n "$$(seq 0 $$(expr $(NUM_ITEST_TRANCHES) - 1))" | xargs -P $(NUM_ITEST_TRANCHES) -n 1 -I {} scripts/itest_part.sh {} $(NUM_ITEST_TRANCHES) $(TEST_FLAGS)
+
 itest-windows: btcd build-itest-windows itest-only
 
 unit: btcd

make/testing_flags.mk

@@ -3,12 +3,18 @@ RPC_TAGS = autopilotrpc chainrpc invoicesrpc routerrpc signrpc verrpc walletrpc
 LOG_TAGS =
 TEST_FLAGS =
 COVER_PKG = $$(go list -deps ./... | grep '$(PKG)' | grep -v lnrpc)
+NUM_ITEST_TRANCHES = 6
 
 # If rpc option is set also add all extra RPC tags to DEV_TAGS
 ifneq ($(with-rpc),)
 DEV_TAGS += $(RPC_TAGS)
 endif
 
+# Scale the number of parallel running itest tranches.
+ifneq ($(tranches),)
+NUM_ITEST_TRANCHES = $(tranches)
+endif
+
 # If specific package is being unit tested, construct the full name of the
 # subpackage.
 ifneq ($(pkg),)
@@ -25,7 +31,7 @@ endif
 
 # Define the integration test.run filter if the icase argument was provided.
 ifneq ($(icase),)
-TEST_FLAGS += -test.run=TestLightningNetworkDaemon/$(icase)
+TEST_FLAGS += -test.run="TestLightningNetworkDaemon/.*-of-.*/.*/$(icase)"
 endif
 
 ifneq ($(tags),)

scripts/itest_part.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# Let's work with absolute paths only, we run in the itest directory itself.
+WORKDIR=$(pwd)/lntest/itest
+
+TRANCHE=$1
+NUM_TRANCHES=$2
+
+# Shift the passed parameters by two, giving us all remaining testing flags in
+# the $@ special variable.
+shift
+shift
+
+# Windows insists on having the .exe suffix for an executable, we need to add
+# that here if necessary.
+EXEC="$WORKDIR"/itest.test"$EXEC_SUFFIX"
+LND_EXEC="$WORKDIR"/lnd-itest"$EXEC_SUFFIX"
+echo $EXEC -test.v "$@" -logoutput -goroutinedump -logdir=.logs-tranche$TRANCHE -lndexec=$LND_EXEC -splittranches=$NUM_TRANCHES -runtranche=$TRANCHE
+
+# Exit code 255 causes the parallel jobs to abort, so if one part fails the
+# other is aborted too.
+cd "$WORKDIR" || exit 255
+$EXEC -test.v "$@" -logoutput -goroutinedump -logdir=.logs-tranche$TRANCHE -lndexec=$LND_EXEC -splittranches=$NUM_TRANCHES -runtranche=$TRANCHE || exit 255