Merge pull request #965 from cfromknecht/makefile-fix-and-timeout

Switch to Makefile Build System
This commit is contained in:
Olaoluwa Osuntokun 2018-03-31 16:09:59 -07:00 committed by GitHub
commit 29c9d78c63
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 260 additions and 260 deletions

@ -1,34 +1,19 @@
language: go
go:
- 1.9.4
- "1.9.4"
- "1.10"
sudo: required
install:
- sudo add-apt-repository -y ppa:bitcoin/bitcoin -y
- sudo apt-get update -q
- sudo apt-get install bitcoind -y
- GLIDE_TAG=v0.12.3
- GLIDE_DOWNLOAD="https://github.com/Masterminds/glide/releases/download/$GLIDE_TAG/glide-$GLIDE_TAG-linux-amd64.tar.gz"
- curl -L $GLIDE_DOWNLOAD | tar -xvz
- export PATH=$PATH:$PWD/linux-amd64/
- BTCD_VERSION=$(cat Gopkg.toml | grep -A1 btcd | tail -n1 | awk '{ print $3 }' | tr -d '"')
- mkdir -p $GOPATH/src/github.com/roasbeef/
- pushd $GOPATH/src/github.com/roasbeef/
- git clone https://github.com/roasbeef/btcd
- pushd btcd
- git checkout $BTCD_VERSION
- glide install
- go install . ./cmd/...
- popd
- popd
env:
global:
- LOGS=true
matrix:
- RACE=false
- RACE=true
script:
- export PATH=$PATH:$HOME/gopath/bin
- ./gotest.sh
- export PATH=$PATH:$GOPATH/bin
- make travis
after_script:
- find *.log | xargs -I{} sh -c "cat {} | nc termbin.com 9999 | xargs -r0 printf '{} uploaded to %s'"

273
Makefile

@ -1,92 +1,241 @@
PKG := github.com/lightningnetwork/lnd
HAVE_DEP := $(shell command -v dep 2> /dev/null)
ESCPKG := github.com\/lightningnetwork\/lnd
DEP_PKG := github.com/golang/dep/cmd/dep
BTCD_PKG := github.com/roasbeef/btcd
GLIDE_PKG := github.com/Masterminds/glide
GOVERALLS_PKG := github.com/mattn/goveralls
LINT_PKG := gopkg.in/alecthomas/gometalinter.v1
GO_BIN := ${GOPATH}/bin
DEP_BIN := $(GO_BIN)/dep
BTCD_BIN := $(GO_BIN)/btcd
GLIDE_BIN := $(GO_BIN)/glide
GOVERALLS_BIN := $(GO_BIN)/goveralls
LINT_BIN := $(GO_BIN)/gometalinter.v1
HAVE_DEP := $(shell command -v $(DEP_BIN) 2> /dev/null)
HAVE_BTCD := $(shell command -v $(BTCD_BIN) 2> /dev/null)
HAVE_GLIDE := $(shell command -v $(GLIDE_BIN) 2> /dev/null)
HAVE_GOVERALLS := $(shell command -v $(GOVERALLS_BIN) 2> /dev/null)
HAVE_LINTER := $(shell command -v $(LINT_BIN) 2> /dev/null)
BTCD_DIR :=${GOPATH}/src/$(BTCD_PKG)
COMMIT := $(shell git rev-parse HEAD)
LDFLAGS := -ldflags "-X main.Commit=$(COMMIT)"
RM := rm
all: scratch
GLIDE_COMMIT := 84607742b10f492430762d038e954236bbaf23f7
BTCD_COMMIT := $(shell cat Gopkg.toml | \
grep -A1 $(BTCD_PKG) | \
tail -n1 | \
awk '{ print $$3 }' | \
tr -d '"')
GOBUILD := go build -v
GOINSTALL := go install -v
GOTEST := go test -v
# ==============================================================================
# INSTALLATION
# ==============================================================================
GOLIST := go list $(PKG)/... | grep -v '/vendor/'
GOLISTCOVER := $(shell go list -f '{{.ImportPath}}' ./... | sed -e 's/^$(ESCPKG)/./')
GOLISTLINT := $(shell go list -f '{{.Dir}}' ./... | grep -v 'lnrpc')
deps:
ifndef HAVE_DEP
@echo "Fetching dep."
go get -u github.com/golang/dep/cmd/dep
endif
@echo "Building dependencies."
COVER = for dir in $(GOLISTCOVER); do \
$(GOTEST) $(TEST_FLAGS) \
-covermode=count \
-coverprofile=$$dir/profile.tmp $$dir; \
\
if [ -f $$dir/profile.tmp ]; then \
cat $$dir/profile.tmp | \
tail -n +2 >> profile.cov; \
$(RM) $$dir/profile.tmp; \
fi \
done
LINT = $(LINT_BIN) \
--disable-all \
--enable=gofmt \
--enable=vet \
--enable=golint \
--line-length=72 \
--deadline=4m $(GOLISTLINT) 2>&1 | \
grep -v 'ALL_CAPS\|OP_' 2>&1 | \
tee /dev/stderr
CGO_STATUS_QUO := ${CGO_ENABLED}
RM := rm -f
CP := cp
MAKE := make
XARGS := xargs -L 1
GREEN := "\\033[0;32m"
NC := "\\033[0m"
define print
echo $(GREEN)$1$(NC)
endef
include make/testing_flags.mk
default: scratch
all: scratch check install
# ============
# DEPENDENCIES
# ============
$(DEP_BIN):
@$(call print, "Fetching dep.")
go get -u $(DEP_PKG)
$(GLIDE_BIN):
@$(call print, "Fetching glide.")
go get -d $(GLIDE_PKG)
cd ${GOPATH}/src/$(GLIDE_PKG) && git checkout $(GLIDE_COMMIT)
$(GOINSTALL) $(GLIDE_PKG)
$(GOVERALLS_BIN):
@$(call print, "Fetching goveralls.")
go get -u $(GOVERALLS_PKG)
$(LINT_BIN):
@$(call print, "Fetching gometalinter.v1")
go get -u $(LINT_PKG)
$(GOINSTALL) $(LINT_PKG)
dep: $(DEP_BIN)
@$(call print, "Compiling dependencies.")
dep ensure -v
build:
@echo "Building lnd and lncli."
go build -v -o lnd $(LDFLAGS) $(PKG)
go build -v -o lncli $(LDFLAGS) $(PKG)/cmd/lncli
$(BTCD_DIR):
@$(call print, "Fetching btcd.")
go get -d github.com/roasbeef/btcd
btcd: $(GLIDE_BIN) $(BTCD_DIR)
@$(call print, "Compiling btcd dependencies.")
cd $(BTCD_DIR) && git checkout $(BTCD_COMMIT) && glide install
@$(call print, "Installing btcd and btcctl.")
$(GOINSTALL) $(BTCD_PKG)
$(GOINSTALL) $(BTCD_PKG)/cmd/btcctl
# ============
# INSTALLATION
# ============
build:
@$(call print, "Building lnd and lncli.")
$(GOBUILD) -o lnd $(LDFLAGS) $(PKG)
$(GOBUILD) -o lncli $(LDFLAGS) $(PKG)/cmd/lncli
install:
@echo "Installing lnd and lncli."
go install -v $(LDFLAGS) $(PKG)
go install -v $(LDFLAGS) $(PKG)/cmd/lncli
@$(call print, "Installing lnd and lncli.")
$(CP) ./lnd $(LND_BIN)
$(CP) ./lncli $(LNCLI_BIN)
scratch: deps build
scratch: dep build
# ==============================================================================
# =======
# TESTING
# ==============================================================================
# Define the integration test.run filter if the icase argument was provided.
ifneq ($(icase),)
ITESTCASE := -test.run=TestLightningNetworkDaemon/$(icase)
endif
# UNIT_TARGTED is defined iff a specific package and/or unit test case is being
# targeted.
UNIT_TARGETED =
# If specific package is being unit tested, construct the full name of the
# subpackage.
ifneq ($(pkg),)
UNITPKG := $(PKG)/$(pkg)
UNIT_TARGETED = yes
endif
# If a specific unit test case is being target, construct test.run filter.
ifneq ($(case),)
UNITCASE := -test.run=$(case)
UNIT_TARGETED = yes
endif
# If no unit targeting was input, default to running all tests. Otherwise,
# construct the command to run the specific package/test case.
ifndef UNIT_TARGETED
UNIT := go list $(PKG)/... | grep -v '/vendor/' | xargs go test
else
UNIT := go test -test.v $(UNITPKG) $(UNITCASE)
endif
# =======
check: unit itest
itest: install
@echo "Running integration tests."
go test -v -tags rpctest -logoutput $(ITESTCASE)
itest: btcd build
@$(call print, "Running integration tests.")
$(ITEST)
unit:
@echo "Running unit tests."
unit: btcd
@$(call print, "Running unit tests.")
$(UNIT)
unit-cover:
@$(call print, "Running unit coverage tests.")
echo "mode: count" > profile.cov
$(COVER)
unit-race:
@$(call print, "Running unit race tests.")
export CGO_ENABLED=1; env GORACE="history_size=7 halt_on_errors=1" $(UNIT_RACE)
export CGO_ENABLED=$(CGO_STATUS_QUO)
# ==============================================================================
# =============
# FLAKE HUNTING
# =============
flakehunter: build
@$(call print, "Flake hunting integration tests.")
$(ITEST)
while [ $$? -eq 0 ]; do /bin/sh -c "$(ITEST)"; done
flake-unit:
@$(call print, "Flake hunting unit tests.")
$(UNIT) -count=1
while [ $$? -eq 0 ]; do /bin/sh -c "$(UNIT) -count=1"; done
# ======
# TRAVIS
# ======
ifeq ($(RACE), false)
travis: lint scratch itest unit-cover $(GOVERALLS_BIN)
@$(call print, "Sending coverage report.")
$(GOVERALLS_BIN) -coverprofile=profile.cov -service=travis-ci
endif
ifeq ($(RACE), true)
travis: lint dep btcd unit-race
endif
# =========
# UTILITIES
# ==============================================================================
# =========
fmt:
go list $(PKG)/... | grep -v '/vendor/' | xargs go fmt -x
@$(call print, "Formatting source.")
$(GOLIST) | $(XARGS) go fmt -x
lint: $(LINT_BIN)
@$(call print, "Linting source.")
$(LINT_BIN) --install 1> /dev/null
test -z "$($(LINT))"
list:
@$(call print, "Listing commands.")
@$(MAKE) -qp | \
awk -F':' '/^[a-zA-Z0-9][^$$#\/\t=]*:([^=]|$$)/ {split($$1,A,/ /);for(i in A)print A[i]}' | \
grep -v Makefile | \
sort
rpc:
@$(call print, "Compiling protos.")
cd ./lnrpc; ./gen_protos.sh
clean:
@$(call print, "Cleaning source.$(NC)")
$(RM) ./lnd ./lncli
$(RM) -rf vendor
$(RM) -r ./vendor
.PHONY: all deps build install scratch check itest unit fmt clean
.PHONY: all \
btcd\
default \
dep \
build \
install \
scratch \
check \
itest \
unit \
unit-cover \
unit-race \
flakehunter \
flake-unit \
travis \
fmt \
lint \
list \
rpc \
clean

179
gotest.sh

@ -1,179 +0,0 @@
#!/bin/bash
# The script does automatic checking on a Go package and its sub-packages, including:
# 1. gofmt (http://golang.org/cmd/gofmt/)
# 2. golint (https://github.com/golang/lint)
# 3. go vet (http://golang.org/cmd/vet)
# 4. gosimple (https://github.com/dominikh/go-simple)
# 5. unconvert (https://github.com/mdempsky/unconvert)
# 6. race detector (http://blog.golang.org/race-detector)
# 7. test coverage (http://blog.golang.org/cover)
#
# gometalinter (github.com/alecthomas/gometalinter) is used to run each static
# checker.
declare GREEN='\033[0;32m'
declare NC='\033[0m'
print () {
echo -e "${GREEN}$1${NC}"
}
# check_test_ports checks that test lnd ports are not used.
check_test_ports() {
# Make sure that test lnd ports are not used.
o=$(lsof -i :19555,19556 | sed '1d' | awk '{ printf "%s\n", $2 }')
if [ "$o" != "" ]; then
printf "Can't run the lnd tests:\n"
printf "some program is using the test lnd ports (19555 | 19556)\n"
exit 1
fi
}
# test_with_profile run test coverage on each subdirectories and merge the
# coverage profile. Be aware that we are skipping the integration tests, as the
# tool won't gather any useful coverage information from them.
test_with_coverage_profile() {
print "* Running tests with creating coverage profile"
echo "mode: count" > profile.cov
# Standard go tooling behavior is to ignore dirs with leading underscores.
for dir in $(find . -maxdepth 10 \
-not -path './.git*' \
-not -path '*/_*' \
-not -path '.' \
-not -path './cmd*' \
-not -path './release*' \
-not -path './vendor*' \
-type d)
do
if ls $dir/*.go &> /dev/null; then
go test -v -covermode=count -coverprofile=$dir/profile.tmp $dir
if [ -f $dir/profile.tmp ]; then
cat $dir/profile.tmp | tail -n +2 >> profile.cov
rm $dir/profile.tmp
fi
fi
done
if [ "$TRAVIS" == "true" ] ; then
print "* Send test coverage to travis server"
# Make sure goveralls is installed and $GOPATH/bin is in your path.
if [ ! -x "$(type -p goveralls)" ]; then
print "** Install goveralls:"
go get github.com/mattn/goveralls
fi
print "** Submit the test coverage result to coveralls.io"
goveralls -coverprofile=profile.cov -service=travis-ci
fi
}
# test_race_conditions run standard go test without creating coverage
# profile but with race condition checks.
test_race_conditions() {
print "* Running tests with the race condition detector"
test_targets=$(go list ./... | grep -v '/vendor/')
env GORACE="history_size=7 halt_on_error=1" go test -v -p 1 -race $test_targets
}
# lint_check runs static checks.
lint_check() {
print "* Run static checks"
# Make sure gometalinter is installed and $GOPATH/bin is in your path.
if [ ! -x "$(type -p gometalinter.v1)" ]; then
print "** Install gometalinter"
go get -u gopkg.in/alecthomas/gometalinter.v1
gometalinter.v1 --install
fi
# Update metalinter if needed.
gometalinter.v1 --install 1>/dev/null
# Automatic checks
linter_targets=$(go list -f '{{.Dir}}' ./... | grep -v lnrpc)
test -z "$(gometalinter.v1 --disable-all \
--enable=gofmt \
--enable=vet \
--enable=golint \
--line-length=72 \
--deadline=4m $linter_targets 2>&1 | grep -v 'ALL_CAPS\|OP_' 2>&1 | tee /dev/stderr)"
}
set -e
print "====+ Start +===="
# Read input flags and initialize variables
NEED_LINT="false"
NEED_COVERAGE="false"
NEED_RACE="false"
NEED_INSTALL="false"
NEED_LOGS="false"
while getopts "lrcio" flag; do
case "${flag}" in
l) NEED_LINT="true" ;;
r) NEED_RACE="true" ;;
c) NEED_COVERAGE="true" ;;
i) NEED_INSTALL="true" ;;
o) NEED_LOGS="true" ;;
*)
printf '\nUsage: %s [-l] [-r] [-c] [-i] [-o], where:\n' $0
printf ' -l: include code lint check\n'
printf ' -r: run tests with race condition check\n'
printf ' -c: run tests with test coverage\n'
printf ' -i: reinstall project dependencies\n'
printf ' -o: generate logs for spawned lnd instances\n'
exit 1 ;;
esac
done
# remove the options from the positional parameters
shift $(( OPTIND - 1 ))
# Make sure dep is installed and $GOPATH/bin is in your path.
if [ ! -x "$(type -p dep)" ]; then
print "* Install dep"
go get -u github.com/golang/dep/cmd/dep
fi
# Install the dependency if vendor directory not exist or if flag have been
# specified.
if [ "$NEED_INSTALL" == "true" ] || [ ! -d "./vendor" ]; then
print "* Install dependencies"
dep ensure
fi
# Required for the integration tests
print "* Build source"
go install -v . ./cmd/...
# Lint check is first because we shouldn't run tests on garbage code.
if [ "$NEED_LINT" == "true" ] || [ "$RACE" == "false" ]; then
lint_check
fi
# Race condition second because we shouldn't check coverage on buggy code.
if [ "$NEED_RACE" == "true" ] || [ "$RACE" == "true" ]; then
test_race_conditions
fi
# Test coverage is third because in this case code should work properly and
# we may calmly send coverage profile (if script is run on travis)
if [ "$NEED_COVERAGE" == "true" ] || [ "$RACE" == "false" ]; then
print "* Running integration tests"
LOGOUTPUT_FLAG=""
if [ "$NEED_LOGS" == "true" ] || [ "$LOGS" == "true" ]; then
LOGOUTPUT_FLAG="-logoutput=true"
fi
go test -v -tags rpctest "$LOGOUTPUT_FLAG"
test_with_coverage_profile
fi
print "=====+ End +====="

@ -245,7 +245,7 @@ func (hn *HarnessNode) start(lndError chan<- error) error {
args := hn.cfg.genArgs()
args = append(args, fmt.Sprintf("--profile=%d", 9000+hn.NodeID))
hn.cmd = exec.Command("lnd", args...)
hn.cmd = exec.Command("./lnd", args...)
// Redirect stderr output to buffer
var errb bytes.Buffer

45
make/testing_flags.mk Normal file

@ -0,0 +1,45 @@
TEST_FLAGS =
# If specific package is being unit tested, construct the full name of the
# subpackage.
ifneq ($(pkg),)
UNITPKG := $(PKG)/$(pkg)
UNIT_TARGETED = yes
endif
# If a specific unit test case is being target, construct test.run filter.
ifneq ($(case),)
TEST_FLAGS += -test.run=$(case)
UNIT_TARGETED = yes
endif
# Define the integration test.run filter if the icase argument was provided.
ifneq ($(icase),)
TEST_FLAGS += -test.run=TestLightningNetworkDaemon/$(icase)
endif
# If a timeout was requested, construct initialize the proper flag for the go
# test command.
ifneq ($(timeout),)
TEST_FLAGS += -test.timeout=$(timeout)
endif
# UNIT_TARGTED is undefined iff a specific package and/or unit test case is
# not being targeted.
UNIT_TARGETED ?= no
# If a specific package/test case was requested, run the unit test for the
# targeted case. Otherwise, default to running all tests.
ifeq ($(UNIT_TARGETED), yes)
UNIT := $(GOTEST) $(TEST_FLAGS) $(UNITPKG)
UNIT_RACE := $(GOTEST) $(TEST_FLAGS) -race $(UNITPKG)
endif
ifeq ($(UNIT_TARGETED), no)
UNIT := $(GOLIST) | $(XARGS) $(GOTEST) $(TEST_FLAGS)
UNIT_RACE := $(UNIT) -race
endif
# Construct the integration test command with the added build flags.
ITEST := $(GOTEST) $(TEST_FLAGS) -tags rpctest -logoutput