track tool dependencies in the repo

.gitignore

@@ -8,6 +8,7 @@ dm-worker.log*
 dm-master.log*
 dmctl.log
 relay_log/*
+/_tools/
 vendor
 */*.DS_Store
 tidb-slow.log

Makefile

@@ -8,17 +8,17 @@ CURDIR   := $(shell pwd)
 GO       := GO111MODULE=on go
 GOBUILD  := CGO_ENABLED=0 $(GO) build
 GOTEST   := CGO_ENABLED=1 $(GO) test
-PACKAGES  := $$(go list ./... | grep -vE 'tests|cmd|vendor|pbmock')
+PACKAGES  := $$(go list ./... | grep -vE 'tests|cmd|vendor|pbmock|_tools')
 PACKAGES_RELAY := $$(go list ./... | grep 'github.com/pingcap/dm/relay')
 PACKAGES_SYNCER := $$(go list ./... | grep 'github.com/pingcap/dm/syncer')
 PACKAGES_PKG_BINLOG := $$(go list ./... | grep 'github.com/pingcap/dm/pkg/binlog')
-PACKAGES_OTHERS  := $$(go list ./... | grep -vE 'tests|cmd|vendor|pbmock|github.com/pingcap/dm/relay|github.com/pingcap/dm/syncer|github.com/pingcap/dm/pkg/binlog')
-FILES    := $$(find . -name "*.go" | grep -vE "vendor")
-TOPDIRS  := $$(ls -d */ | grep -vE "vendor")
+PACKAGES_OTHERS  := $$(go list ./... | grep -vE 'tests|cmd|vendor|pbmock|_tools|github.com/pingcap/dm/relay|github.com/pingcap/dm/syncer|github.com/pingcap/dm/pkg/binlog')
+FILES    := $$(find . -name "*.go" | grep -vE "vendor|_tools")
+TOPDIRS  := $$(ls -d */ | grep -vE "vendor|_tools")
 SHELL    := /usr/bin/env bash
 TEST_DIR := /tmp/dm_test
 FAILPOINT_DIR := $$(for p in $(PACKAGES); do echo $${p\#"github.com/pingcap/dm/"}; done)
-FAILPOINT := bin/failpoint-ctl
+FAILPOINT := retool do failpoint-ctl
 
 RACE_FLAG =
 TEST_RACE_FLAG = -race
@@ -46,7 +46,7 @@ else
 	LDFLAGS += -X "github.com/pingcap/dm/dm/master.SampleConfigFile=$(shell cat dm/master/dm-master.toml | base64)"
 endif
 
-.PHONY: build test unit_test dm_integration_test_build integration_test \
+.PHONY: build retool_setup test unit_test dm_integration_test_build integration_test \
 	coverage check dm-worker dm-master dm-tracer dmctl debug-tools
 
 build: check dm-worker dm-master dm-tracer dmctl dm-portal dm-syncer
@@ -72,50 +72,58 @@ dm-syncer:
 debug-tools:
 	$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/binlog-event-blackhole ./debug-tools/binlog-event-blackhole
 
+retool_setup:
+	@echo "setup retool"
+	go get github.com/twitchtv/retool
+	GO111MODULE=off retool sync
+
+generate_proto: retool_setup
+	./generate-dm.sh
+
+generate_mock: retool_setup
+	./tests/generate-mock.sh
+
 test: unit_test integration_test
 
 define run_unit_test
 	@echo "running unit test for packages:" $(1)
 	bash -x ./tests/wait_for_mysql.sh
 	mkdir -p $(TEST_DIR)
-	which $(FAILPOINT) >/dev/null 2>&1 || $(GOBUILD) -o $(FAILPOINT) github.com/pingcap/failpoint/failpoint-ctl
 	$(FAILPOINT_ENABLE)
 	@export log_level=error; \
 	$(GOTEST) -covermode=atomic -coverprofile="$(TEST_DIR)/cov.$(2).out" $(TEST_RACE_FLAG) $(1) \
 	|| { $(FAILPOINT_DISABLE); exit 1; }
 	$(FAILPOINT_DISABLE)
 endef
 
-unit_test:
+unit_test: retool_setup
 	$(call run_unit_test,$(PACKAGES),unit_test)
 
-unit_test_relay:
+unit_test_relay: retool_setup
 	$(call run_unit_test,$(PACKAGES_RELAY),unit_test_relay)
 
-unit_test_syncer:
+unit_test_syncer: retool_setup
 	$(call run_unit_test,$(PACKAGES_SYNCER),unit_test_syncer)
 
-unit_test_pkg_binlog:
+unit_test_pkg_binlog: retool_setup
 	$(call run_unit_test,$(PACKAGES_PKG_BINLOG),unit_test_pkg_binlog)
 
-unit_test_others:
+unit_test_others: retool_setup
 	$(call run_unit_test,$(PACKAGES_OTHERS),unit_test_others)
 
-check: fmt lint vet terror_check
+check: retool_setup fmt lint vet terror_check
 
 fmt:
 	@echo "gofmt (simplify)"
 	@ gofmt -s -l -w $(FILES) 2>&1 | awk '{print} END{if(NR>0) {exit 1}}'
 
-errcheck:
-	GO111MODULE=off go get github.com/kisielk/errcheck
+errcheck: retool_setup
 	@echo "errcheck"
-	@ errcheck -blank $(PACKAGES) | grep -v "_test\.go" | awk '{print} END{if(NR>0) {exit 1}}'
+	@retool do errcheck -blank $(PACKAGES) | grep -v "_test\.go" | awk '{print} END{if(NR>0) {exit 1}}'
 
-lint:
-	GO111MODULE=off go get golang.org/x/lint/golint
+lint: retool_setup
 	@echo "golint"
-	@ golint -set_exit_status $(PACKAGES)
+	@retool do golint -set_exit_status $(PACKAGES)
 
 vet:
 	$(GO) build -o bin/shadow golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow
@@ -127,8 +135,7 @@ terror_check:
 	@echo "check terror conflict"
 	_utils/terror_gen/check.sh
 
-dm_integration_test_build:
-	which $(FAILPOINT) >/dev/null 2>&1 || $(GOBUILD) -o $(FAILPOINT) github.com/pingcap/failpoint/failpoint-ctl
+dm_integration_test_build: retool_setup
 	$(FAILPOINT_ENABLE)
 	$(GOTEST) -c $(TEST_RACE_FLAG) -cover -covermode=atomic \
 		-coverpkg=github.com/pingcap/dm/... \
@@ -177,13 +184,11 @@ compatibility_test: check_third_party_binary
 coverage_fix_cover_mode:
 	sed -i "s/mode: count/mode: atomic/g" $(TEST_DIR)/cov.*.dmctl.*.out
 
-coverage: coverage_fix_cover_mode
-	GO111MODULE=off go get github.com/zhouqiang-cl/gocovmerge
-	gocovmerge "$(TEST_DIR)"/cov.* | grep -vE ".*.pb.go|.*.__failpoint_binding__.go" > "$(TEST_DIR)/all_cov.out"
-	gocovmerge "$(TEST_DIR)"/cov.unit_test*.out | grep -vE ".*.pb.go|.*.__failpoint_binding__.go" > $(TEST_DIR)/unit_test.out
+coverage: coverage_fix_cover_mode retool_setup
+	retool do gocovmerge "$(TEST_DIR)"/cov.* | grep -vE ".*.pb.go|.*.__failpoint_binding__.go" > "$(TEST_DIR)/all_cov.out"
+	retool do gocovmerge "$(TEST_DIR)"/cov.unit_test*.out | grep -vE ".*.pb.go|.*.__failpoint_binding__.go" > $(TEST_DIR)/unit_test.out
 ifeq ("$(JenkinsCI)", "1")
-	GO111MODULE=off go get github.com/mattn/goveralls
-	@goveralls -coverprofile=$(TEST_DIR)/all_cov.out -service=jenkins-ci -repotoken $(COVERALLS_TOKEN)
+	@retool do goveralls -coverprofile=$(TEST_DIR)/all_cov.out -service=jenkins-ci -repotoken $(COVERALLS_TOKEN)
 	@bash <(curl -s https://codecov.io/bash) -f $(TEST_DIR)/unit_test.out -t $(CODECOV_TOKEN)
 else
 	go tool cover -html "$(TEST_DIR)/all_cov.out" -o "$(TEST_DIR)/all_cov.html"
@@ -198,8 +203,8 @@ check-static:
 	  --enable ineffassign \
 	  ./...
 
-failpoint-enable:
+failpoint-enable: retool_setup
 	$(FAILPOINT_ENABLE)
 
-failpoint-disable:
+failpoint-disable: retool_setup
 	$(FAILPOINT_DISABLE)