vendor: Get rid of ./vendor cleanup

We updated scripts and documentation to not recommend vendoring any
longer and implement best practices of installation.

I performed multiple tests to confirm its not breaking any workflows and
has no negative performance impact:

1. PASSES="fmt unit integration e2e functional" ./test
2. ./scripts/updatebom.sh
3. ./scripts/updatedep.sh
4. ./scripts/genproto.sh - works - ca be simplified - in follow up PR
5. Installation without explicit GOPATH:

```
% unset GOPATH
% [sudo] rm -rf ~/go
% git clone https://github.com/etcd-io/etcd.git
% time ./build
go: downloading google.golang.org/grpc v1.26.0
go: downloading github.com/jonboulle/clockwork v0.1.0
go: downloading github.com/prometheus/client_golang v1.0.0
go: downloading github.com/soheilhy/cmux v0.1.4
go: downloading github.com/gogo/protobuf v1.2.1
go: downloading sigs.k8s.io/yaml v1.1.0
go: downloading golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2
go: downloading github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903
go: downloading go.etcd.io/bbolt v1.3.5
go: downloading go.uber.org/zap v1.15.0
go: downloading golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc
go: downloading github.com/golang/protobuf v1.3.2
go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8
go: downloading github.com/beorn7/perks v1.0.0
go: downloading github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4
go: downloading github.com/coreos/go-systemd/v22 v22.0.0
go: downloading gopkg.in/yaml.v2 v2.2.2
go: downloading github.com/coreos/go-semver v0.2.0
go: downloading github.com/sirupsen/logrus v1.4.2
go: downloading golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7
go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
go: downloading github.com/google/uuid v1.0.0
go: downloading github.com/modern-go/reflect2 v1.0.1
go: downloading github.com/prometheus/common v0.4.1
go: downloading github.com/spf13/cobra v0.0.3
go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2
go: downloading github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c
go: downloading github.com/spf13/pflag v1.0.1
go: downloading github.com/json-iterator/go v1.1.7
go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible
go: downloading github.com/google/btree v1.0.0
go: downloading go.uber.org/atomic v1.6.0
go: downloading github.com/prometheus/procfs v0.0.2
go: downloading go.uber.org/multierr v1.5.0
go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd
go: downloading golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5
go: downloading github.com/grpc-ecosystem/grpc-gateway v1.9.5
go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4
go: downloading github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4
go: downloading google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55
go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1
go: downloading golang.org/x/text v0.3.3
go: downloading github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5
go: downloading github.com/bgentry/speakeasy v0.1.0
go: downloading gopkg.in/cheggaaa/pb.v1 v1.0.25
go: downloading github.com/urfave/cli v1.20.0
go: downloading github.com/mattn/go-runewidth v0.0.2
./build  8.22s user 2.31s system 117% cpu 8.961 total
```

6. Rebuild without changes:

```
% time ./build
./build  1.43s user 0.83s system 168% cpu 1.336 total
```

7. Instantation of vendor directory (assuming ./build loaded them to
$GOPATH/pkg):

```
time go mod vendor
go: downloading github.com/inconshreveable/mousetrap v1.0.0
go: downloading github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa
go: downloading github.com/creack/pty v1.1.11
go: downloading github.com/etcd-io/gofail v0.0.0-20190801230047-ad7f989257ca
go: downloading github.com/konsorten/go-windows-terminal-sequences v1.0.1
go mod vendor  0.51s user 0.44s system 110% cpu 0.861 total
```

8. Fresh instantation of vendor:

```
% rm -rf vendor
% [sudo] rm -rf ~/go

% time go mod vendor
go: downloading github.com/coreos/go-systemd/v22 v22.0.0
go: downloading github.com/spf13/cobra v0.0.3
go: downloading github.com/prometheus/client_golang v1.0.0
go: downloading golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7
go: downloading github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4
go: downloading github.com/gogo/protobuf v1.2.1
go: downloading sigs.k8s.io/yaml v1.1.0
go: downloading google.golang.org/grpc v1.26.0
go: downloading github.com/urfave/cli v1.20.0
go: downloading go.uber.org/zap v1.15.0
go: downloading github.com/spf13/pflag v1.0.1
go: downloading github.com/soheilhy/cmux v0.1.4
go: downloading github.com/json-iterator/go v1.1.7
go: downloading github.com/coreos/go-semver v0.2.0
go: downloading github.com/prometheus/common v0.4.1
go: downloading github.com/prometheus/procfs v0.0.2
go: downloading go.uber.org/atomic v1.6.0
go: downloading github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5
go: downloading github.com/golang/protobuf v1.3.2
go: downloading github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa
go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4
go: downloading github.com/modern-go/reflect2 v1.0.1
go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd
go: downloading go.uber.org/multierr v1.5.0
go: downloading github.com/creack/pty v1.1.11
go: downloading github.com/mattn/go-runewidth v0.0.2
go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
go: downloading golang.org/x/crypto v0.0.0-20191002192127-34f69633bfdc
go: downloading golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5
go: downloading github.com/jonboulle/clockwork v0.1.0
go: downloading gopkg.in/yaml.v2 v2.2.2
go: downloading github.com/etcd-io/gofail v0.0.0-20190801230047-ad7f989257ca
go: downloading github.com/grpc-ecosystem/grpc-gateway v1.9.5
go: downloading github.com/google/btree v1.0.0
go: downloading google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55
go: downloading github.com/beorn7/perks v1.0.0
go: downloading github.com/dgrijalva/jwt-go v3.2.0+incompatible
go: downloading github.com/google/uuid v1.0.0
go: downloading golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2
go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8
go: downloading go.etcd.io/bbolt v1.3.5
go: downloading golang.org/x/text v0.3.3
go: downloading gopkg.in/cheggaaa/pb.v1 v1.0.25
go: downloading github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2
go: downloading github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4
go: downloading github.com/inconshreveable/mousetrap v1.0.0
go: downloading github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c
go: downloading github.com/matttproud/golang_protobuf_extensions v1.0.1
go: downloading github.com/bgentry/speakeasy v0.1.0
go: downloading github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903
go: downloading github.com/sirupsen/logrus v1.4.2
go: downloading github.com/konsorten/go-windows-terminal-sequences v1.0.1
go mod vendor  3.62s user 1.30s system 127% cpu 3.854 total
```

9. Size of the repository - before: 39M, after: 18M

Before:

```
% time git clone https://github.com/etcd-io/etcd.git
Cloning into 'etcd'...
remote: Enumerating objects: 97872, done.
remote: Total 97872 (delta 0), reused 0 (delta 0), pack-reused 97872
Receiving objects: 100% (97872/97872), 58.97 MiB | 20.53 MiB/s, done.
Resolving deltas: 100% (63091/63091), done.
git clone https://github.com/etcd-io/etcd.git  4.66s user 1.02s system 93% cpu 6.068 total

% du -h --exclude .git -d 1
944K	./clientv3
108K	./etcdmain
5.4M	./Documentation
384K	./security
384K	./mvcc
28K	./.github
8.0K	./version
144K	./contrib
240K	./proxy
2.5M	./etcdserver
112K	./embed
536K	./integration
332K	./tools
116K	./lease
108K	./logos
896K	./tests
960K	./raft
216K	./client
52K	./scripts
100K	./hack
464K	./etcdctl
3.0M	./pkg
620K	./functional
136K	./wal
152K	./auth
21M	./vendor
39M
```

After:
```
% time git clone https://github.com/ptabor/etcd.git -b 20200908-no-vendor
Cloning into 'etcd'...
remote: Enumerating objects: 38, done.
remote: Counting objects: 100% (38/38), done.
remote: Compressing objects: 100% (37/37), done.
remote: Total 98489 (delta 10), reused 8 (delta 1), pack-reused 98451
Receiving objects: 100% (98489/98489), 59.23 MiB | 21.26 MiB/s, done.
Resolving deltas: 100% (63572/63572), done.
git clone https://github.com/ptabor/etcd.git -b 20200908-no-vendor  5.56s user 1.05s system 105% cpu 6.260 total

% du -h --exclude .git -d 1
944K	./clientv3
108K	./etcdmain
5.4M	./Documentation
384K	./security
384K	./mvcc
28K	./.github
8.0K	./version
144K	./contrib
240K	./proxy
2.5M	./etcdserver
112K	./embed
536K	./integration
332K	./tools
116K	./lease
108K	./logos
896K	./tests
960K	./raft
216K	./client
56K	./scripts
100K	./hack
464K	./etcdctl
3.0M	./pkg
620K	./functional
136K	./wal
152K	./auth
19M	.
```

.gitignore

@@ -18,4 +18,3 @@ hack/tls-setup/certs
 .vendor
 
 *.bak
-

Documentation/dl-build.md

@@ -22,7 +22,7 @@ $ cd etcd
 $ ./build
 ```
 
-To build a vendored `etcd` from the `master` branch via `go get`:
+To build a modularized `etcd` from the `master` branch via `go get`:
 
 ```sh
 # GOPATH should be set

Makefile

@@ -23,11 +23,9 @@ clean:
 	rm -f ./*.coverprofile
 	rm -f ./*.log
 	rm -f ./bin/Dockerfile-release
-	rm -rf ./bin/*.etcd
+	rm -rf ./bin/etcd*
 	rm -rf ./default.etcd
 	rm -rf ./tests/e2e/default.etcd
-	rm -rf ./gopath
-	rm -rf ./gopath.proto
 	rm -rf ./release
 	rm -f ./snapshot/localhost:*
 	rm -f ./tools/etcd-dump-metrics/localhost:*

client/README.md

@@ -4,7 +4,7 @@ etcd/client is the Go client library for etcd.
 
 [![GoDoc](https://godoc.org/go.etcd.io/etcd/client?status.png)](https://godoc.org/go.etcd.io/etcd/client)
 
-For full compatibility, it is recommended to vendor builds using etcd's vendored packages, using tools like `golang/dep`, as in [vendor directories](https://golang.org/cmd/go/#hdr-Vendor_Directories).
+For full compatibility, it is recommended to install released versions of clients using go modules.
 
 ## Install
 

clientv3/README.md

@@ -41,7 +41,7 @@ if err != nil {
 // use the response
 ```
 
-For full compatibility, it is recommended to vendor builds using etcd's vendored packages, using tools like `golang/dep`, as in [vendor directories](https://golang.org/cmd/go/#hdr-Vendor_Directories).
+For full compatibility, it is recommended to install released versions of clients using go modules.
 
 ## Error Handling
 

scripts/install_tool.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+# Usage ./install_tool.sh {go_module}
+#
+# Install given tool and makes it available on $PATH (assuming standard config),
+# without modification to vendor or go.mod file.
+#
+# When https://github.com/golang/go/issues/40276 is implemented, usage
+# of this script should get replaced by pure: 
+# 
+# go install {go_module}@latest
+# 
+set -e
+
+>&2 echo "installing: $1"
+(cd $(mktemp -d); GO111MODULE=on go get "$1")

scripts/updatebom.sh

@@ -7,10 +7,7 @@ if ! [[ "$0" =~ scripts/updatebom.sh ]]; then
 	exit 255
 fi
 
-export GO111MODULE=off # Don't add BOM tool to etcd go.mod
-echo "installing 'bill-of-materials.json'"
-go get -v -u github.com/coreos/license-bill-of-materials
-export GO111MODULE=on
+./scripts/install_tool.sh github.com/coreos/license-bill-of-materials
 
 echo "generating bill-of-materials.json"
 license-bill-of-materials \

scripts/updatedep.sh

@@ -13,10 +13,16 @@ if [ -d "gopath.proto" ]; then
   exit 255
 fi
 
-if [[ $(go version) != "go version go1.14"* ]]; then
+if [[ $(go version) < "go version go1.14"* ]]; then
   echo "expect Go 1.14+, got:" "$(go version)"
   exit 255
 fi
 
 GO111MODULE=on go mod tidy -v
 GO111MODULE=on go mod vendor -v
+
+RED='\033[0;31m'
+NC='\033[0m' # No Color
+printf "\n${RED} WARNING: In etcd >=3.5 we use go modules rather than vendoring\n" 
+printf "${RED}          Please refactor your logic to depend on modules directly.${NC}\n"
+

test

@@ -459,7 +459,7 @@ function govet_shadow_pass {
 	# shellcheck disable=SC2206
 	fmtpkgs=($fmtpkgs)
 	# Golang 1.12 onwards the experimental -shadow option is no longer available with go vet
-	go get -v golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow
+	./scripts/install_tool.sh golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow
 	export PATH=${GOPATH}/bin:${PATH}
 	shadow_tool=$(which shadow)
 	vetRes=$(go vet -all -vettool="${shadow_tool}" "${TEST[@]}")