$ sudo ps -aef | grep etcd | grep -v grep | awk '{print $2}' | xargs kill -s KILL
rm -rf infra*.etcd test.etcd
rm /home/chaochn/log/etcd/infra*.log
 
$ git checkout release-3.5
 
$ gvm use go1.16.15

$ pushd server
$ go get github.com/etcd-io/gofail/runtime
$ popd server

$ FAILPOINTS=1 ./build
 
$ bin/etcd --version
etcd Version: 3.5.4
Git SHA: 9e95685d0-FAILPOINTS
Go Version: go1.16.15
Go OS/Arch: linux/amd64
 
$ cp bin/etcd /local/home/chaochn/etcd-binaries/v3.5.4-FAILPOINTS/
 
$ GOFAIL_HTTP="127.0.0.1:1234" ~/etcd-binaries/v3.5.4-FAILPOINTS/etcd --name infra1 \
--data-dir infra1.etcd --logger=zap \
--initial-cluster=new \
--listen-client-urls http://127.0.0.1:2379 \
--advertise-client-urls http://127.0.0.1:2379 \
--listen-peer-urls http://127.0.0.1:12380 \
--initial-advertise-peer-urls http://127.0.0.1:12380 \
--initial-cluster-token etcd-cluster-1 --initial-cluster 'infra1=http://127.0.0.1:12380,infra2=http://127.0.0.1:22380,infra3=http://127.0.0.1:32380' --initial-cluster-state new \
--enable-pprof > /home/chaochn/log/etcd/infra1.log 2>&1 &
etcd_PID_1=$!
 
~/etcd-binaries/v3.5.4/etcd --name infra2 \
--data-dir infra2.etcd --logger=zap \
--initial-cluster=new \
--listen-client-urls http://127.0.0.1:22379 \
--advertise-client-urls http://127.0.0.1:22379 \
--listen-peer-urls http://127.0.0.1:22380 \
--initial-advertise-peer-urls http://127.0.0.1:22380 \
--initial-cluster-token etcd-cluster-1 --initial-cluster 'infra1=http://127.0.0.1:12380,infra2=http://127.0.0.1:22380,infra3=http://127.0.0.1:32380' --initial-cluster-state new \
--enable-pprof > /home/chaochn/log/etcd/infra2.log 2>&1 &
etcd_PID_2=$!
 
~/etcd-binaries/v3.5.4/etcd --name static_ip_3 \
--data-dir infra3.etcd --logger=zap \
--initial-cluster=new \
--listen-client-urls http://127.0.0.1:32379 \
--advertise-client-urls http://127.0.0.1:32379 \
--listen-peer-urls http://127.0.0.1:32380 \
--initial-advertise-peer-urls http://127.0.0.1:32380 \
--initial-cluster-token etcd-cluster-1 --initial-cluster 'static_ip_1=http://127.0.0.1:12380,static_ip_2=http://127.0.0.1:22380,static_ip_3=http://127.0.0.1:32380' --initial-cluster-state new \
--enable-pprof > /home/chaochn/log/etcd/infra3.log 2>&1 &
etcd_PID_3=$!
 
[1] 22735
[2] 22736
[3] 22737
 
$ curl http://127.0.0.1:1234/
backend/afterCommit=
backend/beforeCommit=
backend/defragBeforeCopy=
backend/defragBeforeRename=
etcdserver/raftAfterApplySnap=
etcdserver/raftAfterSave=
etcdserver/raftAfterSaveSnap=
etcdserver/raftAfterWALRelease=
etcdserver/raftBeforeApplySnap=
etcdserver/raftBeforeFollowerSend=
etcdserver/raftBeforeLeaderSend=
etcdserver/raftBeforeSave=
etcdserver/raftBeforeSaveSnap=
etcdserver/raftBeforeWALSaveSnaphot=
 
$ etcdctl endpoint status --cluster -w table
+------------------------+------------------+---------+-----------------+---------+----------------+-----------+------------+-----------+------------+--------------------+--------+
|        ENDPOINT        |        ID        | VERSION | STORAGE VERSION | DB SIZE | DB SIZE IN USE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+------------------------+------------------+---------+-----------------+---------+----------------+-----------+------------+-----------+------------+--------------------+--------+
|  http://127.0.0.1:2379 | 8211f1d0f64f3269 |  3.5.4 |                 |   20 kB |          16 kB |     false |      false |         2 |          8 |                  8 |        |
| http://127.0.0.1:22379 | 91bc3c398fb3c146 |  3.5.4 |                 |   20 kB |          16 kB |      true |      false |         2 |          8 |                  8 |        |
| http://127.0.0.1:32379 | fd422379fda50e48 |  3.5.4 |                 |   20 kB |          16 kB |     false |      false |         2 |          8 |                  8 |        |
+------------------------+------------------+---------+-----------------+---------+----------------+-----------+------------+-----------+------------+--------------------+--------+
 
$ curl http://127.0.0.1:1234/etcdserver/raftAfterSave -XPUT -d'sleep(600000)'
 
$ etcdctl --endpoints http://127.0.0.1:22379,http://127.0.0.1:32379 member remove 8211f1d0f64f3269
Member 8211f1d0f64f3269 removed from cluster ef37ad9dc622a7c4

$ etcdctl endpoint status -w table --endpoints http://127.0.0.1:2379,http://127.0.0.1:22379,http://127.0.0.1:32379
+------------------------+------------------+---------+-----------------+---------+----------------+-----------+------------+-----------+------------+--------------------+--------+
|        ENDPOINT        |        ID        | VERSION | STORAGE VERSION | DB SIZE | DB SIZE IN USE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS |
+------------------------+------------------+---------+-----------------+---------+----------------+-----------+------------+-----------+------------+--------------------+--------+
|  http://127.0.0.1:2379 | 8211f1d0f64f3269 |   3.5.4 |                 |   20 kB |          16 kB |      true |      false |         2 |          8 |                  8 |        |
| http://127.0.0.1:22379 | 91bc3c398fb3c146 |   3.5.4 |                 |   20 kB |          16 kB |     false |      false |         3 |         10 |                 10 |        |
| http://127.0.0.1:32379 | fd422379fda50e48 |   3.5.4 |                 |   20 kB |          16 kB |      true |      false |         3 |         10 |                 10 |        |
+------------------------+------------------+---------+-----------------+---------+----------------+-----------+------------+-----------+------------+--------------------+--------+

# kill with SIGTERM cannot terminate the process
 
$ less /home/chaochn/log/etcd/infra1.log