gravitational · knisbet · May 23, 2018 · Apr 10, 2018 · Apr 10, 2018 · Apr 10, 2018
diff --git a/Gopkg.lock b/Gopkg.lock
diff --git a/Gopkg.toml b/Gopkg.toml
@@ -26,6 +26,10 @@ ignored = ["github.com/Sirupsen/logrus"]
   go-tests = true
   unused-packages = true
 
+[[override]]
+  name = "github.com/coreos/go-systemd"
+  version = "v16"
+
 [[override]]
   name = "bitbucket.org/ww/goautoneg"
   # Bitbucket doesn't support git protocol, which causes dep to timeout trying to connect
@@ -36,13 +40,19 @@ ignored = ["github.com/Sirupsen/logrus"]
   name = "github.com/hashicorp/serf"
   revision = "11bb88abf7b17f0b794b51416a9107d781e95f35"
 
+[[override]]
+  # target codec version from etcd, since dep doesn't seem to grab it correctly
+  # https://github.com/coreos/etcd/issues/8715#issuecomment-377013707
+  name = "github.com/ugorji/go"
+  version = "v1.1.1"
+
 [[constraint]]
   name = "github.com/cenkalti/backoff"
   version = "2.0.0"
 
 [[constraint]]
   name = "github.com/coreos/etcd"
-  version = "3.2.4"
+  version = "3.3.3"
 
 [[constraint]]
   name = "github.com/docker/docker"

diff --git a/Makefile b/Makefile
@@ -48,9 +48,14 @@ SECCOMP_VER :=  2.3.1-2.1
 DOCKER_VER := 17.03.2
 # we currently use our own flannel fork: gravitational/flannel
 FLANNEL_VER := master
-ETCD_VER := v2.3.8
 HELM_VER := v2.8.1
 
+# ETCD Versions to include in the release
+# This list needs to include every version of etcd that we can upgrade from + latest
+ETCD_VER := v2.3.8 v3.3.4
+# This is the version of etcd we should upgrade to (from the version list)
+ETCD_LATEST_VER := v3.3.4
+
 PUBLIC_IP := 127.0.0.1
 export
 PLANET_PACKAGE_PATH=$(PWD)

diff --git a/build.assets/docker/buildbox.dockerfile b/build.assets/docker/buildbox.dockerfile
@@ -3,7 +3,7 @@ FROM planet/base
 ENV GOPATH /gopath
 ENV GOROOT /opt/go
 ENV PATH $PATH:$GOPATH/bin:$GOROOT/bin
-ENV GOVERSION 1.8.3
+ENV GOVERSION 1.10.1
 
 # Have our own /etc/passwd with users populated from 990 to 1000
 COPY passwd /etc/passwd

diff --git a/build.assets/makefiles/base/network/flanneld.service b/build.assets/makefiles/base/network/flanneld.service
@@ -18,11 +18,11 @@ ExecStartPre=/usr/bin/etcdctl \
     --cert-file=/var/state/etcd.cert \
     --key-file=/var/state/etcd.key \
     --ca-file=/var/state/root.cert \
-    --peers https://127.0.0.1:4001 set /coreos.com/network/config '{"Network":"${KUBE_POD_SUBNET}", "Backend": {"Type": "${FLANNEL_BACKEND}", "RouteTableFilter": ["tag:KubernetesCluster=${KUBE_CLUSTER_ID}"]}}'
+    --peers https://127.0.0.1:2379 set /coreos.com/network/config '{"Network":"${KUBE_POD_SUBNET}", "Backend": {"Type": "${FLANNEL_BACKEND}", "RouteTableFilter": ["tag:KubernetesCluster=${KUBE_CLUSTER_ID}"]}}'
 ExecStartPre=/bin/systemctl is-active etcd.service
 ExecStart=/usr/bin/flanneld \
     --ip-masq=true \
-    --etcd-endpoints https://127.0.0.1:4001,https://127.0.0.1:2379 \
+    --etcd-endpoints https://127.0.0.1:2379 \
     --etcd-cafile=/var/state/root.cert \
     --etcd-certfile=/var/state/etcd.cert \
     --etcd-keyfile=/var/state/etcd.key \

diff --git a/build.assets/makefiles/base/network/wait-for-etcd.sh b/build.assets/makefiles/base/network/wait-for-etcd.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+PEERS=${1:-https://127.0.0.1:2379}
+
 n=0
 until [ $n -ge 10 ]
 do
@@ -9,7 +11,7 @@ do
       --ca-file=/var/state/root.cert \
       --timeout="5s" \
       --total-timeout="30s" \
-      --peers https://127.0.0.1:4001 cluster-health && exit 0
+      --peers ${PEERS} cluster-health && exit 0
     n=$[$n+1]
     sleep 3
 done
diff --git a/build.assets/makefiles/buildbox.mk b/build.assets/makefiles/buildbox.mk
@@ -30,14 +30,16 @@ build:
 		make -e \
 			KUBE_VER=$(KUBE_VER) \
 			FLANNEL_VER=$(FLANNEL_VER) \
-			ETCD_VER=$(ETCD_VER) \
+			ETCD_VER="$(ETCD_VER)" \
+			ETCD_LATEST_VER=$(ETCD_LATEST_VER) \
 			-C /assets/makefiles -f $(TARGET)-docker.mk
 ifeq ($(TARGET),master)
 	$(MAKE) -C $(ASSETS)/makefiles/master/k8s-master -e -f containers.mk
 endif
 
 planet-image:
 	cp $(ASSETS)/orbit.manifest.json $(TARGETDIR)
+	sed -i "s/REPLACE_ETCD_LATEST_VERSION/$(ETCD_LATEST_VER)/g" $(TARGETDIR)/orbit.manifest.json
 	cp $(ASSETDIR)/planet $(ROOTFS)/usr/bin/
 	cp $(ASSETDIR)/docker-import $(ROOTFS)/usr/bin/
 	@echo -e "\n---> Moving current symlink to $(TARGETDIR)\n"

diff --git a/build.assets/makefiles/etcd/etcd-gateway.dropin b/build.assets/makefiles/etcd/etcd-gateway.dropin
@@ -0,0 +1,9 @@
+# This systemd drop in file, will change the etcd unit to run a gateway
+# instead of the etcd service
+
+[Service]
+ExecStart=
+ExecStart=/usr/bin/etcd gateway start \
+        --endpoints=${PLANET_ETCD_GW_ENDPOINTS} \
+        --listen-addr=0.0.0.0:2379 \
+        --trusted-ca-file=/var/state/root.cert
diff --git a/build.assets/makefiles/etcd/etcd-upgrade.service b/build.assets/makefiles/etcd/etcd-upgrade.service
@@ -0,0 +1,37 @@
+[Unit]
+Description=Temporary Etcd Service used for upgrades
+Conflicts=etcd.service
+
+# This works by launching etcd, but bound to a non-default loopback interface.
+# This is to prevent etcd from being used, while it is being upgraded, and the
+# database is inconsistent
+
+[Service]
+Restart=always
+RestartSec=5
+StartLimitInterval=3600
+StartLimitBurst=720
+Type=notify
+TimeoutStartSec=0
+EnvironmentFile=/etc/container-environment
+EnvironmentFile=-/ext/etcd/etcd-version.txt
+ExecStartPre=/usr/bin/planet etcd init
+ExecStart=/usr/bin/etcd \
+        --name=${PLANET_ETCD_MEMBER_NAME} \
+        --data-dir=/ext/etcd/${PLANET_ETCD_VERSION} \
+        --initial-advertise-peer-urls=https://${PLANET_PUBLIC_IP}:2380 \
+        --advertise-client-urls=https://127.0.0.2:2379,https://127.0.0.2:4001 \
+        --listen-client-urls=https://127.0.0.2:2379,https://127.0.0.2:4001 \
+        --listen-peer-urls=https://${PLANET_PUBLIC_IP}:2380,https://${PLANET_PUBLIC_IP}:7001 \
+        --cert-file=/var/state/etcd.cert \
+        --key-file=/var/state/etcd.key \
+        --trusted-ca-file=/var/state/root.cert \
+        --client-cert-auth \
+        --peer-cert-file=/var/state/etcd.cert \
+        --peer-key-file=/var/state/etcd.key \
+        --peer-trusted-ca-file=/var/state/root.cert \
+        --peer-client-cert-auth $ETCD_OPTS \
+        --initial-cluster-state new
+User=planet
+Group=planet
+PermissionsStartOnly=true
diff --git a/build.assets/makefiles/etcd/etcd.mk b/build.assets/makefiles/etcd/etcd.mk
@@ -1,19 +1,35 @@
 .PHONY: all
 
 ARCH := amd64
-TARGET := etcd-$(ETCD_VER)-linux-$(ARCH)
-TARGET_TARBALL := $(TARGET).tar.gz
 
-DOWNLOAD:=$(ASSETDIR)/$(TARGET_TARBALL)
 
-all: $(DOWNLOAD)
-	@echo "\n---> Building etcd:\n"
-	cd $(ASSETDIR) && tar -xzf $(ASSETDIR)/$(TARGET_TARBALL)
-	mkdir -p $(ROOTFS)/var/etcd
-	cp -afv $(ASSETDIR)/$(TARGET)/etcd $(ROOTFS)/usr/bin
-	cp -afv $(ASSETDIR)/$(TARGET)/etcdctl $(ROOTFS)/usr/bin
+all: $(ETCD_VER)
+	@echo -e "\n---> Building etcd:\n"
+
+	@echo -e "\n---> Setup etcd services:\n"
+	cd $(ASSETDIR)
 	cp -afv ./etcd.service $(ROOTFS)/lib/systemd/system/
-	ln -sf /lib/systemd/system/etcd.service  $(ROOTFS)/lib/systemd/system/multi-user.target.wants/
+	cp -afv ./etcd-upgrade.service $(ROOTFS)/lib/systemd/system/
+	cp -afv ./etcd-gateway.dropin $(ROOTFS)/lib/systemd/system/
+	cp -afv ./etcdctl3 $(ROOTFS)/usr/bin/etcdctl3
+	chmod +x $(ROOTFS)/usr/bin/etcdctl3
+	ln -sf /lib/systemd/system/etcd.service $(ROOTFS)/lib/systemd/system/multi-user.target.wants/
+
+	# mask the etcd-upgrade service so that it can only be run if intentionally unmasked
+	ln -sf /dev/null $(ROOTFS)/etc/systemd/system/etcd-upgrade.service
+
+	# Write to the release file to indicate the latest release
+	echo PLANET_ETCD_VERSION=$(ETCD_LATEST_VER) >> $(ROOTFS)/etc/planet-release
+
+.PHONY: $(ETCD_VER)
+$(ETCD_VER):
+	@echo -e "\n---> $@ - Downloading etcd\n"
+	curl -L https://github.com/coreos/etcd/releases/download/$@/etcd-$@-linux-$(ARCH).tar.gz \
+	-o $(ASSETDIR)/$@.tar.gz;
+
+	@echo -e "\n---> $@ - Extracting etcd\n"
+	cd $(ASSETDIR)
+	tar -xzf $(ASSETDIR)/$@.tar.gz
 
-$(DOWNLOAD):
-	curl -L https://github.com/coreos/etcd/releases/download/$(ETCD_VER)/$(TARGET_TARBALL) -o $(DOWNLOAD)
+	cp -afv etcd-$@-linux-$(ARCH)/etcd $(ROOTFS)/usr/bin/etcd-$@
+	cp -afv etcd-$@-linux-$(ARCH)/etcdctl $(ROOTFS)/usr/bin/etcdctl-$@
diff --git a/build.assets/makefiles/etcd/etcd.service b/build.assets/makefiles/etcd/etcd.service
@@ -1,5 +1,6 @@
 [Unit]
 Description=Etcd Service
+Conflicts=etcd-upgrade.service
 
 [Service]
 Restart=always
@@ -9,9 +10,11 @@ StartLimitBurst=720
 Type=notify
 TimeoutStartSec=0
 EnvironmentFile=/etc/container-environment
+EnvironmentFile=-/ext/etcd/etcd-version.txt
+ExecStartPre=/usr/bin/planet etcd init
 ExecStart=/usr/bin/etcd \
-        --name=${ETCD_MEMBER_NAME} \
-        --data-dir=/ext/etcd \
+        --name=${PLANET_ETCD_MEMBER_NAME} \
+        --data-dir=/ext/etcd/${PLANET_ETCD_VERSION} \
         --initial-advertise-peer-urls=https://${PLANET_PUBLIC_IP}:2380 \
         --advertise-client-urls=https://${PLANET_PUBLIC_IP}:2379,https://${PLANET_PUBLIC_IP}:4001 \
         --listen-client-urls=https://0.0.0.0:2379,https://0.0.0.0:4001 \
@@ -26,3 +29,4 @@ ExecStart=/usr/bin/etcd \
         --peer-client-cert-auth $ETCD_OPTS
 User=planet
 Group=planet
+PermissionsStartOnly=true
diff --git a/build.assets/makefiles/etcd/etcdctl3 b/build.assets/makefiles/etcd/etcdctl3
@@ -0,0 +1,7 @@
+#!/bin/bash
+#
+# This is a helper script, to make it easier to access the etcd3 datastore
+#
+
+ETCDCTL_API=3 ETCDCTL_CERT_FILE="" ETCDCTL_KEY_FILE="" ETCDCTL_CA_FILE="" ETCDCTL_PEERS="" \
+/usr/bin/etcdctl --key /var/state/etcd.key --cert /var/state/etcd.cert --cacert /var/state/root.cert "$@"
diff --git a/build.assets/makefiles/master/k8s-master/kube-apiserver.service b/build.assets/makefiles/master/k8s-master/kube-apiserver.service
@@ -4,6 +4,11 @@ Documentation=https://github.com/GoogleCloudPlatform/kubernetes
 Wants=etcd.service
 
 [Service]
+# Default to etcd storage backend
+# TODO - remove when we no longer support etcd2 upgrade paths
+Environment=KUBE_STORAGE_BACKEND=etcd2
+# Override KUBE_STORAGE_BACKEND if we're using a version 3 compatible etcd
+EnvironmentFile=-/ext/etcd/etcd-version.txt
 EnvironmentFile=/etc/container-environment
 ExecStartPre=/usr/bin/scripts/wait-for-etcd.sh
 ExecStartPre=/bin/systemctl is-active etcd.service
@@ -30,7 +35,7 @@ ExecStart=/usr/bin/kube-apiserver \
         --etcd-cafile=/var/state/root.cert \
         --etcd-certfile=/var/state/etcd.cert \
         --etcd-keyfile=/var/state/etcd.key \
-        --storage-backend=etcd2 \
+        --storage-backend=${KUBE_STORAGE_BACKEND} \
         --event-ttl=24h0m0s \
         --bind-address=${PLANET_PUBLIC_IP} \
         --logtostderr=true $KUBE_CLOUD_FLAGS \

diff --git a/build.assets/orbit.manifest.json b/build.assets/orbit.manifest.json
@@ -1,7 +1,8 @@
 {
     "version": "0.0.1",
     "labels": [
-      {"name": "os", "value": "linux"}
+      {"name": "os", "value": "linux"},
+      {"name": "version-etcd", "value": "REPLACE_ETCD_LATEST_VERSION"}
     ],
     "commands": [
       {"name": "start", "args": ["rootfs/usr/bin/planet", "start"]},

diff --git a/tool/planet/cfg.go b/tool/planet/cfg.go
@@ -70,6 +70,8 @@ type Config struct {
 	// EtcdInitialCluster configures the value of ETCD_INITIAL_CLUSTER environment variable
 	// inside the container
 	EtcdInitialCluster string
+	// EtcdGatewayList is a list of etcd endpoints that the etcd gateway can use to reach the cluster
+	EtcdGatewayList string
 	// EtcdInitialClusterState configures the value of ETCD_INITIAL_CLUSTER_STATE environment variable
 	// inside the container
 	EtcdInitialClusterState string