Fix etcd snapshot integration tests

Snapshot delete/prune tests were only working because the delete command
would report success even when deleting a snapshot that didn't exist,
and the test regex was finding the snapshot name multiple times in
the list output and deleting it twice.

Snapshot restore tests seem to have expected the deployment to be rolled out
immediately, which is not a reasonable expectation.

Signed-off-by: Brad Davidson <brad.davidson@rancher.com>

tests/integration/etcdrestore/etcd_restore_int_test.go

@@ -41,6 +41,11 @@ var _ = Describe("etcd snapshot restore", Ordered, func() {
 			Expect(result).To(ContainSubstring("deployment.apps/nginx-deployment created"))
 			Expect(err).NotTo(HaveOccurred())
 		})
+		It("make sure workload exists", func() {
+			res, err := testutil.K3sCmd("kubectl", "rollout", "status", "deployment", "nginx-deployment", "--watch=true", "--timeout=360s")
+			Expect(res).To(ContainSubstring("successfully rolled out"))
+			Expect(err).ToNot(HaveOccurred())
+		})
 		It("saves an etcd snapshot", func() {
 			Expect(testutil.K3sCmd("etcd-snapshot", "save", "-d", tmpdDataDir, "--name", "snapshot-to-restore")).
 				To(ContainSubstring("saved"))
@@ -83,15 +88,15 @@ var _ = Describe("etcd snapshot restore", Ordered, func() {
 				return testutil.K3sDefaultDeployments()
 			}, "360s", "5s").Should(Succeed())
 		})
-		It("Make sure Workload 1 exists", func() {
-			Eventually(func() (string, error) {
-				return testutil.K3sCmd("kubectl", "get", "deployment", "nginx-deployment")
-			}, "360s", "5s").Should(ContainSubstring("3/3"))
+		It("make sure workload 1 exists", func() {
+			res, err := testutil.K3sCmd("kubectl", "rollout", "status", "deployment", "nginx-deployment", "--watch=true", "--timeout=360s")
+			Expect(res).To(ContainSubstring("successfully rolled out"))
+			Expect(err).ToNot(HaveOccurred())
 		})
-		It("Make sure Workload 2 does not exists", func() {
+		It("make sure workload 2 does not exists", func() {
 			res, err := testutil.K3sCmd("kubectl", "get", "deployment", "nginx-deployment-post-snapshot")
-			Expect(err).To(HaveOccurred())
 			Expect(res).To(ContainSubstring("not found"))
+			Expect(err).To(HaveOccurred())
 		})
 		It("check if CA cert hash matches", func() {
 			// get md5sum of the CA certs

tests/integration/etcdrestore/testdata/temp_depl.yaml

@@ -6,6 +6,9 @@ metadata:
     app: nginx
 spec:
   replicas: 3
+  revisionHistoryLimit: 0
+  strategy:
+    type: Recreate
   selector:
     matchLabels:
       app: nginx
@@ -18,4 +21,4 @@ spec:
       - name: nginx
         image: nginx:1.14.2
         ports:
-        - containerPort: 80
+        - containerPort: 80

tests/integration/etcdrestore/testdata/temp_depl2.yaml

@@ -6,6 +6,9 @@ metadata:
     app: nginx
 spec:
   replicas: 3
+  revisionHistoryLimit: 0
+  strategy:
+    type: Recreate
   selector:
     matchLabels:
       app: nginx
@@ -18,4 +21,4 @@ spec:
       - name: nginx
         image: nginx:1.14.2
         ports:
-        - containerPort: 80
+        - containerPort: 80

tests/integration/etcdsnapshot/etcdsnapshot_int_test.go

@@ -54,7 +54,7 @@ var _ = Describe("etcd snapshots", Ordered, func() {
 		It("deletes a snapshot", func() {
 			lsResult, err := testutil.K3sCmd("etcd-snapshot", "ls")
 			Expect(err).ToNot(HaveOccurred())
-			reg, err := regexp.Compile(`on-demand[^\s]+`)
+			reg, err := regexp.Compile(`(?m)^on-demand[^\s]+`)
 			Expect(err).ToNot(HaveOccurred())
 			snapshotName := reg.FindString(lsResult)
 			Expect(testutil.K3sCmd("etcd-snapshot", "delete", snapshotName)).
@@ -69,7 +69,7 @@ var _ = Describe("etcd snapshots", Ordered, func() {
 		It("deletes that snapshot", func() {
 			lsResult, err := testutil.K3sCmd("etcd-snapshot", "ls")
 			Expect(err).ToNot(HaveOccurred())
-			reg, err := regexp.Compile(`ALIVEBEEF[^\s]+`)
+			reg, err := regexp.Compile(`(?m)^ALIVEBEEF[^\s]+`)
 			Expect(err).ToNot(HaveOccurred())
 			snapshotName := reg.FindString(lsResult)
 			Expect(testutil.K3sCmd("etcd-snapshot", "delete", snapshotName)).
@@ -91,7 +91,7 @@ var _ = Describe("etcd snapshots", Ordered, func() {
 		It("lists all 3 snapshots", func() {
 			lsResult, err := testutil.K3sCmd("etcd-snapshot", "ls")
 			Expect(err).ToNot(HaveOccurred())
-			reg, err := regexp.Compile(`:///var/lib/rancher/k3s/server/db/snapshots/PRUNE_TEST`)
+			reg, err := regexp.Compile(`(?m):///var/lib/rancher/k3s/server/db/snapshots/PRUNE_TEST`)
 			Expect(err).ToNot(HaveOccurred())
 			sepLines := reg.FindAllString(lsResult, -1)
 			Expect(sepLines).To(HaveLen(3))
@@ -101,15 +101,15 @@ var _ = Describe("etcd snapshots", Ordered, func() {
 				To(ContainSubstring("Removing local snapshot"))
 			lsResult, err := testutil.K3sCmd("etcd-snapshot", "ls")
 			Expect(err).ToNot(HaveOccurred())
-			reg, err := regexp.Compile(`:///var/lib/rancher/k3s/server/db/snapshots/PRUNE_TEST`)
+			reg, err := regexp.Compile(`(?m):///var/lib/rancher/k3s/server/db/snapshots/PRUNE_TEST`)
 			Expect(err).ToNot(HaveOccurred())
 			sepLines := reg.FindAllString(lsResult, -1)
 			Expect(sepLines).To(HaveLen(2))
 		})
 		It("cleans up remaining snapshots", func() {
 			lsResult, err := testutil.K3sCmd("etcd-snapshot", "ls")
 			Expect(err).ToNot(HaveOccurred())
-			reg, err := regexp.Compile(`PRUNE_TEST[^\s]+`)
+			reg, err := regexp.Compile(`(?m)^PRUNE_TEST[^\s]+`)
 			Expect(err).ToNot(HaveOccurred())
 			for _, snapshotName := range reg.FindAllString(lsResult, -1) {
 				Expect(testutil.K3sCmd("etcd-snapshot", "delete", snapshotName)).

tests/integration/integration.go

@@ -280,6 +280,9 @@ func K3sStopServer(server *K3sServer) error {
 // K3sKillServer terminates the running K3s server and its children.
 // Equivalent to k3s-killall.sh
 func K3sKillServer(server *K3sServer) error {
+	if server == nil {
+		return nil
+	}
 	if server.log != nil {
 		server.log.Close()
 		os.Remove(server.log.Name())