-
Notifications
You must be signed in to change notification settings - Fork 69
/
virt_backup_restore_suite_test.go
317 lines (272 loc) · 12.3 KB
/
virt_backup_restore_suite_test.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
package e2e_test
import (
"fmt"
"io"
"log"
"net/http"
"strings"
"time"
"github.com/onsi/ginkgo/v2"
"github.com/onsi/gomega"
"k8s.io/apimachinery/pkg/util/wait"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/openshift/oadp-operator/api/v1alpha1"
"github.com/openshift/oadp-operator/tests/e2e/lib"
)
// TODO duplication of todoListReady in tests/e2e/backup_restore_suite_test.go
func vmTodoListReady(preBackupState bool, twoVol bool, database string) VerificationFunction {
return VerificationFunction(func(ocClient client.Client, namespace string) error {
log.Printf("checking for the NAMESPACE: %s", namespace)
gomega.Eventually(lib.IsDeploymentReady(ocClient, namespace, database), time.Minute*10, time.Second*10).Should(gomega.BeTrue())
// in VM tests, DeploymentConfig was refactored to Deployment (to avoid deprecation warnings)
// gomega.Eventually(lib.IsDCReady(ocClient, namespace, "todolist"), time.Minute*10, time.Second*10).Should(gomega.BeTrue())
gomega.Eventually(lib.IsDeploymentReady(ocClient, namespace, "todolist"), time.Minute*10, time.Second*10).Should(gomega.BeTrue())
gomega.Eventually(lib.AreApplicationPodsRunning(kubernetesClientForSuiteRun, namespace), time.Minute*9, time.Second*5).Should(gomega.BeTrue())
// This test confirms that SCC restore logic in our plugin is working
err := lib.DoesSCCExist(ocClient, database+"-persistent-scc")
if err != nil {
return err
}
err = lib.VerifyBackupRestoreData(runTimeClientForSuiteRun, kubernetesClientForSuiteRun, kubeConfig, artifact_dir, namespace, "todolist-route", "todolist", "todolist", preBackupState, twoVol)
return err
})
}
func getLatestCirrosImageURL() (string, error) {
cirrosVersionURL := "https://download.cirros-cloud.net/version/released"
resp, err := http.Get(cirrosVersionURL)
if err != nil {
return "", err
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return "", err
}
latestCirrosVersion := strings.TrimSpace(string(body))
imageURL := fmt.Sprintf("https://download.cirros-cloud.net/%s/cirros-%s-x86_64-disk.img", latestCirrosVersion, latestCirrosVersion)
return imageURL, nil
}
func vmPoweredOff(vmnamespace, vmname string) VerificationFunction {
return VerificationFunction(func(ocClient client.Client, namespace string) error {
isOff := func() bool {
status, err := lib.GetVmStatus(dynamicClientForSuiteRun, vmnamespace, vmname)
if err != nil {
log.Printf("Error getting VM status: %v", err)
}
log.Printf("VM status is: %s\n", status)
return status == "Stopped"
}
gomega.Eventually(isOff, time.Minute*10, time.Second*10).Should(gomega.BeTrue())
return nil
})
}
type VmBackupRestoreCase struct {
BackupRestoreCase
Template string
InitDelay time.Duration
PowerState string
}
func runVmBackupAndRestore(brCase VmBackupRestoreCase, expectedErr error, updateLastBRcase func(brCase VmBackupRestoreCase), updateLastInstallTime func(), v *lib.VirtOperator) {
updateLastBRcase(brCase)
// Create DPA
backupName, restoreName := prepareBackupAndRestore(brCase.BackupRestoreCase, func() {})
err := lib.CreateNamespace(v.Clientset, brCase.Namespace)
gomega.Expect(err).To(gomega.BeNil())
err = lib.InstallApplication(v.Client, brCase.Template)
if err != nil {
fmt.Printf("Failed to install VM template %s: %v", brCase.Template, err)
}
gomega.Expect(err).To(gomega.BeNil())
// Wait for VM to start, then give some time for cloud-init to run.
// Afterward, run through the standard application verification to make sure
// the application itself is working correctly.
err = wait.PollImmediate(10*time.Second, 10*time.Minute, func() (bool, error) {
status, err := v.GetVmStatus(brCase.Namespace, brCase.Name)
return status == "Running", err
})
gomega.Expect(err).ToNot(gomega.HaveOccurred())
// TODO: find a better way to check for clout-init completion
if brCase.InitDelay > 0*time.Second {
log.Printf("Sleeping to wait for cloud-init to be ready...")
time.Sleep(brCase.InitDelay)
}
// Check if this VM should be running or stopped for this test.
// Depend on pre-backup verification function to poll state.
if brCase.PowerState == "Stopped" {
log.Print("Stopping VM before backup as specified in test case.")
err = v.StopVm(brCase.Namespace, brCase.Name)
gomega.Expect(err).To(gomega.BeNil())
}
// Run optional custom verification
if brCase.PreBackupVerify != nil {
log.Printf("Running pre-backup custom function for case %s", brCase.Name)
err := brCase.PreBackupVerify(dpaCR.Client, brCase.Namespace)
gomega.Expect(err).ToNot(gomega.HaveOccurred())
}
// Back up VM
nsRequiresResticDCWorkaround := runBackup(brCase.BackupRestoreCase, backupName)
// Delete everything in test namespace
err = v.RemoveVm(brCase.Namespace, brCase.Name, 5*time.Minute)
gomega.Expect(err).To(gomega.BeNil())
err = lib.DeleteNamespace(v.Clientset, brCase.Namespace)
gomega.Expect(err).To(gomega.BeNil())
gomega.Eventually(lib.IsNamespaceDeleted(kubernetesClientForSuiteRun, brCase.Namespace), time.Minute*5, time.Second*5).Should(gomega.BeTrue())
// Do restore
runRestore(brCase.BackupRestoreCase, backupName, restoreName, nsRequiresResticDCWorkaround)
// Run optional custom verification
if brCase.PostRestoreVerify != nil {
log.Printf("Running post-restore custom function for VM case %s", brCase.Name)
err = brCase.PostRestoreVerify(dpaCR.Client, brCase.Namespace)
gomega.Expect(err).ToNot(gomega.HaveOccurred())
}
// avoid finalizers in namespace deletion
err = v.RemoveVm(brCase.Namespace, brCase.Name, 5*time.Minute)
gomega.Expect(err).To(gomega.BeNil())
}
var _ = ginkgo.Describe("VM backup and restore tests", ginkgo.Ordered, func() {
var v *lib.VirtOperator
var err error
wasInstalledFromTest := false
var lastBRCase VmBackupRestoreCase
var lastInstallTime time.Time
updateLastBRcase := func(brCase VmBackupRestoreCase) {
lastBRCase = brCase
}
updateLastInstallTime := func() {
lastInstallTime = time.Now()
}
var _ = ginkgo.BeforeAll(func() {
v, err = lib.GetVirtOperator(runTimeClientForSuiteRun, kubernetesClientForSuiteRun, dynamicClientForSuiteRun)
gomega.Expect(err).To(gomega.BeNil())
gomega.Expect(v).ToNot(gomega.BeNil())
if !v.IsVirtInstalled() {
err = v.EnsureVirtInstallation()
gomega.Expect(err).To(gomega.BeNil())
wasInstalledFromTest = true
}
err = v.EnsureEmulation(20 * time.Second)
gomega.Expect(err).To(gomega.BeNil())
url, err := getLatestCirrosImageURL()
gomega.Expect(err).To(gomega.BeNil())
err = v.EnsureDataVolumeFromUrl("openshift-virtualization-os-images", "cirros", url, "150Mi", 5*time.Minute)
gomega.Expect(err).To(gomega.BeNil())
err = v.CreateDataSourceFromPvc("openshift-virtualization-os-images", "cirros")
gomega.Expect(err).To(gomega.BeNil())
dpaCR.VeleroDefaultPlugins = append(dpaCR.VeleroDefaultPlugins, v1alpha1.DefaultPluginKubeVirt)
err = v.CreateImmediateModeStorageClass("test-sc-immediate")
gomega.Expect(err).To(gomega.BeNil())
})
var _ = ginkgo.AfterAll(func() {
v.RemoveDataSource("openshift-virtualization-os-images", "cirros")
v.RemoveDataVolume("openshift-virtualization-os-images", "cirros", 2*time.Minute)
if v != nil && wasInstalledFromTest {
v.EnsureVirtRemoval()
}
err := v.RemoveStorageClass("test-sc-immediate")
gomega.Expect(err).To(gomega.BeNil())
})
var _ = ginkgo.AfterEach(func(ctx ginkgo.SpecContext) {
tearDownBackupAndRestore(lastBRCase.BackupRestoreCase, lastInstallTime, ctx.SpecReport())
})
ginkgo.DescribeTable("Backup and restore virtual machines",
func(brCase VmBackupRestoreCase, expectedError error) {
runVmBackupAndRestore(brCase, expectedError, updateLastBRcase, updateLastInstallTime, v)
},
ginkgo.Entry("no-application CSI datamover backup and restore, CirrOS VM", ginkgo.Label("virt"), VmBackupRestoreCase{
Template: "./sample-applications/virtual-machines/cirros-test/cirros-test.yaml",
InitDelay: 2 * time.Minute, // Just long enough to get to login prompt, VM is marked running while kernel messages are still scrolling by
BackupRestoreCase: BackupRestoreCase{
Namespace: "cirros-test",
Name: "cirros-test",
SkipVerifyLogs: true,
BackupRestoreType: lib.CSIDataMover,
BackupTimeout: 20 * time.Minute,
},
}, nil),
ginkgo.Entry("no-application CSI backup and restore, CirrOS VM", ginkgo.Label("virt"), VmBackupRestoreCase{
Template: "./sample-applications/virtual-machines/cirros-test/cirros-test.yaml",
InitDelay: 2 * time.Minute, // Just long enough to get to login prompt, VM is marked running while kernel messages are still scrolling by
BackupRestoreCase: BackupRestoreCase{
Namespace: "cirros-test",
Name: "cirros-test",
SkipVerifyLogs: true,
BackupRestoreType: lib.CSI,
BackupTimeout: 20 * time.Minute,
},
}, nil),
ginkgo.Entry("no-application CSI backup and restore, powered-off CirrOS VM", ginkgo.Label("virt"), VmBackupRestoreCase{
Template: "./sample-applications/virtual-machines/cirros-test/cirros-test.yaml",
InitDelay: 2 * time.Minute,
PowerState: "Stopped",
BackupRestoreCase: BackupRestoreCase{
Namespace: "cirros-test",
Name: "cirros-test",
SkipVerifyLogs: true,
BackupRestoreType: lib.CSI,
BackupTimeout: 20 * time.Minute,
PreBackupVerify: vmPoweredOff("cirros-test", "cirros-test"),
},
}, nil),
ginkgo.Entry("immediate binding no-application CSI datamover backup and restore, CirrOS VM", ginkgo.Label("virt"), VmBackupRestoreCase{
Template: "./sample-applications/virtual-machines/cirros-test/cirros-test-immediate.yaml",
InitDelay: 2 * time.Minute, // Just long enough to get to login prompt, VM is marked running while kernel messages are still scrolling by
BackupRestoreCase: BackupRestoreCase{
Namespace: "cirros-test",
Name: "cirros-test",
SkipVerifyLogs: true,
BackupRestoreType: lib.CSIDataMover,
BackupTimeout: 20 * time.Minute,
},
}, nil),
ginkgo.Entry("immediate binding no-application CSI backup and restore, CirrOS VM", ginkgo.Label("virt"), VmBackupRestoreCase{
Template: "./sample-applications/virtual-machines/cirros-test/cirros-test-immediate.yaml",
InitDelay: 2 * time.Minute, // Just long enough to get to login prompt, VM is marked running while kernel messages are still scrolling by
BackupRestoreCase: BackupRestoreCase{
Namespace: "cirros-test",
Name: "cirros-test",
SkipVerifyLogs: true,
BackupRestoreType: lib.CSI,
BackupTimeout: 20 * time.Minute,
},
}, nil),
ginkgo.Entry("immediate binding no-application CSI+datamover backup and restore, powered-off CirrOS VM", ginkgo.Label("virt"), VmBackupRestoreCase{
Template: "./sample-applications/virtual-machines/cirros-test/cirros-test-immediate.yaml",
InitDelay: 2 * time.Minute,
PowerState: "Stopped",
BackupRestoreCase: BackupRestoreCase{
Namespace: "cirros-test",
Name: "cirros-test",
SkipVerifyLogs: true,
BackupRestoreType: lib.CSIDataMover,
BackupTimeout: 20 * time.Minute,
PreBackupVerify: vmPoweredOff("cirros-test", "cirros-test"),
},
}, nil),
ginkgo.Entry("immediate binding no-application CSI backup and restore, powered-off CirrOS VM", ginkgo.Label("virt"), VmBackupRestoreCase{
Template: "./sample-applications/virtual-machines/cirros-test/cirros-test-immediate.yaml",
InitDelay: 2 * time.Minute,
PowerState: "Stopped",
BackupRestoreCase: BackupRestoreCase{
Namespace: "cirros-test",
Name: "cirros-test",
SkipVerifyLogs: true,
BackupRestoreType: lib.CSI,
BackupTimeout: 20 * time.Minute,
PreBackupVerify: vmPoweredOff("cirros-test", "cirros-test"),
},
}, nil),
ginkgo.Entry("todolist CSI backup and restore, in a Fedora VM", ginkgo.Label("virt"), VmBackupRestoreCase{
Template: "./sample-applications/virtual-machines/fedora-todolist/fedora-todolist.yaml",
InitDelay: 3 * time.Minute, // For cloud-init
BackupRestoreCase: BackupRestoreCase{
Namespace: "mysql-persistent",
Name: "fedora-todolist",
SkipVerifyLogs: true,
BackupRestoreType: lib.CSI,
PreBackupVerify: vmTodoListReady(true, false, "mysql"),
PostRestoreVerify: vmTodoListReady(false, false, "mysql"),
BackupTimeout: 45 * time.Minute,
},
}, nil),
)
})