Add ability to snapshot notebook with generic CSI driver

If taking snapshots is possible with the current CSI driver, selecting "Use this notebook's volumes" or "Clone Notebook Volume" will create a snapshot of the notebook'ss PVC (multiple PVCs not tested but should work as well). At this point there is still an issue with getting the snapshot status when performing this procedure.
kubeflow-kale · Nov 24, 2020 · c354d25 · c354d25
1 parent 523dcbf
commit c354d25
Show file tree

Hide file tree

Showing 3 changed files with 84 additions and 19 deletions.
diff --git a/backend/kale/common/snapshotutils.py b/backend/kale/common/snapshotutils.py
@@ -150,12 +150,12 @@ def check_snapshot_status(snapshot_name):
     task = get_pvc_snapshot(snapshot_name=snapshot_name)
     status = task['status']['readyToUse']
 
-    if status is True:
-        log.info("Successfully created volume snapshot")
-    elif status is False:
-        raise RuntimeError("Snapshot not ready (status: %s)" % status)
-    else:
-        raise RuntimeError("Unknown snapshot task status: %s" % status)
+    # if status is True:
+    #    log.info("Successfully created volume snapshot")
+    # elif status is False:
+    #    raise log.info("Snapshot not ready (status: %s)" % status)
+    # else:
+    #    raise log.info("Unknown snapshot task status: %s" % status)
     return status
 
 

diff --git a/labextension/src/lib/Commands.ts b/labextension/src/lib/Commands.ts
@@ -20,6 +20,7 @@ import {
   _legacy_executeRpc,
   _legacy_executeRpcAndShowRPCError,
   RPCError,
+  IRPCError,
 } from './RPCUtils';
 import { wait } from './Utils';
 import {
@@ -37,6 +38,7 @@ import {
 } from '../widgets/VolumesPanel';
 import { IDocumentManager } from '@jupyterlab/docmanager';
 import CellUtils from './CellUtils';
+import * as React from 'react';
 
 enum RUN_CELL_STATUS {
   OK = 'ok',
@@ -73,6 +75,8 @@ interface IKatibRunArgs {
 }
 
 export default class Commands {
+  rokError: IRPCError;
+  snapshotError: IRPCError;
   private readonly _notebook: NotebookPanel;
   private readonly _kernel: Kernel.IKernelConnection;
 
@@ -89,6 +93,14 @@ export default class Commands {
     );
   };
 
+  genericsnapshotNotebook = async () => {
+    return await _legacy_executeRpcAndShowRPCError(
+      this._notebook,
+      this._kernel,
+      'snapshot.snapshot_notebook',
+    );
+  };
+
   getSnapshotProgress = async (task_id: string, ms?: number) => {
     const task = await _legacy_executeRpcAndShowRPCError(
       this._notebook,
@@ -104,18 +116,31 @@ export default class Commands {
     return task;
   };
 
+  genericgetSnapshotStatus = async (snapshot_name: string, ms?: number) => {
+    const isReady = await _legacy_executeRpcAndShowRPCError(
+      this._notebook,
+      this._kernel,
+      'snapshot.check_snapshot_status',
+      {
+        snapshot_name,
+      },
+    );
+    if (ms) {
+      await wait(ms);
+    }
+    return isReady;
+  };
+
   runSnapshotProcedure = async (onUpdate: Function) => {
     const showSnapshotProgress = true;
     const snapshot = await this.snapshotNotebook();
     const taskId = snapshot.task.id;
     let task = await this.getSnapshotProgress(taskId);
     onUpdate({ task, showSnapshotProgress });
-
     while (!['success', 'error', 'canceled'].includes(task.status)) {
       task = await this.getSnapshotProgress(taskId, 1000);
       onUpdate({ task });
     }
-
     if (task.status === 'success') {
       console.log('Snapshotting successful!');
       return task;
@@ -130,6 +155,29 @@ export default class Commands {
     return null;
   };
 
+  runGenericSnapshotProcedure = async (onUpdate: Function) => {
+    const showSnapshotProgress = true;
+    const snapshot = await this.genericsnapshotNotebook();
+    let snapshot_names = snapshot;
+    for (let i of snapshot_names) {
+      let isReady = await this.genericgetSnapshotStatus(i);
+      onUpdate({ isReady, showSnapshotProgress });
+      while ((isReady = false)) {
+        isReady = await this.genericgetSnapshotStatus(i, 1000);
+        onUpdate({ isReady });
+      }
+      if ((isReady = true)) {
+        console.log('Snapshotting successful!');
+        return isReady;
+      } else if ((isReady = false)) {
+        console.error('Snapshot not ready');
+        console.error('Stopping the deployment...');
+      }
+    }
+
+    return null;
+  };
+
   replaceClonedVolumes = async (
     bucket: string,
     obj: string,

diff --git a/labextension/src/widgets/LeftPanel.tsx b/labextension/src/widgets/LeftPanel.tsx
@@ -683,18 +683,35 @@ export class KubeflowKaleLeftPanel extends React.Component<IProps, IState> {
       metadata.volumes.filter((v: IVolumeMetadata) => v.type === 'clone')
         .length > 0
     ) {
-      const task = await commands.runSnapshotProcedure(_updateDeployProgress);
-      console.log(task);
-      if (!task) {
-        this.setState({ runDeployment: false });
-        return;
+      if (!this.props.rokError) {
+        const task = await commands.runSnapshotProcedure(_updateDeployProgress);
+        console.log(task);
+        if (!task) {
+          this.setState({ runDeployment: false });
+          return;
+        }
+        metadata.volumes = await commands.replaceClonedVolumes(
+          task.bucket,
+          task.result.event.object,
+          task.result.event.version,
+          metadata.volumes,
+        );
+      } else if (!this.props.snapshotError) {
+        const task = await commands.runGenericSnapshotProcedure(
+          _updateDeployProgress,
+        );
+        console.log(task);
+        if (!task) {
+          this.setState({ runDeployment: false });
+          return;
+        }
+        metadata.volumes = await commands.replaceClonedVolumes(
+          task.bucket,
+          task.result.event.object,
+          task.result.event.version,
+          metadata.volumes,
+        );
       }
-      metadata.volumes = await commands.replaceClonedVolumes(
-        task.bucket,
-        task.result.event.object,
-        task.result.event.version,
-        metadata.volumes,
-      );
     }
 
     // CREATE PIPELINE