diff --git a/.gitignore b/.gitignore index 15b32997e..0af94a7c0 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ yarn-error.log *.tsbuildinfo package-lock.json +backend/.vscode/settings.json diff --git a/backend/kale/common/k8sutils.py b/backend/kale/common/k8sutils.py index da530a352..9155e387f 100644 --- a/backend/kale/common/k8sutils.py +++ b/backend/kale/common/k8sutils.py @@ -19,6 +19,7 @@ _api_client = None _api_v1_client = None _k8s_co_client = None +_k8s_storage_client = None def _load_config(): @@ -57,6 +58,15 @@ def get_co_client(): return _k8s_co_client +def get_storage_client(): + """Get the Kubernetes V1beta1 StorageClient.""" + global _k8s_storage_client + if not _k8s_storage_client: + _load_config() + _k8s_storage_client = kubernetes.client.StorageV1beta1Api() + return _k8s_storage_client + + def annotate_object(group, version, plural, name, namespace, annotations): """Annotate a custom Kubernetes object.""" patch = {"apiVersion": "%s/%s" % (group, version), diff --git a/backend/kale/common/podutils.py b/backend/kale/common/podutils.py index ea2b82150..b3e65fa8a 100644 --- a/backend/kale/common/podutils.py +++ b/backend/kale/common/podutils.py @@ -145,6 +145,8 @@ def _get_mount_path(container, volume): def _list_volumes(client, namespace, pod_name, container_name): pod = client.read_namespaced_pod(pod_name, namespace) container = _get_pod_container(pod, container_name) + snapshotclass_provisioners = list_snapshotclass_storage_provisioners() + provisioner_names = get_snapshotclass_provisioners_names() rok_volumes = [] for volume in pod.spec.volumes: @@ -158,18 +160,23 @@ def _list_volumes(client, namespace, pod_name, container_name): # result in an incomplete notebook snapshot. pvc = client.read_namespaced_persistent_volume_claim(pvc.claim_name, namespace) - if pvc.spec.storage_class_name != ROK_CSI_STORAGE_CLASS: - msg = ("Found PVC with storage class '%s'. Only storage class '%s'" - " is supported." - % (pvc.spec.storage_class_name, ROK_CSI_STORAGE_CLASS)) + if (pvc.spec.storage_class_name != ROK_CSI_STORAGE_CLASS + and pvc.spec.storage_class_name not in provisioner_names): + msg = ("Found PVC with storage class '%s'. " + "Only storage classes able to take snapshots and " + "'%s' are supported." + % (pvc.spec.storage_class_name, + ROK_CSI_STORAGE_CLASS)) raise RuntimeError(msg) ann = pvc.metadata.annotations provisioner = ann.get("volume.beta.kubernetes.io/storage-provisioner", None) - if provisioner != ROK_CSI_STORAGE_PROVISIONER: - msg = ("Found PVC storage provisioner '%s'. Only storage" - " provisioner '%s' is supported." + if (provisioner != ROK_CSI_STORAGE_PROVISIONER + and provisioner not in snapshotclass_provisioners): + msg = ("Found PVC storage provisioner '%s'. " + "Only storage provisioners able to take snapshots and " + "'%s' are supported." % (provisioner, ROK_CSI_STORAGE_PROVISIONER)) raise RuntimeError(msg) @@ -300,3 +307,63 @@ def compute_component_id(pod): component_id = component_name + "@sha256=" + component_spec_digest log.info("Computed component ID: %s", component_id) return component_id + + +def get_snapshotclasses(label_selector=""): + """List snapshotclasses.""" + co_client = k8sutils.get_co_client() + + snapshotclasses = co_client.list_cluster_custom_object( + group="snapshot.storage.k8s.io", + version="v1beta1", + plural="volumesnapshotclasses", + label_selector=label_selector) + return snapshotclasses + + +def list_snapshotclass_storage_provisioners(label_selector=""): + """List the storage provisioners of the snapshotclasses.""" + snapshotclasses = get_snapshotclasses(label_selector)["items"] + snapshotclass_provisioners = [] + for i in snapshotclasses: + snapshotclass_provisioners.append(i["driver"]) + return snapshotclass_provisioners + + +def check_snapshot_availability(): + """Check if snapshotclasses are available for notebook.""" + client = k8sutils.get_v1_client() + namespace = get_namespace() + pod_name = get_pod_name() + pod = client.read_namespaced_pod(pod_name, namespace) + snapshotclass_provisioners = list_snapshotclass_storage_provisioners() + + for volume in pod.spec.volumes: + pvc = volume.persistent_volume_claim + if not pvc: + continue + pvc = client.read_namespaced_persistent_volume_claim(pvc.claim_name, + namespace) + ann = pvc.metadata.annotations + provisioner = ann.get("volume.beta.kubernetes.io/storage-provisioner", + None) + if provisioner not in snapshotclass_provisioners: + msg = ("Found PVC storage provisioner '%s'. " + "Only storage provisioners able to take snapshots " + "are supported." + % (provisioner)) + raise RuntimeError(msg) + + +def get_snapshotclass_provisioners_names(): + """Get the names of snapshotclass storage provisioners.""" + client = k8sutils.get_storage_client() + classes = client.list_storage_class().items + snapshotclass_provisioners = list_snapshotclass_storage_provisioners() + storage_class_names = [] + for i in classes: + name = i.metadata.name + provisioner = i.provisioner + if provisioner in snapshotclass_provisioners: + storage_class_names.append(name) + return storage_class_names diff --git a/backend/kale/common/snapshotutils.py b/backend/kale/common/snapshotutils.py new file mode 100644 index 000000000..c3ac0a2f6 --- /dev/null +++ b/backend/kale/common/snapshotutils.py @@ -0,0 +1,367 @@ +# Copyright 2019-2020 The Kale Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# To allow a notebook to create Rook CephFS snapshot a ClusterRole +# and RoleBinding for the default-editor service account in the +# given namespace must be applied. Below are examples for use with the +# namespace "admin". +# +# !!!WARNING!!! +# Might not be secure, only use for testing +# +# apiVersion: rbac.authorization.k8s.io/v1 +# kind: ClusterRole +# metadata: +# name: snapshot-access +# rules: +# - apiGroups: ["snapshot.storage.k8s.io"] +# resources: ["volumesnapshots"] +# verbs: ["create", "get", "list", "watch", "patch", "delete"] +# - apiGroups: ["snapshot.storage.k8s.io"] +# resources: ["volumesnapshotcontents"] +# verbs: ["create", "get", "list", "watch", "update", "delete"] +# - apiGroups: ["snapshot.storage.k8s.io"] +# resources: ["volumesnapshotclasses"] +# verbs: ["get", "list", "watch"] +# - apiGroups: ["snapshot.storage.k8s.io"] +# resources: ["volumesnapshotcontents/status"] +# verbs: ["update"] +# - apiGroups: ["snapshot.storage.k8s.io"] +# resources: ["volumesnapshots/status"] +# verbs: ["update"] +# +# apiVersion: rbac.authorization.k8s.io/v1 +# kind: RoleBinding +# metadata: +# name: allow-snapshot-nb-admin +# namespace: admin +# subjects: +# - kind: ServiceAccount +# name: default-editor +# namespace: admin +# roleRef: +# kind: ClusterRole +# name: snapshot-access +# apiGroup: rbac.authorization.k8s.io + +import logging +import kubernetes +import time + +from kale.common import podutils, k8sutils + +NOTEBOOK_SNAPSHOT_COMMIT_MESSAGE = """\ +This is a snapshot of notebook {} in namespace {}. +This snapshot was created by Kale in order to clone the volumes of the notebook +and use them to spawn a Kubeflow pipeline.\ +""" + +log = logging.getLogger(__name__) + + +def snapshot_pvc(snapshot_name, pvc_name, image="", path="", **kwargs): + """Perform a snapshot over a PVC.""" + snapshot_resource = { + "apiVersion": "snapshot.storage.k8s.io/v1beta1", + "kind": "VolumeSnapshot", + "metadata": { + "name": snapshot_name, + "annotations": { + "container_image": image, + "volume_path": path + }, + "labels": kwargs + }, + "spec": { + "volumeSnapshotClassName": "csi-cephfsplugin-snapclass", + "source": {"persistentVolumeClaimName": pvc_name} + } + } + co_client = k8sutils.get_co_client() + namespace = podutils.get_namespace() + log.info("Taking a snapshot of PVC %s in namespace %s ..." + % (pvc_name, namespace)) + task_info = co_client.create_namespaced_custom_object( + group="snapshot.storage.k8s.io", + version="v1beta1", + namespace=namespace, + plural="volumesnapshots", + body=snapshot_resource) + + return task_info + + +def snapshot_pod(): + """Take snapshots of the current Pod's PVCs.""" + volumes = [(path, volume.name, size) + for path, volume, size in podutils.list_volumes()] + namespace = podutils.get_namespace() + pod_name = podutils.get_pod_name() + log.info("Taking a snapshot of pod %s in namespace %s ..." + % (pod_name, namespace)) + snapshot_names = [] + for i in volumes: + snapshot_pvc( + "pod-snapshot-" + i[1], + i[1], + pod=pod_name, + default_container=podutils.get_container_name()) + snapshot_names.append("snapshot." + i[1]) + return snapshot_names + + +def snapshot_notebook(): + """Take snapshots of the current Notebook's PVCs and store its metadata.""" + volumes = [(path, volume.name, size) + for path, volume, size in podutils.list_volumes()] + namespace = podutils.get_namespace() + pod_name = podutils.get_pod_name() + log.info("Taking a snapshot of notebook %s in namespace %s ..." + % (pod_name, namespace)) + snapshot_names = [] + for i in volumes: + snapshot_pvc( + "nb-snapshot-" + i[1], + i[1], + image=podutils.get_docker_base_image(), + path=i[0], + pod=pod_name, + default_container=podutils.get_container_name(), + is_workspace_dir=str(podutils.is_workspace_dir(i[0]))) + snapshot_names.append("nb-snapshot-" + i[1]) + return snapshot_names + + +def check_snapshot_status(snapshot_name): + """Check if volume snapshot is ready to use.""" + log.info("Checking snapshot with snapshot name: %s", snapshot_name) + count = 0 + max_count = 60 + task = None + status = None + while status is not True and count <= max_count: + count += 1 + try: + task = get_pvc_snapshot(snapshot_name=snapshot_name) + status = task['status']['readyToUse'] + log.info(task) + log.info(status) + time.sleep(2) + except KeyError: + log.info("Snapshot resource %s does not seem to be ready", snapshot_name) + time.sleep(2) + if status is True: + log.info("Successfully created volume snapshot") + elif status is False: + raise log.info("Snapshot not ready (status: %s)" % status) + else: + raise log.info("Unknown snapshot task status: %s" % status) + return task + + +def get_pvc_snapshot(snapshot_name): + """Get info about a pvc snapshot.""" + co_client = k8sutils.get_co_client() + namespace = podutils.get_namespace() + + pvc_snapshot = co_client.get_namespaced_custom_object( + group="snapshot.storage.k8s.io", + version="v1beta1", + namespace=namespace, + plural="volumesnapshots", + name=snapshot_name) + return pvc_snapshot + + +def list_pvc_snapshots(label_selector=""): + """List pvc snapshots.""" + co_client = k8sutils.get_co_client() + namespace = podutils.get_namespace() + + pvc_snapshots = co_client.list_namespaced_custom_object( + group="snapshot.storage.k8s.io", + version="v1beta1", + namespace=namespace, + plural="volumesnapshots", + label_selector=label_selector) + return pvc_snapshots + + +def hydrate_pvc_from_snapshot(new_pvc_name, source_snapshot_name): + """Create a new PVC out of a volume snapshot.""" + log.info("Creating new PVC '%s' from snapshot %s ..." % + (new_pvc_name, source_snapshot_name)) + snapshot_info = get_pvc_snapshot(source_snapshot_name) + size_repr = snapshot_info['status']['restoreSize'] + content_name = snapshot_info['status']['boundVolumeSnapshotContentName'] + log.info("Using snapshot with content: %s" % content_name) + + # todo: kubernetes python client v11 have a + # kubernetes.utils.create_from_dict that would make it much more nicer + # here. (KFP support kubernetes <= 10) + pvc = kubernetes.client.V1PersistentVolumeClaim( + api_version="v1", + kind="PersistentVolumeClaim", + metadata=kubernetes.client.V1ObjectMeta( + annotations={"snapshot_origin": content_name}, + name=new_pvc_name + ), + spec=kubernetes.client.V1PersistentVolumeClaimSpec( + data_source=kubernetes.client.V1TypedLocalObjectReference( + api_group="snapshot.storage.k8s.io", + kind="VolumeSnapshot", + name=source_snapshot_name + ), + access_modes=["ReadWriteMany"], + resources=kubernetes.client.V1ResourceRequirements( + requests={"storage": size_repr} + ) + ) + ) + k8s_client = k8sutils.get_v1_client() + ns = podutils.get_namespace() + status = check_snapshot_status(source_snapshot_name)['status']['readyToUse'] + if status is True: + ns_pvc = k8s_client.create_namespaced_persistent_volume_claim(ns, pvc) + elif status is False: + raise RuntimeError("Snapshot not ready (status: %s)" % status) + else: + raise RuntimeError("Unknown Rok task status: %s" % status) + return {"name": ns_pvc.metadata.name} + + +def get_nb_name_from_snapshot(snapshot_name): + """Get the name of the notebook that the snapshot was taken from.""" + snapshot = get_pvc_snapshot(snapshot_name=snapshot_name) + orig_notebook = snapshot["metadata"]["labels"]["default_container"] + return orig_notebook + + +def get_nb_image_from_snapshot(snapshot_name): + """Get the image of the notebook that the snapshot was taken from.""" + snapshot = get_pvc_snapshot(snapshot_name=snapshot_name) + image = snapshot["metadata"]["annotations"]["container_image"] + return image + + +def get_nb_pvcs_from_snapshot(snapshot_name): + """Get all the PVCs that were mounted to the NB when the snapshot was taken. + + Returns JSON list. + """ + selector = "default_container=" + get_nb_name_from_snapshot(snapshot_name) + all_volumes = list_pvc_snapshots(label_selector=selector)["items"] + volumes = [] + for i in all_volumes: + snapshot_name = i["metadata"]["name"] + source_pvc_name = i["spec"]["source"]["persistentVolumeClaimName"] + path = i["metadata"]["annotations"]["volume_path"] + row = { + "mountPath": path, + "snapshot_name": snapshot_name, + "source_pvc": source_pvc_name} + volumes.append(row) + return volumes + + +def restore_pvcs_from_snapshot(snapshot_name): + """Restore the NB PVCs from their snapshots.""" + source_snapshots = get_nb_pvcs_from_snapshot(snapshot_name) + replaced_volume_mounts = [] + for i in source_snapshots: + new_pvc_name = "restored-" + i["source_pvc"] + pvc_name = hydrate_pvc_from_snapshot(new_pvc_name, i["snapshot_name"]) + path = i["mountPath"] + row = {"mountPath": path, "name": pvc_name["name"]} + replaced_volume_mounts.append(row) + return replaced_volume_mounts + + +def replace_cloned_volumes(volume_mounts): + """Replace the volumes with the volumes restored from the snapshot.""" + replaced_volumes = [] + for i in volume_mounts: + name = i["name"] + row = {"name": name, "persistentVolumeClaim": {"claimName": name}} + replaced_volumes.append(row) + return replaced_volumes + + +def restore_notebook(snapshot_name): + """Restore a notebook from a PVC snapshot.""" + name = "restored-" + get_nb_name_from_snapshot(snapshot_name) + namespace = podutils.get_namespace() + image = get_nb_image_from_snapshot(snapshot_name) + volume_mounts = restore_pvcs_from_snapshot(snapshot_name) + volumes = replace_cloned_volumes(volume_mounts) + notebook_resource = { + "apiVersion": "kubeflow.org/v1alpha1", + "kind": "Notebook", + "metadata": { + "labels": { + "app": name + }, + "name": name, + "namespace": namespace}, + "spec": { + "template": { + "spec": { + "containers": [ + { + "env": [], + "image": image, + "name": name, + "resources": { + "requests": { + "cpu": "0.5", + "memory": "1.0Gi"}}, + "volumeMounts": volume_mounts}], + "serviceAccountName": "default-editor", + "ttlSecondsAfterFinished": 300, + "volumes": volumes}}}} + co_client = k8sutils.get_co_client() + log.info("Restoring notebook %s from PVC snapshot %s in namespace %s ..." + % (name, snapshot_name, namespace)) + task_info = co_client.create_namespaced_custom_object( + group="kubeflow.org", + version="v1alpha1", + namespace=namespace, + plural="notebooks", + body=notebook_resource) + + return task_info + + +def delete_pvc(pvc_name): + """Delete a pvc.""" + client = k8sutils.get_v1_client() + namespace = podutils.get_namespace() + client.delete_namespaced_persistent_volume_claim( + namespace=namespace, + name=pvc_name) + return + + +def delete_pvc_snapshot(snapshot_name): + """Delete a pvc snapshot.""" + co_client = podutils._get_k8s_custom_objects_client() + namespace = podutils.get_namespace() + + co_client.delete_namespaced_custom_object( + group="snapshot.storage.k8s.io", + version="v1beta1", + namespace=namespace, + plural="volumesnapshots", + name=snapshot_name) + return diff --git a/backend/kale/rpc/snapshot.py b/backend/kale/rpc/snapshot.py new file mode 100644 index 000000000..ce444635b --- /dev/null +++ b/backend/kale/rpc/snapshot.py @@ -0,0 +1,50 @@ +# Copyright 2019-2020 The Kale Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from kale.common import podutils, snapshotutils +from kale.rpc.errors import (RPCServiceUnavailableError) +from kale.rpc.log import create_adapter + + +logger = create_adapter(logging.getLogger(__name__)) + + +def check_snapshot_availability(request): + """Check if snapshotclasses are available for notebook.""" + log = request.log if hasattr(request, "log") else logger + try: + podutils.check_snapshot_availability() + except Exception: + log.exception("No snapshotclass is available for this notebook") + raise RPCServiceUnavailableError(details=("No snapshotclass" + " is available for" + " this notebook"), + trans_id=request.trans_id) + + +def check_snapshot_status(request, snapshot_name): + """Check if volume snapshot is ready to use.""" + return snapshotutils.check_snapshot_status(snapshot_name) + + +def snapshot_notebook(request): + """Take snapshots of the current Notebook's PVCs and store its metadata.""" + return snapshotutils.snapshot_notebook() + + +def replace_cloned_volumes(request, volume_mounts): + """Replace the volumes with the volumes restored from the snapshot.""" + return snapshotutils.replace_cloned_volumes(volume_mounts) diff --git a/backend/kale/templates/pipeline_template.jinja2 b/backend/kale/templates/pipeline_template.jinja2 index 414450640..0200ff515 100644 --- a/backend/kale/templates/pipeline_template.jinja2 +++ b/backend/kale/templates/pipeline_template.jinja2 @@ -91,6 +91,9 @@ def auto_generated_pipeline({%- for arg in pipeline.pps_names -%} storage_class="{{ storage_class_name }}", {%- endif %} size='{{ pvc_size }}' + {%- if data_source_name %} + data_source="{{ data_source_name }}", + {%- endif %} ) _kale_volume = _kale_vop{{ loop.index }}.volume _kale_volume_step_names.append(_kale_vop{{ loop.index }}.name) diff --git a/labextension/src/lib/Commands.ts b/labextension/src/lib/Commands.ts index 9892e01ff..54913436e 100644 --- a/labextension/src/lib/Commands.ts +++ b/labextension/src/lib/Commands.ts @@ -20,6 +20,7 @@ import { _legacy_executeRpc, _legacy_executeRpcAndShowRPCError, RPCError, + IRPCError, } from './RPCUtils'; import { wait } from './Utils'; import { @@ -37,6 +38,7 @@ import { } from '../widgets/VolumesPanel'; import { IDocumentManager } from '@jupyterlab/docmanager'; import CellUtils from './CellUtils'; +import * as React from 'react'; enum RUN_CELL_STATUS { OK = 'ok', @@ -73,6 +75,8 @@ interface IKatibRunArgs { } export default class Commands { + rokError: IRPCError; + snapshotError: IRPCError; private readonly _notebook: NotebookPanel; private readonly _kernel: Kernel.IKernelConnection; @@ -89,6 +93,14 @@ export default class Commands { ); }; + genericsnapshotNotebook = async () => { + return await _legacy_executeRpcAndShowRPCError( + this._notebook, + this._kernel, + 'snapshot.snapshot_notebook', + ); + }; + getSnapshotProgress = async (task_id: string, ms?: number) => { const task = await _legacy_executeRpcAndShowRPCError( this._notebook, @@ -104,18 +116,31 @@ export default class Commands { return task; }; + genericgetSnapshotStatus = async (snapshot_name: string, ms?: number) => { + const isReady = await _legacy_executeRpcAndShowRPCError( + this._notebook, + this._kernel, + 'snapshot.check_snapshot_status', + { + snapshot_name, + }, + ); + if (ms) { + await wait(ms); + } + return isReady; + }; + runSnapshotProcedure = async (onUpdate: Function) => { const showSnapshotProgress = true; const snapshot = await this.snapshotNotebook(); const taskId = snapshot.task.id; let task = await this.getSnapshotProgress(taskId); onUpdate({ task, showSnapshotProgress }); - while (!['success', 'error', 'canceled'].includes(task.status)) { task = await this.getSnapshotProgress(taskId, 1000); onUpdate({ task }); } - if (task.status === 'success') { console.log('Snapshotting successful!'); return task; @@ -130,6 +155,29 @@ export default class Commands { return null; }; + runGenericSnapshotProcedure = async (onUpdate: Function) => { + const showSnapshotProgress = true; + const snapshot = await this.genericsnapshotNotebook(); + let snapshot_names = snapshot; + for (let i of snapshot_names) { + let isReady = await this.genericgetSnapshotStatus(i); + onUpdate({ isReady, showSnapshotProgress }); + while ((isReady = false)) { + isReady = await this.genericgetSnapshotStatus(i, 1000); + onUpdate({ isReady }); + } + if ((isReady = true)) { + console.log('Snapshotting successful!'); + return isReady; + } else if ((isReady = false)) { + console.error('Snapshot not ready'); + console.error('Stopping the deployment...'); + } + } + + return null; + }; + replaceClonedVolumes = async ( bucket: string, obj: string, @@ -149,6 +197,19 @@ export default class Commands { ); }; + replaceGenericClonedVolumes = async ( + volumes: IVolumeMetadata[], + ) => { + return await _legacy_executeRpcAndShowRPCError( + this._notebook, + this._kernel, + 'snapshot.replace_cloned_volumes', + { + volumes, + }, + ); + }; + getMountedVolumes = async (currentNotebookVolumes: IVolumeMetadata[]) => { let notebookVolumes: IVolumeMetadata[] = await _legacy_executeRpcAndShowRPCError( this._notebook, diff --git a/labextension/src/lib/RPCUtils.tsx b/labextension/src/lib/RPCUtils.tsx index ef593a42f..ef11e2779 100644 --- a/labextension/src/lib/RPCUtils.tsx +++ b/labextension/src/lib/RPCUtils.tsx @@ -84,6 +84,17 @@ export const rokErrorTooltip = (rokError: IRPCError) => { ); }; +export const snapshotErrorTooltip = (snapshotError: IRPCError) => { + return ( + +
+ This feature requires snapshot support.{' '} + showRpcError(snapshotError)}>More info... +
+ + ); +}; + const serialize = (obj: any) => window.btoa(JSON.stringify(obj)); const deserialize = (raw_data: string) => window.atob(raw_data.substring(1, raw_data.length - 1)); diff --git a/labextension/src/widget.tsx b/labextension/src/widget.tsx index 65649b3b2..c9dcb06f8 100644 --- a/labextension/src/widget.tsx +++ b/labextension/src/widget.tsx @@ -84,6 +84,7 @@ async function activate( // env we are in (like Local Laptop, MiniKF, GCP, UI without Kale, ...) const backend = await getBackend(kernel); let rokError: IRPCError = null; + let snapshotError: IRPCError = null; if (backend) { try { await executeRpc(kernel, 'log.setup_logging'); @@ -111,6 +112,26 @@ async function activate( throw error; } } + + try { + await executeRpc(kernel, 'snapshot.check_snapshot_availability'); + } catch (error) { + const unexpectedErrorCodes = [ + RPC_CALL_STATUS.EncodingError, + RPC_CALL_STATUS.ImportError, + RPC_CALL_STATUS.UnhandledError, + ]; + if ( + error instanceof RPCError && + !unexpectedErrorCodes.includes(error.error.code) + ) { + snapshotError = error.error; + console.warn('Snapshots are not available', snapshotError); + } else { + globalUnhandledRejection({ reason: error }); + throw error; + } + } } else { rokError = { rpc: 'rok.check_rok_availability', @@ -175,6 +196,7 @@ async function activate( backend={backend} kernel={kernel} rokError={rokError} + snapshotError={snapshotError} />, ); widget.id = 'kubeflow-kale/kubeflowDeployment'; diff --git a/labextension/src/widgets/LeftPanel.tsx b/labextension/src/widgets/LeftPanel.tsx index ca9fb34f8..7f794eb40 100644 --- a/labextension/src/widgets/LeftPanel.tsx +++ b/labextension/src/widgets/LeftPanel.tsx @@ -60,6 +60,7 @@ interface IProps { backend: boolean; kernel: Kernel.IKernelConnection; rokError: IRPCError; + snapshotError: IRPCError; } interface IState { @@ -442,6 +443,16 @@ export class KubeflowKaleLeftPanel extends React.Component { notebookVolumes, selectVolumeTypes, }); + } else if (!this.props.snapshotError) { + // Get information about volumes currently mounted on the notebook server + const { + notebookVolumes, + selectVolumeTypes, + } = await commands.getMountedVolumes(this.state.notebookVolumes); + this.setState({ + notebookVolumes, + selectVolumeTypes, + }); } else { this.setState((prevState, props) => ({ selectVolumeTypes: prevState.selectVolumeTypes.map(t => { @@ -532,8 +543,25 @@ export class KubeflowKaleLeftPanel extends React.Component { } return volume; }); + let snapstateVolumes = this.props.snapshotError + ? metadataVolumes + : metadataVolumes.map((volume: IVolumeMetadata) => { + if ( + volume.type === 'new_pvc' && + volume.annotations.length > 0 && + volume.annotations[0].key === 'rok/origin' + ) { + return { ...volume, type: 'snap' }; + } + return volume; + }); if (stateVolumes.length === 0 && metadataVolumes.length === 0) { metadataVolumes = stateVolumes = this.state.notebookVolumes; + } else if ( + snapstateVolumes.length === 0 && + metadataVolumes.length === 0 + ) { + metadataVolumes = snapstateVolumes = this.state.notebookVolumes; } else { metadataVolumes = metadataVolumes.concat(this.state.notebookVolumes); stateVolumes = stateVolumes.concat(this.state.notebookVolumes); @@ -557,11 +585,15 @@ export class KubeflowKaleLeftPanel extends React.Component { }, autosnapshot: notebookMetadata['autosnapshot'] === undefined - ? !this.props.rokError && this.state.notebookVolumes.length > 0 + ? !this.props.rokError && + !this.props.snapshotError && + this.state.notebookVolumes.length > 0 : notebookMetadata['autosnapshot'], snapshot_volumes: notebookMetadata['snapshot_volumes'] === undefined - ? !this.props.rokError && this.state.notebookVolumes.length > 0 + ? !this.props.rokError && + !this.props.snapshotError && + this.state.notebookVolumes.length > 0 : notebookMetadata['snapshot_volumes'], // fixme: for now we are using the 'steps_defaults' field just for poddefaults // so we replace any existing value every time @@ -577,9 +609,13 @@ export class KubeflowKaleLeftPanel extends React.Component { ...DefaultState.metadata, volumes: prevState.notebookVolumes, snapshot_volumes: - !this.props.rokError && prevState.notebookVolumes.length > 0, + !this.props.rokError && + !this.props.snapshotError && + prevState.notebookVolumes.length > 0, autosnapshot: - !this.props.rokError && prevState.notebookVolumes.length > 0, + !this.props.rokError && + !this.props.snapshotError && + prevState.notebookVolumes.length > 0, }, volumes: prevState.notebookVolumes, })); @@ -647,18 +683,32 @@ export class KubeflowKaleLeftPanel extends React.Component { metadata.volumes.filter((v: IVolumeMetadata) => v.type === 'clone') .length > 0 ) { - const task = await commands.runSnapshotProcedure(_updateDeployProgress); - console.log(task); - if (!task) { - this.setState({ runDeployment: false }); - return; + if (!this.props.rokError) { + const task = await commands.runSnapshotProcedure(_updateDeployProgress); + console.log(task); + if (!task) { + this.setState({ runDeployment: false }); + return; + } + metadata.volumes = await commands.replaceClonedVolumes( + task.bucket, + task.result.event.object, + task.result.event.version, + metadata.volumes, + ); + } else if (!this.props.snapshotError) { + const task = await commands.runGenericSnapshotProcedure( + _updateDeployProgress, + ); + console.log(task); + if (!task) { + this.setState({ runDeployment: false }); + return; + } + metadata.volumes = await commands.replaceGenericClonedVolumes( + metadata.volumes, + ); } - metadata.volumes = await commands.replaceClonedVolumes( - task.bucket, - task.result.event.object, - task.result.event.version, - metadata.volumes, - ); } // CREATE PIPELINE @@ -817,6 +867,7 @@ export class KubeflowKaleLeftPanel extends React.Component { autosnapshot={this.state.metadata.autosnapshot} updateAutosnapshotSwitch={this.updateAutosnapshotSwitch} rokError={this.props.rokError} + snapshotError={this.props.snapshotError} updateVolumes={this.updateVolumes} storageClassName={this.state.metadata.storage_class_name} updateStorageClassName={this.updateStorageClassName} diff --git a/labextension/src/widgets/VolumesPanel.tsx b/labextension/src/widgets/VolumesPanel.tsx index 708044df4..5fffc57b0 100644 --- a/labextension/src/widgets/VolumesPanel.tsx +++ b/labextension/src/widgets/VolumesPanel.tsx @@ -19,7 +19,11 @@ import { Button, Switch, Zoom } from '@material-ui/core'; import AddIcon from '@material-ui/icons/Add'; import DeleteIcon from '@material-ui/icons/Delete'; import { IVolumeMetadata } from './LeftPanel'; -import { IRPCError, rokErrorTooltip } from '../lib/RPCUtils'; +import { + IRPCError, + rokErrorTooltip, + snapshotErrorTooltip, +} from '../lib/RPCUtils'; import { Input } from '../components/Input'; import { Select, ISelectOption } from '../components/Select'; import { LightTooltip } from '../components/LightTooltip'; @@ -93,6 +97,7 @@ interface VolumesPanelProps { useNotebookVolumes: boolean; autosnapshot: boolean; rokError: IRPCError; + snapshotError: IRPCError; updateVolumes: Function; updateVolumesSwitch: Function; updateAutosnapshotSwitch: Function; @@ -118,9 +123,9 @@ export const VolumesPanel: React.FunctionComponent = props => const addVolume = () => { // If we add a volume to an empty list, turn autosnapshot on const autosnapshot = - !props.rokError && props.volumes.length === 0 + !props.rokError && !props.snapshotError && props.volumes.length === 0 ? true - : !props.rokError && props.autosnapshot; + : !props.rokError && !props.snapshotError && props.autosnapshot; props.updateVolumes( [...props.volumes, DEFAULT_EMPTY_VOLUME], [...props.metadataVolumes, DEFAULT_EMPTY_VOLUME], @@ -517,13 +522,16 @@ export const VolumesPanel: React.FunctionComponent = props =>
@@ -531,7 +539,8 @@ export const VolumesPanel: React.FunctionComponent = props => props.updateVolumesSwitch()} color="primary" @@ -548,13 +557,16 @@ export const VolumesPanel: React.FunctionComponent = props =>
@@ -563,7 +575,10 @@ export const VolumesPanel: React.FunctionComponent = props =>
props.updateAutosnapshotSwitch()} color="primary" name="enableKale"