feat: add cloud storage commands

.gitignore

@@ -94,6 +94,7 @@ helm-chart/renku-core/charts
 renku/templates/
 temp/
 tmp/
+.ropeproject/
 
 # pytest-recording cache
 cassettes

docs/reference/commands/storage.rst

@@ -3,4 +3,4 @@
 renku storage
 *************
 
-.. automodule:: renku.ui.cli.storage
+.. automodule:: renku.ui.cli.lfs

renku/command/checks/storage.py

@@ -16,7 +16,7 @@
 """Check for large files in Git history."""
 
 from renku.command.util import WARNING
-from renku.core.storage import check_external_storage, check_lfs_migrate_info
+from renku.core.lfs import check_external_storage, check_lfs_migrate_info
 
 
 def check_lfs_info(**_):

renku/command/command_builder/command.py

@@ -17,6 +17,7 @@
 
 import contextlib
 import functools
+import shutil
 import threading
 from collections import defaultdict
 from pathlib import Path
@@ -455,6 +456,13 @@ def require_clean(self) -> "Command":
 
         return RequireClean(self)
 
+    @check_finalized
+    def require_login(self) -> "Command":
+        """Check that the user is logged in."""
+        from renku.command.command_builder.repo import RequireLogin
+
+        return RequireLogin(self)
+
     @check_finalized
     def with_communicator(self, communicator: CommunicationCallback) -> "Command":
         """Create a communicator.
@@ -496,3 +504,26 @@ def __init__(self, output, error, status) -> None:
         self.output = output
         self.error = error
         self.status = status
+
+
+class RequireExecutable(Command):
+    """Builder to check if an executable is installed."""
+
+    HOOK_ORDER = 4
+
+    def __init__(self, builder: Command, executable: str) -> None:
+        """__init__ of RequireExecutable."""
+        self._builder = builder
+        self._executable = executable
+
+    def _pre_hook(self, builder: Command, context: dict, *args, **kwargs) -> None:
+        """Check if an executable exists on the system.
+
+        Args:
+            builder(Command): Current ``CommandBuilder``.
+            context(dict): Current context.
+        """
+        if not shutil.which(self._executable):
+            raise errors.ExecutableNotFound(
+                f"Couldn't find the executable '{self._executable}' on this system. Please make sure it's installed"
+            )

renku/command/command_builder/repo.py

@@ -21,6 +21,7 @@
 from renku.command.command_builder.command import Command, CommandResult, check_finalized
 from renku.core import errors
 from renku.core.git import ensure_clean
+from renku.core.login import ensure_login
 from renku.domain_model.project_context import project_context
 
 
@@ -42,6 +43,7 @@ def __init__(
         """__init__ of Commit.
 
         Args:
+            builder(Command): The current ``CommandBuilder``.
             message (str): The commit message. Auto-generated if left empty (Default value = None).
             commit_if_empty (bool): Whether to commit if there are no modified files (Default value = None).
             raise_if_empty (bool): Whether to raise an exception if there are no modified files (Default value = None).
@@ -164,6 +166,39 @@ def build(self) -> Command:
         return self._builder.build()
 
 
+class RequireLogin(Command):
+    """Builder to check if a user is logged in."""
+
+    HOOK_ORDER = 4
+
+    def __init__(self, builder: Command) -> None:
+        """__init__ of RequireLogin."""
+        self._builder = builder
+
+    def _pre_hook(self, builder: Command, context: dict, *args, **kwargs) -> None:
+        """Check if the user is logged in.
+
+        Args:
+            builder(Command): Current ``CommandBuilder``.
+            context(dict): Current context.
+        """
+        if not project_context.has_context():
+            raise ValueError("RequireLogin builder needs a ProjectContext to be set.")
+
+        ensure_login()
+
+    @check_finalized
+    def build(self) -> Command:
+        """Build the command.
+
+        Returns:
+            Command: Finalized version of this command.
+        """
+        self._builder.add_pre_hook(self.HOOK_ORDER, self._pre_hook)
+
+        return self._builder.build()
+
+
 class Isolation(Command):
     """Builder to run a command in git isolation."""
 

renku/command/format/storage.py

@@ -0,0 +1,65 @@
+# Copyright Swiss Data Science Center (SDSC). A partnership between
+# École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Serializers for storage."""
+
+import json
+from typing import List, Optional
+
+from renku.command.format.tabulate import tabulate
+from renku.domain_model.cloud_storage import CloudStorage
+
+
+def tabular(cloud_storages: List[CloudStorage], *, columns: Optional[str] = None):
+    """Format cloud_storages with a tabular output."""
+    if not columns:
+        columns = "id,start_time,status,provider,url"
+
+        if any(s.ssh_enabled for s in cloud_storages):
+            columns += ",ssh"
+
+    return tabulate(collection=cloud_storages, columns=columns, columns_mapping=cloud_storage_COLUMNS)
+
+
+def log(cloud_storages: List[CloudStorage], *, columns: Optional[str] = None):
+    """Format cloud_storages in a log like output."""
+    from renku.ui.cli.utils.terminal import style_header, style_key
+
+    output = []
+
+    for cloud_storage in cloud_storages:
+        output.append(style_header(f"CloudStorage {cloud_storage.name}"))
+        output.append(style_key("Id: ") + cloud_storage.storage_id)
+        output.append(style_key("Source Path: ") + cloud_storage.source_path)
+        output.append(style_key("Target path: ") + cloud_storage.target_path)
+        output.append(style_key("Private: ") + "Yes" if cloud_storage.private else "No")
+        output.append(style_key("Configuration: \n") + json.dumps(cloud_storage.configuration, indent=4))
+        output.append("")
+    return "\n".join(output)
+
+
+CLOUD_STORAGE_FORMATS = {"tabular": tabular, "log": log}
+"""Valid formatting options."""
+
+CLOUD_STORAGE_COLUMNS = {
+    "id": ("id", "id"),
+    "status": ("status", "status"),
+    "url": ("url", "url"),
+    "ssh": ("ssh_enabled", "SSH enabled"),
+    "start_time": ("start_time", "start_time"),
+    "commit": ("commit", "commit"),
+    "branch": ("branch", "branch"),
+    "provider": ("provider", "provider"),
+}

renku/command/lfs.py

@@ -0,0 +1,137 @@
+#
+# Copyright 2018-2023- Swiss Data Science Center (SDSC)
+# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
+# Eidgenössische Technische Hochschule Zürich (ETHZ).
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Renku storage command."""
+
+from typing import List
+
+from pydantic import validate_arguments
+
+from renku.command.command_builder.command import Command
+from renku.core.lfs import (
+    check_lfs_migrate_info,
+    check_requires_tracking,
+    clean_storage_cache,
+    migrate_files_to_lfs,
+    pull_paths_from_storage,
+)
+from renku.core.util import communication
+from renku.domain_model.project_context import project_context
+
+
+@validate_arguments(config=dict(arbitrary_types_allowed=True))
+def _check_lfs(everything: bool = False):
+    """Check if large files are not in lfs.
+
+    Args:
+        everything: Whether to check whole history (Default value = False).
+
+    Returns:
+        List of large files.
+    """
+    files = check_lfs_migrate_info(everything)
+
+    if files:
+        communication.warn("Git history contains large files\n\t" + "\n\t".join(files))
+
+    return files
+
+
+def check_lfs_command():
+    """Check lfs command."""
+    return Command().command(_check_lfs)
+
+
+@validate_arguments(config=dict(arbitrary_types_allowed=True))
+def _fix_lfs(paths: List[str]):
+    """Migrate large files into lfs.
+
+    Args:
+        paths(List[str]): Paths to migrate to LFS.
+    """
+    migrate_files_to_lfs(paths)
+
+
+def fix_lfs_command():
+    """Fix lfs command."""
+    return (
+        Command()
+        .command(_fix_lfs)
+        .require_clean()
+        .require_migration()
+        .with_database(write=True)
+        .with_commit(commit_if_empty=False)
+    )
+
+
+@validate_arguments(config=dict(arbitrary_types_allowed=True))
+def _pull(paths: List[str]):
+    """Pull the specified paths from external storage.
+
+    Args:
+        paths(List[str]): Paths to pull from LFS.
+    """
+    pull_paths_from_storage(project_context.repository, *paths)
+
+
+def pull_command():
+    """Command to pull the specified paths from external storage."""
+    return Command().command(_pull)
+
+
+@validate_arguments(config=dict(arbitrary_types_allowed=True))
+def _clean(paths: List[str]):
+    """Remove files from lfs cache/turn them back into pointer files.
+
+    Args:
+        paths:List[str]: Paths to turn back to pointer files.
+    """
+    untracked_paths, local_only_paths = clean_storage_cache(*paths)
+
+    if untracked_paths:
+        communication.warn(
+            "These paths were ignored as they are not tracked"
+            + " in git LFS:\n\t{}\n".format("\n\t".join(untracked_paths))
+        )
+
+    if local_only_paths:
+        communication.warn(
+            "These paths were ignored as they are not pushed to "
+            + "a remote with git LFS:\n\t{}\n".format("\n\t".join(local_only_paths))
+        )
+
+
+def clean_command():
+    """Command to remove files from lfs cache/turn them back into pointer files."""
+    return Command().command(_clean)
+
+
+@validate_arguments(config=dict(arbitrary_types_allowed=True))
+def _check_lfs_hook(paths: List[str]):
+    """Check if paths should be in LFS.
+
+    Args:
+        paths(List[str]): Paths to check
+
+    Returns:
+        List of files that should be in LFS.
+    """
+    return check_requires_tracking(*paths)
+
+
+def check_lfs_hook_command():
+    """Command to pull the specified paths from external storage."""
+    return Command().command(_check_lfs_hook)

renku/command/move.py

@@ -28,7 +28,7 @@
 from renku.core.dataset.dataset import move_files
 from renku.core.dataset.datasets_provenance import DatasetsProvenance
 from renku.core.interface.dataset_gateway import IDatasetGateway
-from renku.core.storage import track_paths_in_storage, untrack_paths_from_storage
+from renku.core.lfs import track_paths_in_storage, untrack_paths_from_storage
 from renku.core.util import communication
 from renku.core.util.metadata import is_protected_path
 from renku.core.util.os import get_relative_path, is_subpath

renku/command/remove.py

@@ -28,7 +28,7 @@
 from renku.core import errors
 from renku.core.dataset.datasets_provenance import DatasetsProvenance
 from renku.core.interface.dataset_gateway import IDatasetGateway
-from renku.core.storage import check_external_storage, untrack_paths_from_storage
+from renku.core.lfs import check_external_storage, untrack_paths_from_storage
 from renku.core.util import communication
 from renku.core.util.git import get_git_user
 from renku.core.util.os import delete_dataset_file, expand_directories

renku/command/save.py

@@ -21,7 +21,7 @@
 
 from renku.command.command_builder.command import Command
 from renku.core import errors
-from renku.core.storage import track_paths_in_storage
+from renku.core.lfs import track_paths_in_storage
 from renku.domain_model.project_context import project_context