From 8f662cb27453dd8d7cae23796163a5eeaefcf41d Mon Sep 17 00:00:00 2001 From: Jialei Date: Tue, 20 Dec 2022 18:59:49 +0800 Subject: [PATCH] chore(client): runtime building support no-cache option --- client/starwhale/core/runtime/cli.py | 9 +++++++++ client/starwhale/core/runtime/model.py | 23 +++++++++++++++++---- client/starwhale/core/runtime/view.py | 2 ++ client/tests/core/test_runtime.py | 28 ++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 4 deletions(-) diff --git a/client/starwhale/core/runtime/cli.py b/client/starwhale/core/runtime/cli.py index b45b09e5e4..778e565152 100644 --- a/client/starwhale/core/runtime/cli.py +++ b/client/starwhale/core/runtime/cli.py @@ -156,6 +156,13 @@ def _quickstart( is_flag=True, help="Disable virtualenv/conda environment dependencies lock, and the cli supports three methods to lock environment that are shell(auto-detect), prefix_path or env_name", ) +@click.option( + "-nc", + "--no-cache", + is_flag=True, + help="Invalid the cached(installed) packages in the isolate env when env-lock is enabled, \ + only for auto-generated environments", +) @optgroup.group( # type: ignore "Python environment selectors", cls=MutuallyExclusiveOptionGroup, @@ -180,6 +187,7 @@ def _build( gen_all_bundles: bool, include_editable: bool, disable_env_lock: bool, + no_cache: bool, env_prefix_path: str, env_name: str, env_use_shell: bool, @@ -191,6 +199,7 @@ def _build( gen_all_bundles=gen_all_bundles, include_editable=include_editable, disable_env_lock=disable_env_lock, + no_cache=no_cache, env_prefix_path=env_prefix_path, env_name=env_name, env_use_shell=env_use_shell, diff --git a/client/starwhale/core/runtime/model.py b/client/starwhale/core/runtime/model.py index dde48e26be..30bb9fa64d 100644 --- a/client/starwhale/core/runtime/model.py +++ b/client/starwhale/core/runtime/model.py @@ -661,6 +661,7 @@ def lock( env_name: str = "", env_prefix_path: str = "", disable_auto_inject: bool = False, + no_cache: bool = False, stdout: bool = False, include_editable: bool = False, emit_pip_options: bool = False, @@ -672,6 +673,7 @@ def lock( env_name, env_prefix_path, disable_auto_inject, + no_cache, stdout, include_editable, emit_pip_options, @@ -757,6 +759,7 @@ def build( # type: ignore[override] ) -> None: yaml_name = kw.get("yaml_name", DefaultYAMLName.RUNTIME) disable_env_lock = kw.get("disable_env_lock", False) + no_cache = kw.get("no_cache", False) env_name = kw.get("env_name", "") env_prefix_path = kw.get("env_prefix_path", "") env_use_shell = kw.get("env_use_shell", False) @@ -772,6 +775,7 @@ def build( # type: ignore[override] env_name=env_name, env_prefix_path=env_prefix_path, disable_auto_inject=False, + no_cache=no_cache, stdout=False, include_editable=include_editable, env_use_shell=env_use_shell, @@ -1218,8 +1222,16 @@ def activate(cls, path: str = "", uri: str = "") -> None: @classmethod def _ensure_isolated_python_env( - cls, env_dir: Path, python_version: str, mode: str, invalid_rebuild: bool = True + cls, + env_dir: Path, + python_version: str, + mode: str, + no_cache: bool, + recreate_env_if_broken: bool = True, ) -> None: + if no_cache: + empty_dir(env_dir) + if env_dir.exists(): is_valid_conda = mode == PythonRunEnv.CONDA and check_valid_conda_prefix( env_dir @@ -1228,11 +1240,10 @@ def _ensure_isolated_python_env( env_dir ) - # TODO: add rebuild option if is_valid_conda or is_valid_venv: return else: - if invalid_rebuild: + if recreate_env_if_broken: empty_dir(env_dir) ensure_dir(env_dir) else: @@ -1250,6 +1261,7 @@ def lock( env_name: str = "", env_prefix_path: str = "", disable_auto_inject: bool = False, + no_cache: bool = False, stdout: bool = False, include_editable: bool = False, emit_pip_options: bool = False, @@ -1262,6 +1274,7 @@ def lock( :param env_name: conda environment name (used by conda env) :param env_prefix_path: python env prefix path (used by both venv and conda) :param disable_auto_inject: disable putting lock file info into configured runtime yaml file + :param no_cache: invalid all pkgs installed :param stdout: just print the lock info into stdout without saving lock file :param include_editable: include the editable pkg (only for venv) :param emit_pip_options: use user's pip configuration when freeze pkgs (only for venv) @@ -1296,7 +1309,9 @@ def lock( prefix_path = env_prefix_path else: _sw_auto_path = target_dir / SW_AUTO_DIRNAME / mode - cls._ensure_isolated_python_env(_sw_auto_path, expected_pyver, mode) + cls._ensure_isolated_python_env( + _sw_auto_path, expected_pyver, mode, no_cache + ) prefix_path = str(_sw_auto_path) cls._install_dependencies_with_runtime_yaml( diff --git a/client/starwhale/core/runtime/view.py b/client/starwhale/core/runtime/view.py index c13b395569..dfebeaa879 100644 --- a/client/starwhale/core/runtime/view.py +++ b/client/starwhale/core/runtime/view.py @@ -110,6 +110,7 @@ def build( gen_all_bundles: bool = False, include_editable: bool = False, disable_env_lock: bool = False, + no_cache: bool = False, env_prefix_path: str = "", env_name: str = "", env_use_shell: bool = False, @@ -184,6 +185,7 @@ def build( gen_all_bundles=gen_all_bundles, include_editable=include_editable, disable_env_lock=disable_env_lock, + no_cache=no_cache, env_prefix_path=env_prefix_path, env_name=env_name, env_use_shell=env_use_shell, diff --git a/client/tests/core/test_runtime.py b/client/tests/core/test_runtime.py index 23bc947331..d7f1d00d91 100644 --- a/client/tests/core/test_runtime.py +++ b/client/tests/core/test_runtime.py @@ -643,6 +643,34 @@ def test_build_conda( {"deps": "conda-sw-lock.yaml", "kind": "conda_env_file"}, ] + @patch("starwhale.utils.venv.check_call") + @patch("starwhale.utils.venv.subprocess.check_output") + @patch("starwhale.core.runtime.model.check_valid_conda_prefix") + def test_build_with_no_cache(self, m_check: MagicMock, *args: t.Any): + target_dir = "/home/starwhale/workdir" + ensure_dir(target_dir) + ensure_file( + f"{target_dir}/{DefaultYAMLName.RUNTIME}", + yaml.safe_dump( + { + "name": "test", + "mode": "conda", + } + ), + ) + # make sure the dir is not deleted by "recreate_env_if_broken" + m_check.return_value = True + env_dir = f"{target_dir}/{SW_AUTO_DIRNAME}/conda" + ensure_dir(env_dir) + my_garbage = f"{env_dir}/garbage" + ensure_dir(my_garbage) + + StandaloneRuntime.lock(target_dir) + assert Path(my_garbage).exists() + + StandaloneRuntime.lock(target_dir, no_cache=True) + assert not Path(my_garbage).exists() + @patch("os.environ", {}) @patch("starwhale.core.runtime.model.get_python_version") @patch("starwhale.utils.venv.get_user_runtime_python_bin")