From ef829c5b27a4f3960781fb2310a6ec3325b2e0d8 Mon Sep 17 00:00:00 2001 From: gaoxinxing <15031259256@163.com> Date: Wed, 23 Nov 2022 16:24:39 +0800 Subject: [PATCH] fix use docker --- client/scripts/sw-docker-entrypoint | 18 +++++++++++------- client/starwhale/consts/env.py | 2 ++ client/starwhale/core/eval/executor.py | 10 ++++------ client/tests/core/test_executor.py | 17 ++++++++++------- 4 files changed, 27 insertions(+), 20 deletions(-) diff --git a/client/scripts/sw-docker-entrypoint b/client/scripts/sw-docker-entrypoint index 9484f6e0fe..0f0bc061aa 100755 --- a/client/scripts/sw-docker-entrypoint +++ b/client/scripts/sw-docker-entrypoint @@ -11,6 +11,7 @@ ulimit -n 65535 || true CONDA_BIN="/opt/miniconda3/bin" PIP_CACHE_DIR=${SW_PIP_CACHE_DIR:=/root/.cache/pip} VERBOSE="-vvvv" +MODE=${RUN_MODE:-"remote"} STEP=${SW_TASK_STEP:-""} TASK_INDEX=${SW_TASK_INDEX:-0} TASK_NUM=${SW_TASK_NUM:-0} @@ -57,8 +58,7 @@ set_pip_cache() { } set_python() { - # yq only installed for py3.8 - _MANIFEST_RUNTIME=$(swcli -o json runtime info sw-runtime/version/v0 | python3.8 -c 'import sys,json; print(json.load(sys.stdin)["config"]["environment"]["python"])') || exit 1 + _MANIFEST_RUNTIME=$(swcli -o json runtime info ${SW_RUNTIME_VERSION} | jq -r '.config["environment"]["python"]') || exit 1 _RUNTIME="python${_MANIFEST_RUNTIME}" echo "**** DETECT RUNTIME: ${_RUNTIME}" @@ -84,16 +84,16 @@ restore_activate_runtime() { pull_model_and_runtime() { echo '--> pulling model and runtime ...' swcli instance login --token "${SW_TOKEN}" --alias server ${SW_INSTANCE_URI} - swcli model copy cloud://server/project/${SW_PROJECT}/model/${SW_MODEL_VERSION} local/project/self/sw-model - swcli runtime copy cloud://server/project/${SW_PROJECT}/runtime/${SW_RUNTIME_VERSION} local/project/self/sw-runtime + swcli model copy cloud://server/project/${SW_PROJECT}/model/${SW_MODEL_VERSION} . + swcli runtime copy cloud://server/project/${SW_PROJECT}/runtime/${SW_RUNTIME_VERSION} . } run() { echo "--> start to run evaluation: ${STEP}, use $(which swcli) cli ..." swcli ${VERBOSE} eval run \ --step=${STEP} --task-index=${TASK_INDEX} --override-task-num=${TASK_NUM} \ - --model sw-model/version/v0 \ - --runtime sw-runtime/version/v0 \ + --model ${SW_MODEL_VERSION} \ + --runtime ${SW_RUNTIME_VERSION} \ --version=${SW_EVALUATION_VERSION} || exit 1 } @@ -111,7 +111,11 @@ welcome() { eval_task_prepare(){ welcome $1 pre_config - pull_model_and_runtime + # only remote + if [ "${MODE}" == "remote" ] + then + pull_model_and_runtime + fi set_python set_pip_cache } diff --git a/client/starwhale/consts/env.py b/client/starwhale/consts/env.py index a8d81ba8aa..74e43a6daf 100644 --- a/client/starwhale/consts/env.py +++ b/client/starwhale/consts/env.py @@ -3,6 +3,8 @@ class SWEnv: project = "SW_PROJECT" task_id = "SW_TASK_ID" eval_version = "SW_EVALUATION_VERSION" + model_version = "SW_MODEL_VERSION" + runtime_version = "SW_RUNTIME_VERSION" status_dir = "SW_TASK_STATUS_DIR" log_dir = "SW_TASK_LOG_DIR" dataset_uri = "SW_DATASET_URI" diff --git a/client/starwhale/core/eval/executor.py b/client/starwhale/core/eval/executor.py index 2d1a4fd712..309b283e9b 100644 --- a/client/starwhale/core/eval/executor.py +++ b/client/starwhale/core/eval/executor.py @@ -203,6 +203,8 @@ def _gen_run_container_cmd(self, typ: str, step: str, task_index: int) -> str: f"{self._version}-{step}-{task_index}", "-e", "DEBUG=1", + "-e", + "RUN_MODE=local", "-l", f"version={self._version}", ] @@ -214,12 +216,6 @@ def _gen_run_container_cmd(self, typ: str, step: str, task_index: int) -> str: f"{self.sw_config.rootdir}:/root/.starwhale", "-v", f"{self.sw_config.object_store_dir}:{self.sw_config.object_store_dir}", - "-v", - f"{self._model_dir}:{_CNTR_WORKDIR}/{RunSubDirType.SWMP}/src", - "-v", - f"{self._model_dir}/{DefaultYAMLName.MODEL}:{_CNTR_WORKDIR}/{RunSubDirType.SWMP}/{DefaultYAMLName.MODEL}", - "-v", - f"{self._runtime_dir}:{_CNTR_WORKDIR}/{RunSubDirType.SWRT}", ] if typ == EvalTaskType.SINGLE: @@ -230,6 +226,8 @@ def _gen_run_container_cmd(self, typ: str, step: str, task_index: int) -> str: cmd.extend(["-e", f"{SWEnv.project}={self.project_uri.project}"]) cmd.extend(["-e", f"{SWEnv.eval_version}={self._version}"]) + cmd.extend(["-e", f"{SWEnv.model_version}={self.model_uri}"]) + cmd.extend(["-e", f"{SWEnv.runtime_version}={self.runtime_uri}"]) cmd.extend( [ "-e", diff --git a/client/tests/core/test_executor.py b/client/tests/core/test_executor.py index a0e159d276..dc13830b7c 100644 --- a/client/tests/core/test_executor.py +++ b/client/tests/core/test_executor.py @@ -89,12 +89,16 @@ def test_run(self, m_scheduler: MagicMock, m_call: MagicMock) -> None: ensure_file(runtime_bundle_path, " ") ensure_dir(runtime_workdir_path) ensure_file(runtime_workdir_path / DEFAULT_MANIFEST_NAME, "{}") + + model_version = "mnist/version/gnstmntggi4t" + runtime_version = "mnist/version/ga4doztfg4yw" + dataset_version = "mnist/version/me4dczleg" # use docker ee = EvalExecutor( - model_uri="mnist/version/gnstmntggi4t", - dataset_uris=["mnist/version/me4dczleg"], + model_uri=model_version, + dataset_uris=[dataset_version], project_uri=URI(""), - runtime_uri="mnist/version/ga4doztfg4yw", + runtime_uri=runtime_version, use_docker=True, ) @@ -116,15 +120,14 @@ def test_run(self, m_scheduler: MagicMock, m_call: MagicMock) -> None: assert ppl_cmd == " ".join( [ - f"docker run --net=host --rm --name {build_version}--0 -e DEBUG=1 -l version={build_version}", + f"docker run --net=host --rm --name {build_version}--0 -e DEBUG=1 -e RUN_MODE=local -l version={build_version}", f"-v {job_dir}:/opt/starwhale", f"-v {sw.rootdir}:/root/.starwhale", f"-v {sw.object_store_dir}:{sw.object_store_dir}", - f"-v {project_dir}/workdir/model/mnist/gn/gnstmntggi4t111111111111/src:/opt/starwhale/swmp/src", - f"-v {project_dir}/workdir/model/mnist/gn/gnstmntggi4t111111111111/src/model.yaml:/opt/starwhale/swmp/model.yaml", - f"-v {project_dir}/workdir/runtime/mnist/ga/ga4doztfg4yw11111111111111:/opt/starwhale/swrt", "-e SW_PROJECT=self", f"-e SW_EVALUATION_VERSION={build_version}", + f"-e SW_MODEL_VERSION={model_version}", + f"-e SW_RUNTIME_VERSION={runtime_version}", "-e SW_INSTANCE_URI=local", "-e SW_TOKEN=", "-e SW_DATASET_URI=local/project/self/dataset/mnist/version/me4dczleg",