Skip to content

Commit

Permalink
fix use docker
Browse files Browse the repository at this point in the history
  • Loading branch information
goldenxinxing committed Nov 23, 2022
1 parent 1c72eaa commit ef829c5
Show file tree
Hide file tree
Showing 4 changed files with 27 additions and 20 deletions.
18 changes: 11 additions & 7 deletions client/scripts/sw-docker-entrypoint
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ ulimit -n 65535 || true
CONDA_BIN="/opt/miniconda3/bin"
PIP_CACHE_DIR=${SW_PIP_CACHE_DIR:=/root/.cache/pip}
VERBOSE="-vvvv"
MODE=${RUN_MODE:-"remote"}
STEP=${SW_TASK_STEP:-""}
TASK_INDEX=${SW_TASK_INDEX:-0}
TASK_NUM=${SW_TASK_NUM:-0}
Expand Down Expand Up @@ -57,8 +58,7 @@ set_pip_cache() {
}

set_python() {
# yq only installed for py3.8
_MANIFEST_RUNTIME=$(swcli -o json runtime info sw-runtime/version/v0 | python3.8 -c 'import sys,json; print(json.load(sys.stdin)["config"]["environment"]["python"])') || exit 1
_MANIFEST_RUNTIME=$(swcli -o json runtime info ${SW_RUNTIME_VERSION} | jq -r '.config["environment"]["python"]') || exit 1
_RUNTIME="python${_MANIFEST_RUNTIME}"

echo "**** DETECT RUNTIME: ${_RUNTIME}"
Expand All @@ -84,16 +84,16 @@ restore_activate_runtime() {
pull_model_and_runtime() {
echo '--> pulling model and runtime ...'
swcli instance login --token "${SW_TOKEN}" --alias server ${SW_INSTANCE_URI}
swcli model copy cloud://server/project/${SW_PROJECT}/model/${SW_MODEL_VERSION} local/project/self/sw-model
swcli runtime copy cloud://server/project/${SW_PROJECT}/runtime/${SW_RUNTIME_VERSION} local/project/self/sw-runtime
swcli model copy cloud://server/project/${SW_PROJECT}/model/${SW_MODEL_VERSION} .
swcli runtime copy cloud://server/project/${SW_PROJECT}/runtime/${SW_RUNTIME_VERSION} .
}

run() {
echo "--> start to run evaluation: ${STEP}, use $(which swcli) cli ..."
swcli ${VERBOSE} eval run \
--step=${STEP} --task-index=${TASK_INDEX} --override-task-num=${TASK_NUM} \
--model sw-model/version/v0 \
--runtime sw-runtime/version/v0 \
--model ${SW_MODEL_VERSION} \
--runtime ${SW_RUNTIME_VERSION} \
--version=${SW_EVALUATION_VERSION} || exit 1
}

Expand All @@ -111,7 +111,11 @@ welcome() {
eval_task_prepare(){
welcome $1
pre_config
pull_model_and_runtime
# only remote
if [ "${MODE}" == "remote" ]
then
pull_model_and_runtime
fi
set_python
set_pip_cache
}
Expand Down
2 changes: 2 additions & 0 deletions client/starwhale/consts/env.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ class SWEnv:
project = "SW_PROJECT"
task_id = "SW_TASK_ID"
eval_version = "SW_EVALUATION_VERSION"
model_version = "SW_MODEL_VERSION"
runtime_version = "SW_RUNTIME_VERSION"
status_dir = "SW_TASK_STATUS_DIR"
log_dir = "SW_TASK_LOG_DIR"
dataset_uri = "SW_DATASET_URI"
Expand Down
10 changes: 4 additions & 6 deletions client/starwhale/core/eval/executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,8 @@ def _gen_run_container_cmd(self, typ: str, step: str, task_index: int) -> str:
f"{self._version}-{step}-{task_index}",
"-e",
"DEBUG=1",
"-e",
"RUN_MODE=local",
"-l",
f"version={self._version}",
]
Expand All @@ -214,12 +216,6 @@ def _gen_run_container_cmd(self, typ: str, step: str, task_index: int) -> str:
f"{self.sw_config.rootdir}:/root/.starwhale",
"-v",
f"{self.sw_config.object_store_dir}:{self.sw_config.object_store_dir}",
"-v",
f"{self._model_dir}:{_CNTR_WORKDIR}/{RunSubDirType.SWMP}/src",
"-v",
f"{self._model_dir}/{DefaultYAMLName.MODEL}:{_CNTR_WORKDIR}/{RunSubDirType.SWMP}/{DefaultYAMLName.MODEL}",
"-v",
f"{self._runtime_dir}:{_CNTR_WORKDIR}/{RunSubDirType.SWRT}",
]

if typ == EvalTaskType.SINGLE:
Expand All @@ -230,6 +226,8 @@ def _gen_run_container_cmd(self, typ: str, step: str, task_index: int) -> str:

cmd.extend(["-e", f"{SWEnv.project}={self.project_uri.project}"])
cmd.extend(["-e", f"{SWEnv.eval_version}={self._version}"])
cmd.extend(["-e", f"{SWEnv.model_version}={self.model_uri}"])
cmd.extend(["-e", f"{SWEnv.runtime_version}={self.runtime_uri}"])
cmd.extend(
[
"-e",
Expand Down
17 changes: 10 additions & 7 deletions client/tests/core/test_executor.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,12 +89,16 @@ def test_run(self, m_scheduler: MagicMock, m_call: MagicMock) -> None:
ensure_file(runtime_bundle_path, " ")
ensure_dir(runtime_workdir_path)
ensure_file(runtime_workdir_path / DEFAULT_MANIFEST_NAME, "{}")

model_version = "mnist/version/gnstmntggi4t"
runtime_version = "mnist/version/ga4doztfg4yw"
dataset_version = "mnist/version/me4dczleg"
# use docker
ee = EvalExecutor(
model_uri="mnist/version/gnstmntggi4t",
dataset_uris=["mnist/version/me4dczleg"],
model_uri=model_version,
dataset_uris=[dataset_version],
project_uri=URI(""),
runtime_uri="mnist/version/ga4doztfg4yw",
runtime_uri=runtime_version,
use_docker=True,
)

Expand All @@ -116,15 +120,14 @@ def test_run(self, m_scheduler: MagicMock, m_call: MagicMock) -> None:

assert ppl_cmd == " ".join(
[
f"docker run --net=host --rm --name {build_version}--0 -e DEBUG=1 -l version={build_version}",
f"docker run --net=host --rm --name {build_version}--0 -e DEBUG=1 -e RUN_MODE=local -l version={build_version}",
f"-v {job_dir}:/opt/starwhale",
f"-v {sw.rootdir}:/root/.starwhale",
f"-v {sw.object_store_dir}:{sw.object_store_dir}",
f"-v {project_dir}/workdir/model/mnist/gn/gnstmntggi4t111111111111/src:/opt/starwhale/swmp/src",
f"-v {project_dir}/workdir/model/mnist/gn/gnstmntggi4t111111111111/src/model.yaml:/opt/starwhale/swmp/model.yaml",
f"-v {project_dir}/workdir/runtime/mnist/ga/ga4doztfg4yw11111111111111:/opt/starwhale/swrt",
"-e SW_PROJECT=self",
f"-e SW_EVALUATION_VERSION={build_version}",
f"-e SW_MODEL_VERSION={model_version}",
f"-e SW_RUNTIME_VERSION={runtime_version}",
"-e SW_INSTANCE_URI=local",
"-e SW_TOKEN=",
"-e SW_DATASET_URI=local/project/self/dataset/mnist/version/me4dczleg",
Expand Down

0 comments on commit ef829c5

Please sign in to comment.