diff --git a/client/starwhale/api/_impl/data_store.py b/client/starwhale/api/_impl/data_store.py index 0295b13182..0bdcbfcba9 100644 --- a/client/starwhale/api/_impl/data_store.py +++ b/client/starwhale/api/_impl/data_store.py @@ -900,9 +900,9 @@ def dump(self) -> None: class RemoteDataStore: - def __init__(self, instance_uri: str) -> None: + def __init__(self, instance_uri: str, token: str = "") -> None: self.instance_uri = instance_uri - self.token = os.getenv(SWEnv.instance_token) + self.token = token or os.getenv(SWEnv.instance_token) if self.token is None: raise RuntimeError("SW_TOKEN is not found in environment") @@ -1033,12 +1033,16 @@ def scan_tables( ... -def get_data_store() -> DataStore: - instance_uri = os.getenv(SWEnv.instance_uri) - if instance_uri is None or instance_uri == "local": +def get_data_store(instance_uri: str = "") -> DataStore: + _instance_uri = instance_uri or os.getenv(SWEnv.instance_uri) + if _instance_uri is None or _instance_uri == "local": return LocalDataStore.get_instance() else: - return RemoteDataStore(instance_uri) + print(f"instance:{instance_uri}") + return RemoteDataStore( + instance_uri=_instance_uri, + token=SWCliConfigMixed().get_sw_token(instance=instance_uri), + ) def _flatten(record: Dict[str, Any]) -> Dict[str, Any]: diff --git a/client/starwhale/api/_impl/wrapper.py b/client/starwhale/api/_impl/wrapper.py index 1463cf4428..4cdd749591 100644 --- a/client/starwhale/api/_impl/wrapper.py +++ b/client/starwhale/api/_impl/wrapper.py @@ -47,7 +47,7 @@ def _log(self, table_name: str, record: Dict[str, Any]) -> None: class Evaluation(Logger): - def __init__(self, eval_id: str = "", project: str = ""): + def __init__(self, eval_id: str = "", project: str = "", instance: str = ""): eval_id = eval_id or os.getenv(SWEnv.eval_version, "") if not eval_id: raise RuntimeError("eval id should not be None") @@ -64,7 +64,7 @@ def __init__(self, eval_id: str = "", project: str = ""): self._results_table_name = self._get_datastore_table_name("results") self._summary_table_name = f"project/{self.project}/eval/summary" self._init_writers([self._results_table_name, self._summary_table_name]) - self._data_store = data_store.get_data_store() + self._data_store = data_store.get_data_store(instance_uri=instance) def _get_datastore_table_name(self, table_name: str) -> str: return f"project/{self.project}/eval/{self.eval_id[:VERSION_PREFIX_CNT]}/{self.eval_id}/{table_name}" diff --git a/client/starwhale/core/eval/model.py b/client/starwhale/core/eval/model.py index 040dc4cdf0..b5ff7ac929 100644 --- a/client/starwhale/core/eval/model.py +++ b/client/starwhale/core/eval/model.py @@ -93,7 +93,9 @@ def _get_version(self) -> str: def _get_report(self) -> t.Dict[str, t.Any]: evaluation = wrapper.Evaluation( - eval_id=self._get_version(), project=self.uri.project + eval_id=self._get_version(), + project=self.uri.project, + instance=self.uri.instance, ) summary = evaluation.get_metrics() kind = summary.get("kind", "") diff --git a/client/starwhale/core/eval/view.py b/client/starwhale/core/eval/view.py index ef5c72387a..fbb966f762 100644 --- a/client/starwhale/core/eval/view.py +++ b/client/starwhale/core/eval/view.py @@ -153,7 +153,6 @@ def _print_tasks(self, tasks: t.List[t.Dict[str, t.Any]]) -> None: table.add_column("ID", justify="left", style="cyan", no_wrap=True) table.add_column("UUID") table.add_column("Status", style="magenta") - table.add_column("Agent") table.add_column("Duration") table.add_column("Created") table.add_column("Finished") @@ -164,7 +163,6 @@ def _print_tasks(self, tasks: t.List[t.Dict[str, t.Any]]) -> None: _t["id"], _t["uuid"], f"[{style}]{icon}{status}[/]", - _t["agent"]["ip"], "", _t["created_at"], "", @@ -243,7 +241,7 @@ def run( task_index: int = 0, ) -> None: _project_uri = URI(project_uri, expected_type=URIType.PROJECT) - ok, reason = EvaluationJob.run( + ok, version = EvaluationJob.run( _project_uri, model_uri, dataset_uris, @@ -265,16 +263,16 @@ def run( f":clap: success to create job(project id: [red]{_project_uri.full_uri}[/])" ) if _project_uri.instance_type == InstanceType.CLOUD: - _job_uri = f"{_project_uri.full_uri}/job/{reason}" + _job_uri = f"{_project_uri.full_uri}/evaluation/{version}" else: - _job_uri = f"{reason[:SHORT_VERSION_CNT]}" + _job_uri = f"{version[:SHORT_VERSION_CNT]}" console.print( f":bird: run cmd to fetch eval info: [bold green]swcli eval info {_job_uri}[/]" ) else: console.print( - f":collision: failed to create eval job, notice: [red]{reason}[/]" + f":collision: failed to create eval job, notice: [red]{version}[/]" ) @classmethod diff --git a/client/tests/core/test_eval.py b/client/tests/core/test_eval.py index 0fee1c268b..30110db43c 100644 --- a/client/tests/core/test_eval.py +++ b/client/tests/core/test_eval.py @@ -1,5 +1,4 @@ import os -import unittest from pathlib import Path from unittest.mock import patch, MagicMock @@ -10,16 +9,19 @@ from starwhale.consts import HTTPMethod, RECOVER_DIRNAME, DEFAULT_MANIFEST_NAME from starwhale.base.uri import URI from starwhale.base.type import URIType -from starwhale.utils.config import load_swcli_config +from starwhale.utils.config import load_swcli_config, get_swcli_config_path from starwhale.core.eval.view import JobTermView, JobTermViewRich from starwhale.core.eval.model import CloudEvaluationJob, StandaloneEvaluationJob from starwhale.core.eval.store import EvaluationStorage -from .. import ROOT_DIR +from .. import ROOT_DIR, get_predefined_config_yaml _job_data_dir = f"{ROOT_DIR}/data/job" _job_manifest = open(f"{_job_data_dir}/job_manifest.yaml").read() _cmp_report = open(f"{_job_data_dir}/cmp_report.jsonl").read() +_job_list = open(f"{_job_data_dir}/job_list_resp.json").read() +_task_list = open(f"{_job_data_dir}/task_list.json").read() +_existed_config_contents = get_predefined_config_yaml() class StandaloneEvaluationJobTestCase(TestCase): @@ -137,9 +139,14 @@ def test_stanalone_actions(self, m_call: MagicMock, m_call_output: MagicMock): assert m_call.call_count == 3 -class CloudJobTestCase(unittest.TestCase): +class CloudJobTestCase(TestCase): def setUp(self): - self.instance_uri = "http://1.1.1.1:8888" + self.setUpPyfakefs() + sw_config._config = {} + path = get_swcli_config_path() + self.fs.create_file(path, contents=_existed_config_contents) + + self.instance_uri = "http://1.1.1.1:8182" self.project_uri = f"{self.instance_uri}/project/self" self.job_name = "15" self.job_uri = f"{self.project_uri}/{URIType.EVALUATION}/{self.job_name}" @@ -172,7 +179,7 @@ def test_cloud_create(self, rm: Mocker, m_console: MagicMock): resource="gpu:1", ) assert m_console.call_count == 2 - assert "project/self/job/11" in m_console.call_args[0][0] + assert "project/self/evaluation/11" in m_console.call_args[0][0] @Mocker() @patch("starwhale.core.eval.view.console.print") @@ -180,7 +187,7 @@ def test_cloud_list(self, rm: Mocker, m_console: MagicMock): rm.request( HTTPMethod.GET, f"{self.instance_uri}/api/v1/project/self/job", - text=open(f"{_job_data_dir}/job_list_resp.json").read(), + text=_job_list, ) jobs, pager = CloudEvaluationJob.list( @@ -223,15 +230,16 @@ def test_cloud_info( rm.request( HTTPMethod.GET, f"{self.instance_uri}/api/v1/project/self/job/{self.job_name}", - text=open(f"{_job_data_dir}/job_manifest.yaml").read(), + text=_job_manifest, ) rm.request( HTTPMethod.GET, f"{self.instance_uri}/api/v1/project/self/job/{self.job_name}/task", - text=open(f"{_job_data_dir}/task_list.json").read(), + text=_task_list, ) info = CloudEvaluationJob(URI(self.job_uri)).info() + print(f"info oo :{info}") assert len(info["tasks"][0]) == 3 assert info["tasks"][0][0]["taskStatus"] == "SUCCESS" assert info["tasks"][0][0]["id"] == "40"