diff --git a/CHANGELOG.md b/CHANGELOG.md index 71be776d7..262235ef9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Features - Heuristics to help users find out they're writing legacy code with new client API or vice versa ([#607](https://github.com/neptune-ai/neptune-client/pull/607)) +- Lookup for projects without workspace specification and listing user projects and workspaces ([#615](https://github.com/neptune-ai/neptune-client/pull/615)) ## neptune-client 0.9.19 diff --git a/neptune/new/exceptions.py b/neptune/new/exceptions.py index e0a55f874..7577e24c3 100644 --- a/neptune/new/exceptions.py +++ b/neptune/new/exceptions.py @@ -22,6 +22,7 @@ from neptune.new import envs from neptune.new.envs import CUSTOM_RUN_ID_ENV_NAME from neptune.new.internal.utils import replace_patch_version +from neptune.new.internal.backends.api_model import Project, Workspace from neptune.exceptions import STYLES @@ -129,63 +130,98 @@ def __init__(self, status, response): super().__init__(message.format(**inputs)) -class ProjectNotFound(NeptuneException): - def __init__(self, project_id): +class ExceptionWithProjectsWorkspacesListing(NeptuneException): + def __init__(self, + message: str, + available_projects: List[Project] = (), + available_workspaces: List[Workspace] = (), + **kwargs): + available_projects_message = """ +Did you mean any of these? +{projects} +""" + + available_workspaces_message = """ +You can check all of your projects on the Projects page: +{workspaces_urls} +""" + + projects_formated_list = '\n'.join( + map(lambda project: f' - {project.workspace}/{project.name}', available_projects) + ) + + workspaces_formated_list = '\n'.join( + map(lambda workspace: f' - https://app.neptune.ai/{workspace.name}/-/projects', available_workspaces) + ) + + self.inputs = { + 'available_projects_message': available_projects_message.format( + projects=projects_formated_list + ) if available_projects else '', + 'available_workspaces_message': available_workspaces_message.format( + workspaces_urls=workspaces_formated_list + ) if available_workspaces else '', + **STYLES, + **kwargs + } + + super().__init__(message.format(**self.inputs)) + + +class ProjectNotFound(ExceptionWithProjectsWorkspacesListing): + def __init__(self, + project_id: str, + available_projects: List[Project] = (), + available_workspaces: List[Workspace] = ()): message = """ {h1} -----ProjectNotFound------------------------------------------------------------------------- +----NeptuneProjectNotFoundException------------------------------------ {end} -Project {python}{project}{end} not found. +We couldn’t find project {fail}"{project}"{end}. +{available_projects_message}{available_workspaces_message} +You may also want to check the following docs pages: + - https://docs.neptune.ai/administration/workspace-project-and-user-management/projects + - https://docs.neptune.ai/getting-started/hello-world#project -Verify if your project's name was not misspelled. -You can find proper name after logging into Neptune UI. +{correct}Need help?{end}-> https://docs.neptune.ai/getting-started/getting-help """ - inputs = dict(list({'project': project_id}.items()) + list(STYLES.items())) - super().__init__(message.format(**inputs)) - - -class RunNotFound(NeptuneException): - - def __init__(self, run_id: str) -> None: - super().__init__("Run {} not found.".format(run_id)) - - -class RunUUIDNotFound(NeptuneException): - def __init__(self, run_uuid: uuid.UUID): - super().__init__("Run with UUID {} not found. Could be deleted.".format(run_uuid)) + super().__init__(message=message, + available_projects=available_projects, + available_workspaces=available_workspaces, + project=project_id) -class InactiveRunException(NeptuneException): - def __init__(self, short_id: str): +class ProjectNameCollision(ExceptionWithProjectsWorkspacesListing): + def __init__(self, + project_id: str, + available_projects: List[Project] = ()): message = """ {h1} -----InactiveRunException---------------------------------------- +----NeptuneProjectNameCollisionException------------------------------------ {end} -It seems you are trying to log (or fetch) metadata to a run that was stopped ({short_id}). -What should I do? - - Resume the run to continue logging to it: - https://docs.neptune.ai/how-to-guides/neptune-api/resume-run#how-to-resume-run - - Don't invoke `stop()` on a run that you want to access. If you want to stop monitoring only, - you can resume a run in read-only mode: - https://docs.neptune.ai/you-should-know/connection-modes#read-only +Cannot resolve project {fail}"{project}"{end}. +{available_projects_message} You may also want to check the following docs pages: - - https://docs.neptune.ai/api-reference/run#stop - - https://docs.neptune.ai/how-to-guides/neptune-api/resume-run#how-to-resume-run - - https://docs.neptune.ai/you-should-know/connection-modes + - https://docs.neptune.ai/administration/workspace-project-and-user-management/projects + - https://docs.neptune.ai/getting-started/hello-world#project + {correct}Need help?{end}-> https://docs.neptune.ai/getting-started/getting-help """ - inputs = dict(list({'short_id': short_id}.items()) + list(STYLES.items())) - super().__init__(message.format(**inputs)) + super().__init__(message=message, + available_projects=available_projects, + project=project_id) -class NeptuneMissingProjectNameException(NeptuneException): - def __init__(self): +class NeptuneMissingProjectNameException(ExceptionWithProjectsWorkspacesListing): + def __init__(self, + available_projects: List[Project] = (), + available_workspaces: List[Workspace] = ()): message = """ {h1} ----NeptuneMissingProjectNameException---------------------------------------- {end} Neptune client couldn't find your project name. - +{available_projects_message}{available_workspaces_message} There are two options two add it: - specify it in your code - set an environment variable in your operating system. @@ -214,36 +250,43 @@ def __init__(self): {correct}Need help?{end}-> https://docs.neptune.ai/getting-started/getting-help """ - inputs = dict(list({'env_project': envs.PROJECT_ENV_NAME}.items()) + list(STYLES.items())) - super().__init__(message.format(**inputs)) + super().__init__(message=message, + available_projects=available_projects, + available_workspaces=available_workspaces, + env_project=envs.PROJECT_ENV_NAME) -class NeptuneIncorrectProjectNameException(NeptuneException): - def __init__(self, project): - message = """ -{h1} -----NeptuneIncorrectProjectNameException------------------------------------ -{end} -Project name {fail}"{project}"{end} you specified seems to be incorrect. +class RunNotFound(NeptuneException): -The correct project name should look like this {correct}WORKSPACE/PROJECT_NAME{end}. -It has two parts: - - {correct}WORKSPACE{end}: which can be your username or your organization name - - {correct}PROJECT_NAME{end}: which is the actual project name you chose + def __init__(self, run_id: str) -> None: + super().__init__("Run {} not found.".format(run_id)) -For example, a project {correct}neptune-ai/credit-default-prediction{end} parts are: - - {correct}neptune-ai{end}: {underline}WORKSPACE{end} our company organization name - - {correct}credit-default-prediction{end}: {underline}PROJECT_NAME{end} a project name -The URL to this project looks like this: https://app.neptune.ai/neptune-ai/credit-default-prediction +class RunUUIDNotFound(NeptuneException): + def __init__(self, run_uuid: uuid.UUID): + super().__init__("Run with UUID {} not found. Could be deleted.".format(run_uuid)) -You may also want to check the following docs pages: - - https://docs.neptune.ai/administration/workspace-project-and-user-management/projects - - https://docs.neptune.ai/getting-started/hello-world#project +class InactiveRunException(NeptuneException): + def __init__(self, short_id: str): + message = """ +{h1} +----InactiveRunException---------------------------------------- +{end} +It seems you are trying to log (or fetch) metadata to a run that was stopped ({short_id}). +What should I do? + - Resume the run to continue logging to it: + https://docs.neptune.ai/how-to-guides/neptune-api/resume-run#how-to-resume-run + - Don't invoke `stop()` on a run that you want to access. If you want to stop monitoring only, + you can resume a run in read-only mode: + https://docs.neptune.ai/you-should-know/connection-modes#read-only +You may also want to check the following docs pages: + - https://docs.neptune.ai/api-reference/run#stop + - https://docs.neptune.ai/how-to-guides/neptune-api/resume-run#how-to-resume-run + - https://docs.neptune.ai/you-should-know/connection-modes {correct}Need help?{end}-> https://docs.neptune.ai/getting-started/getting-help """ - inputs = dict(list({'project': project}.items()) + list(STYLES.items())) + inputs = dict(list({'short_id': short_id}.items()) + list(STYLES.items())) super().__init__(message.format(**inputs)) @@ -673,8 +716,9 @@ def __init__(self, matplotlib_version, plotly_version): "Unable to convert plotly figure to matplotlib format. " "Your matplotlib ({}) and plotlib ({}) versions are not compatible. " "See https://stackoverflow.com/q/63120058 for details. " - "Downgrade matplotlib to version 3.2 or use as_image to log static chart." - .format(matplotlib_version, plotly_version)) + "Downgrade matplotlib to version 3.2 or use as_image to log static chart.".format( + matplotlib_version, + plotly_version)) class NeptunePossibleLegacyUsageException(NeptuneException): diff --git a/neptune/new/internal/backends/api_model.py b/neptune/new/internal/backends/api_model.py index 49b9673ab..360c8c573 100644 --- a/neptune/new/internal/backends/api_model.py +++ b/neptune/new/internal/backends/api_model.py @@ -30,6 +30,13 @@ def __init__(self, _uuid: uuid.UUID, name: str, workspace: str): self.workspace = workspace +class Workspace: + + def __init__(self, _uuid: uuid.UUID, name: str): + self.uuid = _uuid + self.name = name + + @dataclass class ApiRun: uuid: uuid.UUID diff --git a/neptune/new/internal/backends/hosted_neptune_backend.py b/neptune/new/internal/backends/hosted_neptune_backend.py index 6c94fdab3..783394861 100644 --- a/neptune/new/internal/backends/hosted_neptune_backend.py +++ b/neptune/new/internal/backends/hosted_neptune_backend.py @@ -28,6 +28,7 @@ from packaging import version from neptune.new.envs import NEPTUNE_ALLOW_SELF_SIGNED_CERTIFICATE +from neptune.patterns import PROJECT_QUALIFIED_NAME_PATTERN from neptune.new.exceptions import ( ClientHttpError, FetchAttributeNotFoundException, @@ -38,6 +39,7 @@ NeptuneException, NeptuneLegacyProjectException, ProjectNotFound, + ProjectNameCollision, NeptuneStorageLimitException, UnsupportedClientVersion, ) @@ -55,6 +57,7 @@ IntAttribute, LeaderboardEntry, Project, + Workspace, StringAttribute, StringSeriesAttribute, StringSetAttribute, @@ -174,7 +177,29 @@ def websockets_factory(self, project_uuid: uuid.UUID, run_uuid: uuid.UUID) -> Op def get_project(self, project_id: str) -> Project: verify_type("project_id", project_id, str) + project_spec = re.search(PROJECT_QUALIFIED_NAME_PATTERN, project_id) + workspace, name = project_spec['workspace'], project_spec['project'] + try: + if not workspace: + available_projects = list(filter(lambda p: p.name == name, + self.get_available_projects(search_term=name))) + + if len(available_projects) == 1: + project = available_projects[0] + project_id = f'{project.workspace}/{project.name}' + elif len(available_projects) > 1: + raise ProjectNameCollision( + project_id=project_id, + available_projects=available_projects + ) + else: + raise ProjectNotFound( + project_id=project_id, + available_projects=self.get_available_projects(), + available_workspaces=self.get_available_workspaces() + ) + response = self.backend_client.api.getProject( projectIdentifier=project_id, **self.DEFAULT_REQUEST_KWARGS, @@ -188,7 +213,50 @@ def get_project(self, project_id: str) -> Project: raise NeptuneLegacyProjectException(project_id) return Project(uuid.UUID(project.id), project.name, project.organizationName) except HTTPNotFound: - raise ProjectNotFound(project_id) + raise ProjectNotFound(project_id, + available_projects=self.get_available_projects(workspace_id=workspace), + available_workspaces=list() if workspace else self.get_available_workspaces()) + + @with_api_exceptions_handler + def get_available_projects(self, + workspace_id: Optional[str] = None, + search_term: Optional[str] = None + ) -> List[Project]: + try: + response = self.backend_client.api.listProjects( + limit=5, + organizationIdentifier=workspace_id, + searchTerm=search_term, + sortBy=['lastViewed'], + sortDirection=['descending'], + userRelation='memberOrHigher', + **self.DEFAULT_REQUEST_KWARGS, + ).response() + warning = response.metadata.headers.get('X-Server-Warning') + if warning: + click.echo(warning) # TODO print in color once colored exceptions are added + projects = response.result.entries + return list(map( + lambda project: Project(uuid.UUID(project.id), project.name, project.organizationName), + projects)) + except HTTPNotFound: + return [] + + @with_api_exceptions_handler + def get_available_workspaces(self) -> List[Workspace]: + try: + response = self.backend_client.api.listOrganizations( + **self.DEFAULT_REQUEST_KWARGS, + ).response() + warning = response.metadata.headers.get('X-Server-Warning') + if warning: + click.echo(warning) # TODO print in color once colored exceptions are added + workspaces = response.result + return list(map( + lambda workspace: Workspace(_uuid=uuid.UUID(workspace.id), name=workspace.name), + workspaces)) + except HTTPNotFound: + return [] @with_api_exceptions_handler def get_run(self, run_id: str): diff --git a/neptune/new/internal/backends/neptune_backend.py b/neptune/new/internal/backends/neptune_backend.py index 47835e413..fdd4b8ac3 100644 --- a/neptune/new/internal/backends/neptune_backend.py +++ b/neptune/new/internal/backends/neptune_backend.py @@ -29,6 +29,7 @@ FloatSeriesAttribute, IntAttribute, Project, + Workspace, StringAttribute, StringSeriesAttribute, StringSetAttribute, @@ -55,6 +56,17 @@ def websockets_factory(self, project_uuid: uuid.UUID, run_uuid: uuid.UUID) -> Op def get_project(self, project_id: str) -> Project: pass + @abc.abstractmethod + def get_available_projects(self, + workspace_id: Optional[str] = None, + search_term: Optional[str] = None + ) -> List[Project]: + pass + + @abc.abstractmethod + def get_available_workspaces(self) -> List[Workspace]: + pass + @abc.abstractmethod def get_run(self, run_id: str) -> ApiRun: pass diff --git a/neptune/new/internal/backends/neptune_backend_mock.py b/neptune/new/internal/backends/neptune_backend_mock.py index a32ada877..4a943e3f1 100644 --- a/neptune/new/internal/backends/neptune_backend_mock.py +++ b/neptune/new/internal/backends/neptune_backend_mock.py @@ -38,6 +38,7 @@ FloatSeriesAttribute, IntAttribute, Project, + Workspace, StringAttribute, StringSeriesAttribute, StringSetAttribute, @@ -102,7 +103,16 @@ def get_display_address(self) -> str: return "OFFLINE" def get_project(self, project_id: str) -> Project: - return Project(uuid.uuid4(), "sandbox", "workspace") + return Project(uuid.uuid4(), "project-placeholder", "offline") + + def get_available_projects(self, + workspace_id: Optional[str] = None, + search_term: Optional[str] = None + ) -> List[Project]: + return [Project(uuid.uuid4(), "project-placeholder", "offline")] + + def get_available_workspaces(self) -> List[Workspace]: + return [Workspace(uuid.uuid4(), "offline")] def create_run(self, project_uuid: uuid.UUID, diff --git a/neptune/new/internal/backends/project_name_lookup.py b/neptune/new/internal/backends/project_name_lookup.py new file mode 100644 index 000000000..277057e3d --- /dev/null +++ b/neptune/new/internal/backends/project_name_lookup.py @@ -0,0 +1,44 @@ +# +# Copyright (c) 2021, Neptune Labs Sp. z o.o. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import logging +import os +from typing import Optional + +from neptune.new.envs import PROJECT_ENV_NAME +from neptune.new.exceptions import NeptuneMissingProjectNameException +from neptune.new.internal.backends.neptune_backend import NeptuneBackend +from neptune.new.internal.utils import verify_type +from neptune.new.project import Project +from neptune.new.version import version as parsed_version + +__version__ = str(parsed_version) + +_logger = logging.getLogger(__name__) + + +def project_name_lookup(backend: NeptuneBackend, name: Optional[str] = None) -> Project: + verify_type("name", name, (str, type(None))) + + if not name: + name = os.getenv(PROJECT_ENV_NAME) + if not name: + available_workspaces = backend.get_available_workspaces() + available_projects = backend.get_available_projects() + + raise NeptuneMissingProjectNameException(available_workspaces=available_workspaces, + available_projects=available_projects) + + return backend.get_project(name) diff --git a/neptune/new/internal/get_project_impl.py b/neptune/new/internal/get_project_impl.py index 133075900..df1279945 100644 --- a/neptune/new/internal/get_project_impl.py +++ b/neptune/new/internal/get_project_impl.py @@ -14,15 +14,10 @@ # limitations under the License. # import logging -import os -import re from typing import Optional -from neptune.patterns import PROJECT_QUALIFIED_NAME_PATTERN - -from neptune.new.envs import PROJECT_ENV_NAME -from neptune.new.exceptions import NeptuneIncorrectProjectNameException, NeptuneMissingProjectNameException from neptune.new.internal.backends.hosted_neptune_backend import HostedNeptuneBackend +from neptune.new.internal.backends.project_name_lookup import project_name_lookup from neptune.new.internal.credentials import Credentials from neptune.new.internal.utils import verify_type from neptune.new.project import Project @@ -66,15 +61,7 @@ def get_project(name: Optional[str] = None, api_token: Optional[str] = None) -> verify_type("name", name, (str, type(None))) verify_type("api_token", api_token, (str, type(None))) - if not name: - name = os.getenv(PROJECT_ENV_NAME) - if not name: - raise NeptuneMissingProjectNameException() - if not re.match(PROJECT_QUALIFIED_NAME_PATTERN, name): - raise NeptuneIncorrectProjectNameException(name) - backend = HostedNeptuneBackend(Credentials(api_token=api_token)) - - project_obj = backend.get_project(name) + project_obj = project_name_lookup(backend, name) return Project(project_obj.uuid, backend) diff --git a/neptune/new/internal/init_impl.py b/neptune/new/internal/init_impl.py index 95726f9ef..ad57040b4 100644 --- a/neptune/new/internal/init_impl.py +++ b/neptune/new/internal/init_impl.py @@ -16,7 +16,6 @@ import logging import os -import re import uuid from datetime import datetime from enum import Enum @@ -32,12 +31,12 @@ NEPTUNE_RUNS_DIRECTORY, OFFLINE_DIRECTORY, ) -from neptune.new.envs import CUSTOM_RUN_ID_ENV_NAME, NEPTUNE_NOTEBOOK_ID, NEPTUNE_NOTEBOOK_PATH, PROJECT_ENV_NAME -from neptune.new.exceptions import (NeedExistingRunForReadOnlyMode, NeptuneIncorrectProjectNameException, - NeptuneMissingProjectNameException, NeptuneRunResumeAndCustomIdCollision, +from neptune.new.envs import CUSTOM_RUN_ID_ENV_NAME, NEPTUNE_NOTEBOOK_ID, NEPTUNE_NOTEBOOK_PATH +from neptune.new.exceptions import (NeedExistingRunForReadOnlyMode, NeptuneRunResumeAndCustomIdCollision, NeptunePossibleLegacyUsageException) from neptune.new.internal.backends.hosted_neptune_backend import HostedNeptuneBackend from neptune.new.internal.backends.neptune_backend import NeptuneBackend +from neptune.new.internal.backends.project_name_lookup import project_name_lookup from neptune.new.internal.backends.neptune_backend_mock import NeptuneBackendMock from neptune.new.internal.backends.offline_neptune_backend import OfflineNeptuneBackend from neptune.new.internal.backgroud_job_list import BackgroundJobList @@ -65,7 +64,6 @@ from neptune.new.run import Run from neptune.new.types.series.string_series import StringSeries from neptune.new.version import version as parsed_version -from neptune.patterns import PROJECT_QUALIFIED_NAME_PATTERN __version__ = str(parsed_version) @@ -259,14 +257,10 @@ def init(project: Optional[str] = None, if mode == RunMode.OFFLINE or mode == RunMode.DEBUG: project = 'offline/project-placeholder' - elif not project: - project = os.getenv(PROJECT_ENV_NAME) - if not project: - raise NeptuneMissingProjectNameException() - if not re.match(PROJECT_QUALIFIED_NAME_PATTERN, project): - raise NeptuneIncorrectProjectNameException(project) - - project_obj = backend.get_project(project) + + project_obj = project_name_lookup(backend, project) + project = f'{project_obj.workspace}/{project_obj.name}' + if run: api_run = backend.get_run(project + '/' + run) else: diff --git a/neptune/patterns.py b/neptune/patterns.py index 2c97f1708..5988e7a54 100644 --- a/neptune/patterns.py +++ b/neptune/patterns.py @@ -15,4 +15,4 @@ # -PROJECT_QUALIFIED_NAME_PATTERN = "^([^/]+)/([^/]+)$" +PROJECT_QUALIFIED_NAME_PATTERN = "^((?P[^/]+)/){0,1}(?P[^/]+)$"