Skip to content

Commit

Permalink
fix: make usage of name and slug consistent
Browse files Browse the repository at this point in the history
  • Loading branch information
m-alisafaee committed Sep 21, 2023
1 parent 19142c6 commit 5af000a
Show file tree
Hide file tree
Showing 64 changed files with 943 additions and 744 deletions.
22 changes: 11 additions & 11 deletions renku/command/checks/datasets.py
Original file line number Diff line number Diff line change
Expand Up @@ -77,17 +77,17 @@ def check_missing_files(dataset_gateway: IDatasetGateway, **_):
path = project_context.path / file_.entity.path
file_exists = path.exists() or (file_.is_external and os.path.lexists(path))
if not file_exists:
missing[dataset.name].append(file_.entity.path)
missing[dataset.slug].append(file_.entity.path)

if not missing:
return True, False, None

problems = WARNING + "There are missing files in datasets."

for dataset_name, files in missing.items():
for dataset_slug, files in missing.items():
problems += (
"\n\t"
+ click.style(dataset_name, fg="yellow")
+ click.style(dataset_slug, fg="yellow")
+ ":\n\t "
+ "\n\t ".join(click.style(path, fg="red") for path in files)
)
Expand All @@ -114,9 +114,9 @@ def fix_or_report(dataset):
dataset.unfreeze()
dataset.derived_from = None
dataset.freeze()
communication.info(f"Fixing dataset '{dataset.name}'")
communication.info(f"Fixing dataset '{dataset.slug}'")
else:
invalid_datasets.append(dataset.name)
invalid_datasets.append(dataset.slug)

for dataset in dataset_gateway.get_provenance_tails():
while dataset.derived_from is not None and dataset.derived_from.url_id is not None:
Expand All @@ -137,7 +137,7 @@ def fix_or_report(dataset):
WARNING
+ "There are invalid dataset metadata in the project (use 'renku doctor --fix' to fix them):"
+ "\n\n\t"
+ "\n\t".join(click.style(name, fg="yellow") for name in invalid_datasets)
+ "\n\t".join(click.style(slug, fg="yellow") for slug in invalid_datasets)
+ "\n"
)

Expand Down Expand Up @@ -177,12 +177,12 @@ def check_dataset_files_outside_datadir(fix, dataset_gateway: IDatasetGateway, *
continue

if fix:
communication.info(f"Fixing dataset '{dataset.name}' files.")
communication.info(f"Fixing dataset '{dataset.slug}' files.")
dataset.unfreeze()
for file in detected_files:
dataset.unlink_file(file.entity.path)
dataset.freeze()
add_to_dataset(dataset.name, urls=[file.entity.path for file in detected_files], link=True)
add_to_dataset(dataset.slug, urls=[file.entity.path for file in detected_files], link=True)
else:
invalid_files.extend(detected_files)

Expand Down Expand Up @@ -220,7 +220,7 @@ def check_external_files(fix, dataset_gateway: IDatasetGateway, **_):
for file in dataset.files:
if file.is_external:
external_files.append(file.entity.path)
datasets[dataset.name].append(file)
datasets[dataset.slug].append(file)

if not external_files:
return True, False, None
Expand All @@ -240,7 +240,7 @@ def check_external_files(fix, dataset_gateway: IDatasetGateway, **_):
f"dataset with an external storage backend:\n\t{external_files_str}"
)

for name, files in datasets.items():
file_unlink(name=name, yes=True, dataset_files=files)
for slug, files in datasets.items():
file_unlink(slug=slug, yes=True, dataset_files=files)

return True, False, None
4 changes: 2 additions & 2 deletions renku/command/checks/validate_shacl.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,15 +104,15 @@ def check_datasets_structure(dataset_gateway: IDatasetGateway, **_):
try:
conform, graph, t = _check_shacl_structure(data)
except (Exception, BaseException) as e:
problems.append(f"Couldn't validate dataset '{dataset.name}': {e}\n\n")
problems.append(f"Couldn't validate dataset '{dataset.slug}': {e}\n\n")
continue

if conform:
continue

ok = False

problems.append(f"{dataset.name}\n\t{_shacl_graph_to_string(graph)}\n")
problems.append(f"{dataset.slug}\n\t{_shacl_graph_to_string(graph)}\n")

if ok:
return True, False, None
Expand Down
7 changes: 3 additions & 4 deletions renku/command/dataset.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
#
# Copyright 2017-2023 - Swiss Data Science Center (SDSC)
# A partnership between École Polytechnique Fédérale de Lausanne (EPFL) and
# Copyright Swiss Data Science Center (SDSC). A partnership between
# École Polytechnique Fédérale de Lausanne (EPFL) and
# Eidgenössische Technische Hochschule Zürich (ETHZ).
#
# Licensed under the Apache License, Version 2.0 (the "License");
Expand Down Expand Up @@ -39,7 +38,7 @@


def search_datasets_command():
"""Command to get all the datasets whose name starts with the given string."""
"""Command to get all the datasets whose slug starts with the given string."""
return Command().command(search_datasets).require_migration().with_database()


Expand Down
9 changes: 6 additions & 3 deletions renku/command/format/dataset_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,10 @@ def get_lfs_tracking():
record.is_lfs = False

def naturalize(value) -> str:
return naturalsize(value).upper().replace("BYTES", " B")
try:
return naturalsize(value).upper().replace("BYTES", " B")
except ValueError:
return str(value)

def get_file_sizes():
if not any(r for r in records if r.size is None): # All records already have a size
Expand Down Expand Up @@ -181,8 +184,8 @@ def json(records, **_):
"dataset": ("title", "dataset"),
"full_path": ("full_path", None),
"path": ("path", None),
"short_name": ("dataset_name", "dataset name"),
"dataset_name": ("dataset_name", "dataset name"),
"short_name": ("dataset_slug", "dataset slug"),
"dataset_slug": ("dataset_slug", "dataset slug"),
"size": ("size", None),
"lfs": ("is_lfs", "lfs"),
"source": ("source", None),
Expand Down
5 changes: 3 additions & 2 deletions renku/command/format/datasets.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,13 +90,14 @@ def json(datasets, **kwargs):
"id": ("identifier", "id"),
"created": ("date_created", None),
"date_created": ("date_created", None),
"short_name": ("name", None),
"short_name": ("slug", None),
"name": ("name", None),
"slug": ("slug", None),
"creators": ("creators_csv", "creators"),
"creators_full": ("creators_full_csv", "creators"),
"tags": ("tags_csv", "tags"),
"version": ("version", None),
"title": ("title", "title"),
"title": ("name", None),
"keywords": ("keywords_csv", "keywords"),
"description": ("short_description", "description"),
"storage": ("storage", None),
Expand Down
4 changes: 2 additions & 2 deletions renku/command/move.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ def _move(sources: List[str], destination: str, force: bool, verbose: bool, to_d
absolute_sources = [_get_absolute_path(src) for src in sources]

if to_dataset:
target_dataset = DatasetsProvenance().get_by_name(to_dataset, strict=True)
target_dataset = DatasetsProvenance().get_by_slug(to_dataset, strict=True)
if not is_subpath(absolute_destination, _get_absolute_path(target_dataset.get_datadir())):
raise errors.ParameterError(
f"Destination {destination} must be in {target_dataset.get_datadir()} when moving to a dataset."
Expand Down Expand Up @@ -112,7 +112,7 @@ def _move(sources: List[str], destination: str, force: bool, verbose: bool, to_d
# NOTE: Force-add to include possible ignored files
repository.add(*files.values(), force=True)

move_files(files=files, to_dataset_name=to_dataset)
move_files(files=files, to_dataset_slug=to_dataset)

if verbose:
_show_moved_files(project_context.path, files)
Expand Down
4 changes: 2 additions & 2 deletions renku/command/schema/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -202,11 +202,11 @@ class Meta:
in_language = Nested(schema.inLanguage, LanguageSchema, load_default=None)
keywords = fields.List(schema.keywords, fields.String(), load_default=None)
license = Uri(schema.license, load_default=None)
name = fields.String(renku.slug)
slug = fields.String(renku.slug)
initial_identifier = fields.String(renku.originalIdentifier)
project_id = fields.IRI(renku.hasDataset, reverse=True)
same_as = Nested(schema.sameAs, UrlSchema, load_default=None)
title = fields.String(schema.name)
name = fields.String(schema.name)
version = fields.String(schema.version, load_default=None)

@pre_dump(pass_many=True)
Expand Down
6 changes: 3 additions & 3 deletions renku/command/view_model/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,14 @@
class DatasetViewModel:
"""A view model for a ``Dataset``."""

def __init__(self, name: str, same_as: Optional[str]):
self.name: str = name
def __init__(self, slug: str, same_as: Optional[str]):
self.slug: str = slug
self.same_as: Optional[str] = same_as

@classmethod
def from_dataset(cls, dataset: Dataset) -> "DatasetViewModel":
"""Create view model from ``Dataset``."""
return cls(name=dataset.name, same_as=dataset.same_as.value if dataset.same_as else None)
return cls(slug=dataset.slug, same_as=dataset.same_as.value if dataset.same_as else None)


class DatasetFileViewModel:
Expand Down
14 changes: 7 additions & 7 deletions renku/command/view_model/log.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ class DatasetChangeDetailsViewModel:
modified: bool = False
files_added: Optional[List[str]] = None
files_removed: Optional[List[str]] = None
title_changed: Optional[str] = None
name_changed: Optional[str] = None
description_changed: Optional[str] = None
creators_added: Optional[List[str]] = None
creators_removed: Optional[List[str]] = None
Expand Down Expand Up @@ -181,7 +181,7 @@ def from_dataset(cls, dataset: "Dataset") -> "DatasetLogViewModel":

dataset_gateway = inject.instance(IDatasetGateway)

descriptions = [f"Dataset '{dataset.name}': "]
descriptions = [f"Dataset '{dataset.slug}': "]
details = DatasetChangeDetailsViewModel()

if not dataset.derived_from and not dataset.same_as:
Expand Down Expand Up @@ -231,8 +231,8 @@ def from_dataset(cls, dataset: "Dataset") -> "DatasetLogViewModel":

if not previous_dataset:
# NOTE: Check metadata changes on create/import
if dataset.title:
details.title_changed = dataset.title
if dataset.name:
details.name_changed = dataset.name

if dataset.description:
details.description_changed = dataset.description
Expand All @@ -248,8 +248,8 @@ def from_dataset(cls, dataset: "Dataset") -> "DatasetLogViewModel":
elif not details.deleted:
# NOTE: Check metadata changes to previous dataset
modified = False
if dataset.title != previous_dataset.title:
details.title_changed = dataset.title
if dataset.name != previous_dataset.name:
details.name_changed = dataset.name
modified = True
if dataset.description != previous_dataset.description:
details.description_changed = dataset.description
Expand Down Expand Up @@ -286,7 +286,7 @@ def from_dataset(cls, dataset: "Dataset") -> "DatasetLogViewModel":
descriptions.append("metadata modified")

return DatasetLogViewModel(
id=dataset.name,
id=dataset.slug,
date=dataset.date_removed
if dataset.date_removed
else (
Expand Down
12 changes: 6 additions & 6 deletions renku/core/dataset/context.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,14 @@ class DatasetContext:

def __init__(
self,
name: str,
slug: str,
create: Optional[bool] = False,
commit_database: Optional[bool] = False,
creator: Optional[Person] = None,
datadir: Optional[Path] = None,
storage: Optional[str] = None,
) -> None:
self.name = name
self.slug = slug
self.create = create
self.commit_database = commit_database
self.creator = creator
Expand All @@ -49,17 +49,17 @@ def __init__(

def __enter__(self):
"""Enter context."""
self.dataset = self.dataset_provenance.get_by_name(name=self.name)
self.dataset = self.dataset_provenance.get_by_slug(slug=self.slug)
if self.dataset is None:
if not self.create:
raise errors.DatasetNotFound(name=self.name)
raise errors.DatasetNotFound(slug=self.slug)

# NOTE: Don't update provenance when creating here because it will be updated later
self.dataset = create_dataset(
name=self.name, update_provenance=False, datadir=self.datadir, storage=self.storage
slug=self.slug, update_provenance=False, datadir=self.datadir, storage=self.storage
)
elif self.create:
raise errors.DatasetExistsError(self.name)
raise errors.DatasetExistsError(self.slug)

return self.dataset

Expand Down
Loading

0 comments on commit 5af000a

Please sign in to comment.