ADD project delete, WIP gene cache

server/src/scimodom/app.py

@@ -15,6 +15,7 @@
     add_project,
     add_user_to_project,
     create_project_template,
+    delete_project,
 )
 from scimodom.cli.dataset import add_dataset, add_all, add_selection
 from scimodom.cli.utilities import validate_dataset_title, upsert
@@ -424,6 +425,28 @@ def batch(input_directory, request_uuids, annotation):
             input_directory, project_template_list, request_uuids, annotation_source
         )
 
+    @app.cli.command(
+        "delete", epilog="Check docs at https://dieterich-lab.github.io/scimodom/."
+    )
+    @click.option(
+        "-s",
+        "--selection",
+        default=[],
+        multiple=True,
+        required=False,
+        type=click.INT,
+        help="Selection ID(s) to delete. Repeat parameter to pass multiple selection IDs. Use at your own risk!",
+    )
+    @click.argument("smid", type=click.STRING)
+    def delete(smid, selection):
+        """Delete a project and all associated
+        data from the database. Deleting
+        selections at your own risk.
+
+        SMID is the Sci-ModoM project ID.
+        """
+        delete_project(smid, selection)
+
     @app.cli.command(
         "setup", epilog="Check docs at https://dieterich-lab.github.io/scimodom/."
     )

server/src/scimodom/cli/dataset.py

@@ -1,6 +1,7 @@
 from collections import defaultdict
 from pathlib import Path
 import re
+from typing import Iterator
 
 import click
 from sqlalchemy import select
@@ -9,6 +10,7 @@
 
 from scimodom.database.database import get_session
 from scimodom.database.models import (
+    Dataset,
     Modification,
     DetectionTechnology,
     Organism,
@@ -291,6 +293,29 @@ def add_selection(
         session.rollback()
 
 
+def delete_selection(selection: Selection) -> None:
+    session = get_session()
+    session.delete(selection)
+    session.commit()
+    # TODO delete gene cache
+
+
+def get_selection_from_dataset(dataset: Dataset) -> Iterator[Selection]:
+    session = get_session()
+    modification_ids = [
+        association.modification_id for association in dataset.associations
+    ]
+    for modification_id in modification_ids:
+        selection = session.execute(
+            select(Selection).filter_by(
+                modification_id=modification_id,
+                organism_id=dataset.organism_id,
+                technology_id=dataset.technology_id,
+            )
+        ).scalar_one()
+        yield selection
+
+
 def _get_modification_ids(values):
     modification_ids = []
     for rna, modomics in values:

server/src/scimodom/cli/project.py

@@ -1,16 +1,18 @@
 import csv
 import datetime
-from pathlib import Path
 
 import click
+from sqlalchemy import select
 from sqlalchemy.exc import NoResultFound
 
 from scimodom.cli.utilities import (
     add_assembly_to_template_if_none,
     get_detection_id,
     get_modomics_id,
 )
+from scimodom.cli.dataset import delete_selection, get_selection_from_dataset
 from scimodom.services.assembly import AssemblyNotFoundError, get_assembly_service
+from scimodom.services.dataset import get_dataset_service
 from scimodom.services.permission import get_permission_service
 from scimodom.services.project import get_project_service
 from scimodom.services.user import get_user_service, NoSuchUser
@@ -188,3 +190,97 @@ def create_project_template(
         f"Created request '{uuid}'... ",
         fg="green",
     )
+
+
+def delete_project(smid: str, selection_ids: list[int]) -> None:
+    """Provide a CLI function to delete a project
+    and all associated data.
+
+    Delete from the following tables:
+    - data_annotation
+    - data
+    - dataset_modification_association
+    - bam_file
+    - dataset
+    - project_source
+    - user_project_association
+    - project
+    - project_contact
+
+    BAM files associated with a dataset belonging
+    to this project are deleted from the file system.
+
+    :param smid: Project ID (SMID)
+    :type smid: str
+    :param selection_ids: List of selection IDs to delete,
+    at your own risk.
+    :type selection_ids: list[int]
+    """
+    project_service = get_project_service()
+    dataset_service = get_dataset_service()
+    try:
+        project = project_service.get_by_id(smid)
+    except NoResultFound:
+        click.secho(f"No such Project '{smid}'.", fg="red")
+        return
+
+    datasets = project.datasets
+    click.secho(
+        f"Deleting '{project.id}' with title '{project.title}' incl. data associated with the following datasets: ",
+        fg="green",
+    )
+    for dataset in datasets:
+        click.secho(f"Dataset '{dataset.id}' with title '{dataset.title}'", fg="green")
+    click.secho("Continue [y/n]?", fg="green")
+    c = click.getchar()
+    if c not in ["y", "Y"]:
+        return
+
+    dataset_list = [dataset.id for dataset in datasets]
+    selection_dict = dict()
+    for dataset in datasets:
+        try:
+            for selection in get_selection_from_dataset(dataset):
+                selection_dict[selection.id] = selection
+            dataset_service.delete_dataset(dataset)
+            dataset_list.remove(dataset.id)
+        except Exception as exc:
+            click.secho(f"Failed to delete Dataset '{dataset.id}': {exc}.", fg="red")
+            click.secho(
+                f"The following datasets were not deleted: {','.join(dataset_list)}.",
+                fg="red",
+            )
+            click.secho(f"Project {project.id} will not be deleted.", fg="red")
+            return
+
+    try:
+        project_service.delete_project(project)
+    except Exception as exc:
+        click.secho(f"Failed to delete project '{smid}': {exc}.", fg="red")
+        return
+
+    for selection_id in selection_ids:
+        if selection_id not in selection_dict.keys():
+            click.secho(
+                "This selection does not appear to be associated with this project data... skipping!",
+                fg="yellow",
+            )
+            continue
+        selection = selection_dict[selection_id]
+        click.secho(
+            f"Are you sure you want to delete Selection '{selection.id}'?", fg="green"
+        )
+        click.secho("Continue [y/n]?", fg="green")
+        c = click.getchar()
+        if c not in ["y", "Y"]:
+            click.secho("Skipping...", fg="yellow")
+            continue
+        try:
+            delete_selection(selection)
+        except Exception as exc:
+            click.secho(
+                f"Failed to delete selection '{selection.id}': {exc}.", fg="red"
+            )
+            return
+
+    click.secho("...done deleting project and data!", fg="green")

server/src/scimodom/services/dataset.py

@@ -37,6 +37,7 @@
     AssemblyService,
 )
 from scimodom.services.bedtools import get_bedtools_service, BedToolsService
+from scimodom.services.file import FileService, get_file_service
 from scimodom.utils import utils
 from scimodom.utils.bed_importer import EufImporter
 from scimodom.utils.bedtools_dto import EufRecord
@@ -109,11 +110,13 @@ class DatasetService:
     def __init__(
         self,
         session: Session,
+        file_service: FileService,
         bedtools_service: BedToolsService,
         assembly_service: AssemblyService,
         annotation_service: AnnotationService,
     ):
         self._session = session
+        self._file_service = file_service
         self._bedtools_service = bedtools_service
         self._assembly_service = assembly_service
         self._annotation_service = annotation_service
@@ -289,6 +292,39 @@ def import_dataset(
         )
         return context.eufid
 
+    def delete_dataset(self, dataset: Dataset) -> None:
+        """Delete a dataset and all associated data.
+
+        Delete from the following tables:
+        - data_annotation
+        - data
+        - dataset_modification_association
+        - bam_file
+        - dataset
+
+        Associated BAM files are deleted from
+        the file system.
+
+        :param dataset: Dataset instance to delete
+        :type dataset: Dataset
+        """
+        try:
+            self._delete_data_records(dataset.id)
+            self._session.execute(
+                delete(DatasetModificationAssociation).filter_by(dataset_id=dataset.id)
+            )
+            bam_list = self._file_service.get_bam_file_list(dataset)
+            for bam in bam_list:
+                bam_file = self._file_service.get_bam_file(
+                    dataset, bam["original_file_name"]
+                )
+                self._file_service.remove_bam_file(bam_file)
+            self._session.delete(dataset)
+            self._session.commit()
+        except Exception:
+            self._session.rollback()
+            raise
+
     @staticmethod
     def _check_euf_record(record, importer, context):
         if record.chrom not in context.seqids:
@@ -559,6 +595,7 @@ def get_dataset_service() -> DatasetService:
 
     return DatasetService(
         session=get_session(),
+        file_service=get_file_service(),
         bedtools_service=get_bedtools_service(),
         assembly_service=get_assembly_service(),
         annotation_service=get_annotation_service(),

server/src/scimodom/services/project.py

@@ -6,7 +6,7 @@
 from typing import Any
 
 from sqlalchemy.orm import Session
-from sqlalchemy import tuple_, and_, or_, select, func
+from sqlalchemy import tuple_, and_, or_, select, func, delete
 
 from scimodom.database.database import get_session
 from scimodom.database.models import (
@@ -120,6 +120,37 @@ def create_project(
             self._session.rollback()
             raise
 
+    def delete_project(self, project: Project) -> None:
+        """Delete a project and all associated data. There
+        must be no conflicting foreign key constraints
+        (not using ON DELETE CASCADE), i.e. dataset and
+        associated data must be deleted first.
+
+        Delete from the following tables:
+        - project_source
+        - user_project_association
+        - project
+        - project_contact
+
+        :param smid: Project instance
+        :type smid: Project
+        """
+        try:
+            self._session.execute(
+                delete(ProjectSource).filter_by(project_id=project.id)
+            )
+            self._session.execute(
+                delete(UserProjectAssociation).filter_by(project_id=project.id)
+            )
+            contact = self._session.get_one(ProjectContact, project.contact_id)
+            if len(contact.projects) == 1:
+                self._session.delete(contact)
+            self._session.delete(project)
+            self._session.commit()
+        except Exception:
+            self._session.rollback()
+            raise
+
     @staticmethod
     def _get_prj_src(project_template: ProjectTemplate):
         ors = or_(False)

server/tests/conftest.py

@@ -3,7 +3,7 @@
 
 import pytest
 from sqlalchemy import create_engine
-from sqlalchemy.orm import sessionmaker
+from sqlalchemy.orm import sessionmaker, scoped_session
 
 from scimodom.database.database import init, Base
 from scimodom.utils.specifications import SPECS_EUF
@@ -16,13 +16,15 @@
     "tests.fixtures.selection",
     "tests.fixtures.project",
     "tests.fixtures.dataset",
+    "tests.fixtures.bam_file",
 ]
 
 
 @pytest.fixture()
 def Session():
     engine = create_engine("sqlite:///:memory:")
     session = sessionmaker(autocommit=False, autoflush=False, bind=engine)
+    session = scoped_session(session)
     init(engine, lambda: session)
     Base.metadata.create_all(engine)
 

server/tests/fixtures/bam_file.py

@@ -0,0 +1,31 @@
+from datetime import datetime, timezone
+from uuid import uuid4
+
+import pytest
+
+from scimodom.database.models import BamFile
+
+
+@pytest.fixture
+def bam_file(Session, dataset):
+    name1 = f"{dataset[0].id}_1.bam"
+    bam_file1 = BamFile(
+        dataset_id=dataset[0].id,
+        original_file_name=name1,
+        storage_file_name=f"{dataset[0].id}_{uuid4()}_{name1}"[:256],
+    )
+    name2 = f"{dataset[0].id}_2.bam"
+    bam_file2 = BamFile(
+        dataset_id=dataset[0].id,
+        original_file_name=name2,
+        storage_file_name=f"{dataset[0].id}_{uuid4()}_{name2}"[:256],
+    )
+    name3 = f"{dataset[1].id}.bam"
+    bam_file3 = BamFile(
+        dataset_id=dataset[1].id,
+        original_file_name=name3,
+        storage_file_name=f"{dataset[1].id}_{uuid4()}_{name3}"[:256],
+    )
+    session = Session()
+    session.add_all([bam_file1, bam_file2, bam_file3])
+    session.commit()