From 819cc4e68888089c250594ff991356e36804d757 Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Fri, 3 May 2024 14:21:25 +0200 Subject: [PATCH 01/13] Add group overview page and links to it in the home view and menu. --- webapp/assets/img/icons8-group-96.png | Bin 0 -> 1799 bytes webapp/chlamdb/urls.py | 3 +- webapp/templates/chlamdb/groups_overview.html | 44 ++++++++++++++++++ webapp/templates/chlamdb/home.html | 13 ++++++ webapp/templates/chlamdb/menu.html | 4 ++ webapp/views/groups.py | 21 +++++++++ webapp/views/mixins.py | 4 +- webapp/views/object_type_metadata.py | 14 ++++++ webapp/views/utils.py | 1 + 9 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 webapp/assets/img/icons8-group-96.png create mode 100644 webapp/templates/chlamdb/groups_overview.html create mode 100644 webapp/views/groups.py diff --git a/webapp/assets/img/icons8-group-96.png b/webapp/assets/img/icons8-group-96.png new file mode 100644 index 0000000000000000000000000000000000000000..ead3cc9c4060aea26f025b1805048fbcd0c24c54 GIT binary patch literal 1799 zcmV+i2l)7jP)C>jm8f+!!&j!68YGoQ4~hthl3w2| z)cW8TAJ;yFl_FG7*cFltGBrvN3)CLu6-+YApcJF@T3Q(uWm#Z~Sn4kpcNv>GGw1Hd zoOAX$&JPZ3_TKZ~Yt77_$C}wEiUNT^AP@)y0)apv5C{YUfj}S-2m}fQFba4a*b3|i zj_Yr~o|9`MYi(!40gMCQAvsbQlWJ{u-2-$5mIG%=&l#XaG-~ZfT>|vcnw%HW=u>Nd z>apf;b!6xkU?wmcxJZ9;JWH>+3L(4+DGFe@B@?BiosuB!6rjzLz9nzPI4=MuY5N^# z8_+vt^CfBGh~~t~THgRr+Vt#Cllev=m~TRBWvp%xfRUD_V@eLasTRFaeywgGXrD;M z+2TR7)zEDAW9|9T{LJ*AIoo%kHY~saBeO<((7e*nY>Vn0pr4M#H)}_CE+*5a=j2-7 z47$gR%hxZxLW?EaY9~0lVA1x7@&v#;wj|AwHalj`7nT*(-;P927@|E=q?MsuoZ}uAWLkA(Ge+C%o_ajn z(tO3!bS1CgY0cuPOSdAu2{QUkyIJo57e?A94L^ zQ%HYxSU5k$0>!*~-jPO8{gvf6;1OVx?%O#x8j<6Tdad$HbXDSf^S1=x3UzXY2H6e) zf9hoGj2<7-pxsNr!@#gor9fhUVZc*5#?Q*vz>~n`A?wr4dN*VYKe#vztOv%2f^cX2 zAYiTMj5qlgc(EQwW&b0uGcW^K4SWIos1d{ydi*)CO7Ce(sQI8}HOb64nQ1-`+^ajN zhsM@s<@j2?_JZCkXadAg-(`p6u3Y; zRu{6cGB@LrWM3T?e&f&)g2TUv+f@)({c79VGgMf*aBb0>Aw(k-*$ZhyMT`mUo!@31Yr zXrGjaOluPI;NBg?-#-Et>Hhm6v9@@KuE{Rank8fM{+1dk*KF-_#Xl|8zT#&`{HCiV z%DF=AU8KIb{FOp{(Y~#b2w=_(c-HZ{UY18Ur43CEA6qmJ0L@GH(A=?!b(#+Y zRR0Y_`=D#=T?@3PN;EajT95j7TJrbC_zmlfiyJ*Y{8CHPmb;_#4RF7usTkx@Up{4u zvo8Mqli3E|?D4^}RV2>gvLk^Sp(8+v?ZjE&QU4Z0Tk^bLq>C4E_W63KKG+;@KM@ra z+9yW)bLXX-v7}|T~QVZg+F?(+qb-xN)pFqus)uubWl_)&LbbTnwI#b!+#VGi?31QSu-=>rrukdQtnBg? zTdD+_59TiCO0c;k5}@PMhqU87Qo)CRDz9DvLlL)W1*sHAG;bfM>PBT + + + + +{% include "chlamdb/header.html" %} +{% load custom_tags %} + + + +
+
+
+
+
+
+ {% include "chlamdb/menu.html" %} +

+ {{description|safe}} +

+
+ +
+ +
+
+
+

{{object_name_plural}}

+
+ + {% for group in groups %} + + {% endfor %} +
{{group|safe}}
+
+
+
+
+
+
+ + +{% include "chlamdb/style_menu.html" %} + diff --git a/webapp/templates/chlamdb/home.html b/webapp/templates/chlamdb/home.html index 323e6ed47..41640f5e1 100644 --- a/webapp/templates/chlamdb/home.html +++ b/webapp/templates/chlamdb/home.html @@ -252,6 +252,19 @@

Custom plots

+
+
+ +
+ +
+
+ +

{{metadata.group_metadata.object_name_plural}}

+

{{metadata.group_metadata.overview_description|safe}}

+
+
+

diff --git a/webapp/templates/chlamdb/menu.html b/webapp/templates/chlamdb/menu.html index e0b9f6304..5c9e7bacc 100644 --- a/webapp/templates/chlamdb/menu.html +++ b/webapp/templates/chlamdb/menu.html @@ -106,6 +106,10 @@ Custom plots +
  • + {{metadata.group_metadata.object_name_plural}} +
    • +
  • diff --git a/webapp/views/groups.py b/webapp/views/groups.py new file mode 100644 index 000000000..3d0241112 --- /dev/null +++ b/webapp/views/groups.py @@ -0,0 +1,21 @@ +from collections import namedtuple + +from django.shortcuts import render +from django.views import View + +from views.mixins import BaseViewMixin +from views.object_type_metadata import GroupMetadata + + +class GroupsOverview(BaseViewMixin, View, GroupMetadata): + + template = 'chlamdb/groups_overview.html' + view_name = "groups" + + @property + def metadata(self): + return namedtuple("metadata", "description")(self.overview_description) + + def get(self, request, *args, **kwargs): + groups = [group[0] for group in self.db.get_groups()] + return render(request, self.template, self.get_context(groups=groups)) diff --git a/webapp/views/mixins.py b/webapp/views/mixins.py index da9400083..4de7abd7c 100644 --- a/webapp/views/mixins.py +++ b/webapp/views/mixins.py @@ -64,8 +64,8 @@ def db(self): return self._db def page_title(self): - return page2title.get(getattr(self, "view_name", None), - page2title[f"{self.object_type}_comparison"]) + return page2title.get(getattr(self, "view_name", None)) or \ + page2title[f"{self.object_type}_comparison"] @property def object_column(self): diff --git a/webapp/views/object_type_metadata.py b/webapp/views/object_type_metadata.py index 481644ad1..8424f21fb 100644 --- a/webapp/views/object_type_metadata.py +++ b/webapp/views/object_type_metadata.py @@ -126,6 +126,16 @@ def index_comp_url(self): return None +class GroupMetadata(BaseObjectMetadata): + + object_type = "group" + object_name = "Genome Group" + overview_description = ( + "This page gives an overview of the groups of genomes that were " + "defined in your database.
    Groups can be used for easy selection " + "of genomes in various analysis views.") + + class MetadataGetter(): metadata_classes = [AmrMetadata, CogMetadata, KoMetadata, PfamMetadata, @@ -149,6 +159,10 @@ def get_orthology_metadata(self): return (self.object_type_to_metadata[object_type] for object_type in self._orthology) + @property + def group_metadata(self): + return GroupMetadata() + def my_locals(local_dico): local_dico["optional2status"] = optional2status diff --git a/webapp/views/utils.py b/webapp/views/utils.py index 633e74119..ea20eeb7a 100644 --- a/webapp/views/utils.py +++ b/webapp/views/utils.py @@ -25,6 +25,7 @@ def safe_replace(string, search_string, replace_string): 'Genome alignments: Circos plot': ['circos'], 'Genome alignments: Plot region': ['plot_region'], 'Genomes: table of contents': ['extract_contigs'], + 'Overview of defined groups of genomes.':['groups'], 'Homology search: Blast': ['blast'], 'Kegg Ortholog': ['fam_ko'], 'Kegg metabolic pathways': ['kegg_genomes'], From c53a2cde7f9f0ceaa40ce548bfa8fc12a39be640 Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Fri, 3 May 2024 16:04:36 +0200 Subject: [PATCH 02/13] Use result_table template in genomes view. --- webapp/templates/chlamdb/genomes.html | 51 +-------------------------- webapp/views/utils.py | 5 +++ webapp/views/views.py | 50 ++++++++++++++------------ 3 files changed, 34 insertions(+), 72 deletions(-) diff --git a/webapp/templates/chlamdb/genomes.html b/webapp/templates/chlamdb/genomes.html index cacb50162..cb30103ae 100644 --- a/webapp/templates/chlamdb/genomes.html +++ b/webapp/templates/chlamdb/genomes.html @@ -25,36 +25,8 @@
    - - - - - - {% for header in data_table_header %} - - {% endfor %} - - - - - {% for entry in data_table %} - - - - - - - - - - - - - - {% endfor %} - -
    {{header}}
    {{entry.1}}{{entry.2}}{{entry.3}}{{entry.4}}{{entry.5}}{{entry.6}}{{entry.7}} .faa .fna .ffn .gbk
    + {% include "chlamdb/result_table.html" with results=results %}
    @@ -67,25 +39,4 @@

    Help

    - diff --git a/webapp/views/utils.py b/webapp/views/utils.py index ea20eeb7a..cdbf43c84 100644 --- a/webapp/views/utils.py +++ b/webapp/views/utils.py @@ -206,6 +206,11 @@ def format_swissprot_entry(entry_id): return f'{entry_id}' +def format_genome(taxid_and_description): + taxid, description = taxid_and_description + return f'{description}' + + class DataTableConfig(): def __init__(self, table_id="results", ordering=True, paging=True, diff --git a/webapp/views/views.py b/webapp/views/views.py index 4fa8618ea..0ff7c435e 100644 --- a/webapp/views/views.py +++ b/webapp/views/views.py @@ -45,10 +45,11 @@ from views.errors import errors from views.mixins import CogViewMixin, ComparisonViewMixin, KoViewMixin from views.object_type_metadata import MetadataGetter, my_locals -from views.utils import (TabularResultTab, format_cog, format_gene, format_ko, - format_locus, format_orthogroup, - genomic_region_df_to_js, locusx_genomic_region, - make_div, optional2status, page2title, to_s) +from views.utils import (DataTableConfig, TabularResultTab, format_cog, + format_gene, format_genome, format_ko, format_locus, + format_orthogroup, genomic_region_df_to_js, + locusx_genomic_region, make_div, optional2status, + page2title, to_s) def id_generator(size=6, chars=string.ascii_uppercase + string.ascii_lowercase + string.digits): @@ -166,22 +167,19 @@ def genomes(request): filenames_tax_id_db.index = list(filenames_tax_id_db['taxon_id']) filenames_list = list(filenames_tax_id_db["filename"]) - path_faa = [settings.BLAST_DB_PATH + "/faa/" + filename + ".faa" - for filename in filenames_list] - path_fna = [settings.BLAST_DB_PATH + "/fna/" + filename + ".fna" - for filename in filenames_list] - path_ffn = [settings.BLAST_DB_PATH + "/ffn/" + filename + ".ffn" - for filename in filenames_list] - path_gbk = [settings.BLAST_DB_PATH + "/gbk/" + filename + ".gbk" - for filename in filenames_list] - - filenames_tax_id_db['path_to_faa'] = path_faa - filenames_tax_id_db['path_to_fna'] = path_fna - filenames_tax_id_db['path_to_ffn'] = path_ffn - filenames_tax_id_db['path_to_gbk'] = path_gbk + path_template = settings.BLAST_DB_PATH + "/{ext}/{filename}.{ext}" + link_template = ' .{{ext}} '.format(path_template) + for ext in ["faa", "fna", "ffn", "gbk"]: + filenames_tax_id_db[f'path_to_{ext}'] = [ + link_template.format(filename=filename, ext=ext) + for filename in filenames_list] + filenames_tax_id_db = filenames_tax_id_db[["path_to_faa", "path_to_fna", "path_to_ffn", "path_to_gbk"]] genomes_data = genomes_data.join(filenames_tax_id_db, on="taxon_id") + + genomes_data["accession"] = genomes_data[["id", "description"]].apply(format_genome, axis=1) + data_table_header = [ "Name", "GC %", @@ -195,9 +193,9 @@ def genomes(request): "ffn seq", "gbk file" ] - data_table = genomes_data[[ - "id", - "description", + + table_data_accessors = [ + "accession", "gc", "n_prot", "n_contigs", @@ -207,8 +205,16 @@ def genomes(request): "path_to_faa", "path_to_fna", "path_to_ffn", - "path_to_gbk"]].values.tolist() - return render(request, 'chlamdb/genomes.html', my_locals(locals())) + "path_to_gbk"] + + table_data = genomes_data[table_data_accessors] + + results = {"table_data": table_data, + "table_headers": data_table_header, + "data_table_config": DataTableConfig(), + "table_data_accessors": table_data_accessors} + return render(request, 'chlamdb/genomes.html', + my_locals({"results": results})) def extract_contigs(request, genome): From 2044b4120863422cbf6ab0c6cc80d7872d63dfde Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Fri, 3 May 2024 16:06:41 +0200 Subject: [PATCH 03/13] Clean-up genomes.html template. --- webapp/templates/chlamdb/genomes.html | 67 ++++++++++++++------------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/webapp/templates/chlamdb/genomes.html b/webapp/templates/chlamdb/genomes.html index cb30103ae..cee2ca8f2 100644 --- a/webapp/templates/chlamdb/genomes.html +++ b/webapp/templates/chlamdb/genomes.html @@ -1,42 +1,47 @@ - - - - {% load static %} - {% load custom_tags %} - {% include "chlamdb/header.html" %} - {% include "chlamdb/style_menu.html" %} + + {% load static %} + {% load custom_tags %} + {% include "chlamdb/header.html" %} + {% include "chlamdb/style_menu.html" %} -
    -
    -
    -
    -
    -
    - {% include "chlamdb/menu.html" %} - -
    -
    - - {% include "chlamdb/result_table.html" with results=results %} - -
    -
    -

    Help

    -
    - -

    This table contains the list of genomes included in the database and a summary of their content.
    Clicking on the genome name (first column) a second table with the protein content is displayed. It shows to which contig each protein belongs and provides the link to the locus tags. -
    Fasta and gbk files can be downloaded. -

    -
    -
    - +
    +
    +
    +
    +
    +
    + {% include "chlamdb/menu.html" %} + +
    +
    + + {% include "chlamdb/result_table.html" with results=results %} + +
    +
    +

    Help

    +
    + +

    + This table contains the list of genomes included in the database and a summary of their content.
    Clicking on the genome name (first column) a second table with the protein content is displayed. It shows to which contig each protein belongs and provides the link to the locus tags. +
    Fasta and gbk files can be downloaded. +

    +
    +
    +
    +
    +
    +
    +
    +
    + From 9cc6a24d5580f855a5b80a91b4e6d65182a22c77 Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Fri, 3 May 2024 16:20:00 +0200 Subject: [PATCH 04/13] Rewrite genomes view as class based view. --- webapp/chlamdb/urls.py | 2 +- webapp/views/mixins.py | 18 ++++-- webapp/views/views.py | 121 +++++++++++++++++++++-------------------- 3 files changed, 75 insertions(+), 66 deletions(-) diff --git a/webapp/chlamdb/urls.py b/webapp/chlamdb/urls.py index 1a67883fb..bffa71120 100644 --- a/webapp/chlamdb/urls.py +++ b/webapp/chlamdb/urls.py @@ -54,7 +54,7 @@ re_path(r'^gwas_amr/', gwas.AmrGwasView.as_view(), name="gwas_amr"), re_path(r'^groups/', groups.GroupsOverview.as_view(), name=groups.GroupsOverview.view_name), re_path(r'^get_cog/([a-zA-Z0-9_\.]+)/([a-zA-Z0-9_\.\%]+)$', views.get_cog, name="get_cog"), # noqa - re_path(r'^genomes', views.genomes, name='genomes'), + re_path(r'^genomes', views.Genomes.as_view(), name=views.Genomes.view_name), re_path(r'^favicon\.ico$', favicon_view), re_path(r'^FAQ', views.faq, name='FAQ'), re_path(r'^fam_vf/(VFG[0-9]+)$', fam.FamVfView.as_view(), name="fam_vf"), diff --git a/webapp/views/mixins.py b/webapp/views/mixins.py index 4de7abd7c..0748675ed 100644 --- a/webapp/views/mixins.py +++ b/webapp/views/mixins.py @@ -105,13 +105,19 @@ def metadata(self): def get_context(self, **kwargs): context = { "page_title": self.page_title(), - "description": getattr(self.metadata, "description", ""), - "tab_name": getattr(self.metadata, "name", ""), - "object_type": self.object_type, - "object_name": self.object_name_plural, - "object_name_plural": self.object_name_plural, - "available_views": self.available_views, } + if hasattr(self, "metadata"): + context.update({ + "description": getattr(self.metadata, "description", ""), + "tab_name": getattr(self.metadata, "name", ""), + }) + if hasattr(self, "object_type"): + context.update({ + "object_type": self.object_type, + "object_name": self.object_name_plural, + "object_name_plural": self.object_name_plural, + "available_views": self.available_views, + }) if hasattr(self, "form"): context["form"] = self.form context.update(kwargs) diff --git a/webapp/views/views.py b/webapp/views/views.py index 0ff7c435e..3c942649a 100644 --- a/webapp/views/views.py +++ b/webapp/views/views.py @@ -43,7 +43,8 @@ CategoriesFreqHeatmapMetadata, HeatmapMetadata) from views.errors import errors -from views.mixins import CogViewMixin, ComparisonViewMixin, KoViewMixin +from views.mixins import (BaseViewMixin, CogViewMixin, ComparisonViewMixin, + KoViewMixin) from views.object_type_metadata import MetadataGetter, my_locals from views.utils import (DataTableConfig, TabularResultTab, format_cog, format_gene, format_genome, format_ko, format_locus, @@ -155,66 +156,68 @@ def get_genomes_data(db): return genomes_data -def genomes(request): - biodb_path = settings.BIODB_DB_PATH - db = DB.load_db_from_name(biodb_path) - page_title = page2title["genomes"] - genomes_data = get_genomes_data(db) - - filenames_tax_id = db.get_filenames_to_taxon_id() - filenames_tax_id_db = pd.DataFrame.from_dict(list(filenames_tax_id.items())) - filenames_tax_id_db.columns = ['filename', 'taxon_id'] - filenames_tax_id_db.index = list(filenames_tax_id_db['taxon_id']) - filenames_list = list(filenames_tax_id_db["filename"]) - - path_template = settings.BLAST_DB_PATH + "/{ext}/{filename}.{ext}" - link_template = ' .{{ext}} '.format(path_template) - for ext in ["faa", "fna", "ffn", "gbk"]: - filenames_tax_id_db[f'path_to_{ext}'] = [ - link_template.format(filename=filename, ext=ext) - for filename in filenames_list] +class Genomes(BaseViewMixin, View): - filenames_tax_id_db = filenames_tax_id_db[["path_to_faa", "path_to_fna", - "path_to_ffn", "path_to_gbk"]] - genomes_data = genomes_data.join(filenames_tax_id_db, on="taxon_id") + template = 'chlamdb/genomes.html' + view_name = "genomes" - genomes_data["accession"] = genomes_data[["id", "description"]].apply(format_genome, axis=1) - - data_table_header = [ - "Name", - "GC %", - "N proteins", - "N contigs", - "Size (Mbp)", - "Coding %", - "N plasmid contigs", - "faa seq", - "fna seq", - "ffn seq", - "gbk file" - ] - - table_data_accessors = [ - "accession", - "gc", - "n_prot", - "n_contigs", - "length", - "coding_density", - "has_plasmid", - "path_to_faa", - "path_to_fna", - "path_to_ffn", - "path_to_gbk"] - - table_data = genomes_data[table_data_accessors] - - results = {"table_data": table_data, - "table_headers": data_table_header, - "data_table_config": DataTableConfig(), - "table_data_accessors": table_data_accessors} - return render(request, 'chlamdb/genomes.html', - my_locals({"results": results})) + def get(self, request, *args, **kwargs): + genomes_data = get_genomes_data(self.db) + + filenames_tax_id = self.db.get_filenames_to_taxon_id() + filenames_tax_id_db = pd.DataFrame.from_dict(list(filenames_tax_id.items())) + filenames_tax_id_db.columns = ['filename', 'taxon_id'] + filenames_tax_id_db.index = list(filenames_tax_id_db['taxon_id']) + filenames_list = list(filenames_tax_id_db["filename"]) + + path_template = settings.BLAST_DB_PATH + "/{ext}/{filename}.{ext}" + link_template = ' .{{ext}} '.format(path_template) + for ext in ["faa", "fna", "ffn", "gbk"]: + filenames_tax_id_db[f'path_to_{ext}'] = [ + link_template.format(filename=filename, ext=ext) + for filename in filenames_list] + + filenames_tax_id_db = filenames_tax_id_db[ + ["path_to_faa", "path_to_fna", "path_to_ffn", "path_to_gbk"]] + genomes_data = genomes_data.join(filenames_tax_id_db, on="taxon_id") + + genomes_data["accession"] = genomes_data[["id", "description"]].apply( + format_genome, axis=1) + + data_table_header = [ + "Name", + "GC %", + "N proteins", + "N contigs", + "Size (Mbp)", + "Coding %", + "N plasmid contigs", + "faa seq", + "fna seq", + "ffn seq", + "gbk file" + ] + + table_data_accessors = [ + "accession", + "gc", + "n_prot", + "n_contigs", + "length", + "coding_density", + "has_plasmid", + "path_to_faa", + "path_to_fna", + "path_to_ffn", + "path_to_gbk"] + + table_data = genomes_data[table_data_accessors] + + results = {"table_data": table_data, + "table_headers": data_table_header, + "data_table_config": DataTableConfig(), + "table_data_accessors": table_data_accessors} + return render(request, self.template, self.get_context(results=results)) def extract_contigs(request, genome): From 493e07307daabd5f0f405f0945d9bd25e8e68761 Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Fri, 3 May 2024 16:39:00 +0200 Subject: [PATCH 05/13] Extract creation of genome table to its own mixin class. --- webapp/views/mixins.py | 66 +++++++++++++++++++++++++++++++++-- webapp/views/utils.py | 11 ++++++ webapp/views/views.py | 78 ++++-------------------------------------- 3 files changed, 81 insertions(+), 74 deletions(-) diff --git a/webapp/views/mixins.py b/webapp/views/mixins.py index 0748675ed..bcb8755fe 100644 --- a/webapp/views/mixins.py +++ b/webapp/views/mixins.py @@ -6,8 +6,10 @@ from views.object_type_metadata import (AmrMetadata, CogMetadata, KoMetadata, OrthogroupMetadata, PfamMetadata, VfMetadata, my_locals) -from views.utils import (format_hmm_url, format_ko_module, format_lst_to_html, - format_refseqid_to_ncbi, page2title, safe_replace) +from views.utils import (DataTableConfig, format_genome, format_hmm_url, + format_ko_module, format_lst_to_html, + format_refseqid_to_ncbi, get_genomes_data, page2title, + safe_replace) class Transform(): @@ -425,3 +427,63 @@ def dispatch(self, request, comp_type, *args, **kwargs): # Fix the metadata property self.mixin._metadata_cls = getattr(self, "_metadata_cls", None) return super(ComparisonViewMixin, self).dispatch(request, *args, **kwargs) + + +class GenomesTableMixin(): + + def get_genomes_table(self): + genomes_data = get_genomes_data(self.db) + + filenames_tax_id = self.db.get_filenames_to_taxon_id() + filenames_tax_id_db = pd.DataFrame.from_dict(list(filenames_tax_id.items())) + filenames_tax_id_db.columns = ['filename', 'taxon_id'] + filenames_tax_id_db.index = list(filenames_tax_id_db['taxon_id']) + filenames_list = list(filenames_tax_id_db["filename"]) + + path_template = settings.BLAST_DB_PATH + "/{ext}/{filename}.{ext}" + link_template = ' .{{ext}} '.format(path_template) + for ext in ["faa", "fna", "ffn", "gbk"]: + filenames_tax_id_db[f'path_to_{ext}'] = [ + link_template.format(filename=filename, ext=ext) + for filename in filenames_list] + + filenames_tax_id_db = filenames_tax_id_db[ + ["path_to_faa", "path_to_fna", "path_to_ffn", "path_to_gbk"]] + genomes_data = genomes_data.join(filenames_tax_id_db, on="taxon_id") + + genomes_data["accession"] = genomes_data[["id", "description"]].apply( + format_genome, axis=1) + + data_table_header = [ + "Name", + "GC %", + "N proteins", + "N contigs", + "Size (Mbp)", + "Coding %", + "N plasmid contigs", + "faa seq", + "fna seq", + "ffn seq", + "gbk file" + ] + + table_data_accessors = [ + "accession", + "gc", + "n_prot", + "n_contigs", + "length", + "coding_density", + "has_plasmid", + "path_to_faa", + "path_to_fna", + "path_to_ffn", + "path_to_gbk"] + + table_data = genomes_data[table_data_accessors] + + return {"table_data": table_data, + "table_headers": data_table_header, + "data_table_config": DataTableConfig(), + "table_data_accessors": table_data_accessors} diff --git a/webapp/views/utils.py b/webapp/views/utils.py index cdbf43c84..10cb44446 100644 --- a/webapp/views/utils.py +++ b/webapp/views/utils.py @@ -560,3 +560,14 @@ def extract_choices(self, indices, include_plasmids): taxids.update(self.db.get_taxids_for_groups(groups)) return list(taxids), plasmids + +def get_genomes_data(db): + genomes_data = db.get_genomes_infos() + genomes_descr = db.get_genomes_description() + + genomes_data = genomes_data.join(genomes_descr) + + genomes_data.gc = genomes_data.gc.apply(lambda x: round(100 * x)) + genomes_data.coding_density = genomes_data.coding_density.apply(lambda x: round(100 * x)) + genomes_data.length = genomes_data.length.apply(lambda x: round(x / pow(10, 6), 2)) + return genomes_data diff --git a/webapp/views/views.py b/webapp/views/views.py index 3c942649a..75935309a 100644 --- a/webapp/views/views.py +++ b/webapp/views/views.py @@ -44,11 +44,11 @@ HeatmapMetadata) from views.errors import errors from views.mixins import (BaseViewMixin, CogViewMixin, ComparisonViewMixin, - KoViewMixin) + GenomesTableMixin, KoViewMixin) from views.object_type_metadata import MetadataGetter, my_locals -from views.utils import (DataTableConfig, TabularResultTab, format_cog, - format_gene, format_genome, format_ko, format_locus, - format_orthogroup, genomic_region_df_to_js, +from views.utils import (TabularResultTab, format_cog, format_gene, format_ko, + format_locus, format_orthogroup, + genomic_region_df_to_js, get_genomes_data, locusx_genomic_region, make_div, optional2status, page2title, to_s) @@ -144,79 +144,13 @@ def get(self, request): return render(request, 'chlamdb/index_comp.html', context) -def get_genomes_data(db): - genomes_data = db.get_genomes_infos() - genomes_descr = db.get_genomes_description() - - genomes_data = genomes_data.join(genomes_descr) - - genomes_data.gc = genomes_data.gc.apply(lambda x: round(100 * x)) - genomes_data.coding_density = genomes_data.coding_density.apply(lambda x: round(100 * x)) - genomes_data.length = genomes_data.length.apply(lambda x: round(x / pow(10, 6), 2)) - return genomes_data - - -class Genomes(BaseViewMixin, View): +class Genomes(BaseViewMixin, View, GenomesTableMixin): template = 'chlamdb/genomes.html' view_name = "genomes" def get(self, request, *args, **kwargs): - genomes_data = get_genomes_data(self.db) - - filenames_tax_id = self.db.get_filenames_to_taxon_id() - filenames_tax_id_db = pd.DataFrame.from_dict(list(filenames_tax_id.items())) - filenames_tax_id_db.columns = ['filename', 'taxon_id'] - filenames_tax_id_db.index = list(filenames_tax_id_db['taxon_id']) - filenames_list = list(filenames_tax_id_db["filename"]) - - path_template = settings.BLAST_DB_PATH + "/{ext}/{filename}.{ext}" - link_template = ' .{{ext}} '.format(path_template) - for ext in ["faa", "fna", "ffn", "gbk"]: - filenames_tax_id_db[f'path_to_{ext}'] = [ - link_template.format(filename=filename, ext=ext) - for filename in filenames_list] - - filenames_tax_id_db = filenames_tax_id_db[ - ["path_to_faa", "path_to_fna", "path_to_ffn", "path_to_gbk"]] - genomes_data = genomes_data.join(filenames_tax_id_db, on="taxon_id") - - genomes_data["accession"] = genomes_data[["id", "description"]].apply( - format_genome, axis=1) - - data_table_header = [ - "Name", - "GC %", - "N proteins", - "N contigs", - "Size (Mbp)", - "Coding %", - "N plasmid contigs", - "faa seq", - "fna seq", - "ffn seq", - "gbk file" - ] - - table_data_accessors = [ - "accession", - "gc", - "n_prot", - "n_contigs", - "length", - "coding_density", - "has_plasmid", - "path_to_faa", - "path_to_fna", - "path_to_ffn", - "path_to_gbk"] - - table_data = genomes_data[table_data_accessors] - - results = {"table_data": table_data, - "table_headers": data_table_header, - "data_table_config": DataTableConfig(), - "table_data_accessors": table_data_accessors} + results = self.get_genomes_table() return render(request, self.template, self.get_context(results=results)) From 8c5d62d8bd4f20607e90014cce5a677516d6e527 Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Fri, 3 May 2024 16:54:04 +0200 Subject: [PATCH 06/13] Extract genome table help to GenomeTableMixin class. --- webapp/templates/chlamdb/genomes.html | 3 +-- webapp/views/mixins.py | 14 +++++++++++++- webapp/views/views.py | 1 + 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/webapp/templates/chlamdb/genomes.html b/webapp/templates/chlamdb/genomes.html index cee2ca8f2..385076e81 100644 --- a/webapp/templates/chlamdb/genomes.html +++ b/webapp/templates/chlamdb/genomes.html @@ -32,8 +32,7 @@

    Help

    - This table contains the list of genomes included in the database and a summary of their content.
    Clicking on the genome name (first column) a second table with the protein content is displayed. It shows to which contig each protein belongs and provides the link to the locus tags. -
    Fasta and gbk files can be downloaded. + {{results.table_help|safe}}

    diff --git a/webapp/views/mixins.py b/webapp/views/mixins.py index bcb8755fe..81935efd0 100644 --- a/webapp/views/mixins.py +++ b/webapp/views/mixins.py @@ -431,6 +431,17 @@ def dispatch(self, request, comp_type, *args, **kwargs): class GenomesTableMixin(): + _genome_table_help = ( + "This table contains the list of genomes included in the {} and a " + "summary of their content.
    Clicking on the genome name " + "(first column) a second table with the protein content is displayed. " + "It shows to which contig each protein belongs and provides the link " + "to the locus tags.
    Fasta and gbk files can be downloaded.") + + @property + def genome_table_help(self): + return self._genome_table_help.format(self.genome_source_object) + def get_genomes_table(self): genomes_data = get_genomes_data(self.db) @@ -486,4 +497,5 @@ def get_genomes_table(self): return {"table_data": table_data, "table_headers": data_table_header, "data_table_config": DataTableConfig(), - "table_data_accessors": table_data_accessors} + "table_data_accessors": table_data_accessors, + "table_help": self.genome_table_help} diff --git a/webapp/views/views.py b/webapp/views/views.py index 75935309a..731ba0e12 100644 --- a/webapp/views/views.py +++ b/webapp/views/views.py @@ -148,6 +148,7 @@ class Genomes(BaseViewMixin, View, GenomesTableMixin): template = 'chlamdb/genomes.html' view_name = "genomes" + genome_source_object = "database" def get(self, request, *args, **kwargs): results = self.get_genomes_table() From 2991cb25100ab8c4248664c3f3f1c2e167a8609e Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Fri, 3 May 2024 17:10:23 +0200 Subject: [PATCH 07/13] Add support for getting genome table for a limited number of taxids. --- webapp/lib/db_utils.py | 31 ++++++++++++++++++++----------- webapp/views/mixins.py | 4 ++-- webapp/views/utils.py | 7 ++++--- 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/webapp/lib/db_utils.py b/webapp/lib/db_utils.py index e6cc2e105..0293b4a24 100644 --- a/webapp/lib/db_utils.py +++ b/webapp/lib/db_utils.py @@ -1270,13 +1270,17 @@ def get_term_id(self, term, create_if_absent=False, ontology="Annotation Tags"): gc_term_id = result[0][0] return gc_term_id - def get_genomes_description(self): + def get_genomes_description(self, taxids=None): """ Returns the description of the genome as it has been read from the genbank files, indexed by taxon_id. The output also contains a flag has_plasmid indicating whether the genome contains a plasmid or not. """ - + if taxids: + plchd = self.gen_placeholder_string(taxids) + where_clause = f"WHERE entry.taxon_id IN ({plchd})" + else: + where_clause = "" has_plasmid_query = ( "SELECT * " "FROM bioentry_qualifier_value AS has_plasmid " @@ -1293,13 +1297,13 @@ def get_genomes_description(self): "INNER JOIN taxon_name as txn_name ON entry.taxon_id=txn_name.taxon_id " "INNER JOIN term AS orga_term ON orga.term_id=orga_term.term_id " " AND orga_term.name=\"organism\" " - "GROUP BY entry.taxon_id;" + f"{where_clause} GROUP BY entry.taxon_id;" ) - descr = self.server.adaptor.execute_and_fetchall(query) + descr = self.server.adaptor.execute_and_fetchall(query, taxids) columns = ["taxon_id", "description", "has_plasmid"] return DB.to_pandas_frame(descr, columns).set_index(["taxon_id"]) - def get_genomes_infos(self): + def get_genomes_infos(self, taxids=None): """ Note: for efficiency sake, it would be possible to order the different tables by taxon_id, walk through the table using zip in an iteator @@ -1308,30 +1312,35 @@ def get_genomes_infos(self): However, given the small size of the dataset, it's probably not worth the implementation effort. """ + if taxids: + plchd = self.gen_placeholder_string(taxids) + where_clause = f"WHERE taxon_id IN ({plchd})" + else: + where_clause = "" query = ( "SELECT entry.taxon_id, entry.taxon_id, COUNT(*) " " FROM seqfeature AS seq " " INNER JOIN term AS cds ON cds.term_id = seq.type_term_id AND cds.name=\"CDS\" " " INNER JOIN bioentry AS entry ON entry.bioentry_id = seq.bioentry_id " - " GROUP BY entry.taxon_id;" + f"{where_clause} GROUP BY entry.taxon_id;" ) - n_prot_results = self.server.adaptor.execute_and_fetchall(query) + n_prot_results = self.server.adaptor.execute_and_fetchall(query, taxids) query = ( "SELECT entry.taxon_id, COUNT(*) " "FROM bioentry AS entry " - "GROUP BY entry.taxon_id;" + f"{where_clause} GROUP BY entry.taxon_id;" ) - n_contigs = self.server.adaptor.execute_and_fetchall(query) + n_contigs = self.server.adaptor.execute_and_fetchall(query, taxids) cols = ["taxon_id", "completeness", "contamination", "gc", "length", "coding_density"] query = ( "SELECT taxon_id, completeness, contamination, gc, length, " - "coding_density from genome_summary;" + f"coding_density from genome_summary {where_clause};" ) - all_other_results = self.server.adaptor.execute_and_fetchall(query) + all_other_results = self.server.adaptor.execute_and_fetchall(query, taxids) df_n_prot = DB.to_pandas_frame( n_prot_results, ["taxon_id", "id", "n_prot"]).set_index("taxon_id") diff --git a/webapp/views/mixins.py b/webapp/views/mixins.py index 81935efd0..829f90775 100644 --- a/webapp/views/mixins.py +++ b/webapp/views/mixins.py @@ -442,8 +442,8 @@ class GenomesTableMixin(): def genome_table_help(self): return self._genome_table_help.format(self.genome_source_object) - def get_genomes_table(self): - genomes_data = get_genomes_data(self.db) + def get_genomes_table(self, taxids=None): + genomes_data = get_genomes_data(self.db, taxids=taxids) filenames_tax_id = self.db.get_filenames_to_taxon_id() filenames_tax_id_db = pd.DataFrame.from_dict(list(filenames_tax_id.items())) diff --git a/webapp/views/utils.py b/webapp/views/utils.py index 10cb44446..651bf05bd 100644 --- a/webapp/views/utils.py +++ b/webapp/views/utils.py @@ -561,9 +561,10 @@ def extract_choices(self, indices, include_plasmids): taxids.update(self.db.get_taxids_for_groups(groups)) return list(taxids), plasmids -def get_genomes_data(db): - genomes_data = db.get_genomes_infos() - genomes_descr = db.get_genomes_description() + +def get_genomes_data(db, taxids=None): + genomes_data = db.get_genomes_infos(taxids=taxids) + genomes_descr = db.get_genomes_description(taxids=taxids) genomes_data = genomes_data.join(genomes_descr) From cf83553ccfe40c95fe1c13daf27fc90c38b162d3 Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Fri, 3 May 2024 17:16:35 +0200 Subject: [PATCH 08/13] Add details view for groups of genomes. --- webapp/chlamdb/urls.py | 3 +- webapp/templates/chlamdb/group_details.html | 53 +++++++++++++++++++++ webapp/views/groups.py | 22 ++++++++- webapp/views/object_type_metadata.py | 8 ++++ webapp/views/utils.py | 3 +- 5 files changed, 85 insertions(+), 4 deletions(-) create mode 100644 webapp/templates/chlamdb/group_details.html diff --git a/webapp/chlamdb/urls.py b/webapp/chlamdb/urls.py index bffa71120..d44bc978f 100644 --- a/webapp/chlamdb/urls.py +++ b/webapp/chlamdb/urls.py @@ -52,7 +52,8 @@ re_path(r'^gwas_ko/', gwas.KoGwasView.as_view(), name="gwas_ko"), re_path(r'^gwas_cog/', gwas.CogGwasView.as_view(), name="gwas_cog"), re_path(r'^gwas_amr/', gwas.AmrGwasView.as_view(), name="gwas_amr"), - re_path(r'^groups/', groups.GroupsOverview.as_view(), name=groups.GroupsOverview.view_name), + re_path(r'^groups/([a-zA-Z0-9_\.\(\)\-\'\s]+)', groups.GroupDetails.as_view(), name=groups.GroupDetails.view_name), # noqa + re_path(r'^groups$', groups.GroupsOverview.as_view(), name=groups.GroupsOverview.view_name), # noqa re_path(r'^get_cog/([a-zA-Z0-9_\.]+)/([a-zA-Z0-9_\.\%]+)$', views.get_cog, name="get_cog"), # noqa re_path(r'^genomes', views.Genomes.as_view(), name=views.Genomes.view_name), re_path(r'^favicon\.ico$', favicon_view), diff --git a/webapp/templates/chlamdb/group_details.html b/webapp/templates/chlamdb/group_details.html new file mode 100644 index 000000000..8d485a3ea --- /dev/null +++ b/webapp/templates/chlamdb/group_details.html @@ -0,0 +1,53 @@ + + + + + +{% include "chlamdb/header.html" %} +{% load custom_tags %} + + + +
    +
    +
    +
    +
    +
    + {% include "chlamdb/menu.html" %} +

    + {{description|safe}} +

    +
    + +
    + +
    +
    +
    + + {% include "chlamdb/result_table.html" with results=genome_table %} + +
    +
    +

    Help

    +
    + +

    + {{genome_table.table_help|safe}} +

    + +

    +
    +
    +
    +
    + +
    +
    +
    +
    + + +{% include "chlamdb/style_menu.html" %} + diff --git a/webapp/views/groups.py b/webapp/views/groups.py index 3d0241112..14418c602 100644 --- a/webapp/views/groups.py +++ b/webapp/views/groups.py @@ -3,7 +3,7 @@ from django.shortcuts import render from django.views import View -from views.mixins import BaseViewMixin +from views.mixins import BaseViewMixin, GenomesTableMixin from views.object_type_metadata import GroupMetadata @@ -17,5 +17,23 @@ def metadata(self): return namedtuple("metadata", "description")(self.overview_description) def get(self, request, *args, **kwargs): - groups = [group[0] for group in self.db.get_groups()] + groups = [self.format_entry(group[0], to_url=True) + for group in self.db.get_groups()] return render(request, self.template, self.get_context(groups=groups)) + + +class GroupDetails(BaseViewMixin, View, GroupMetadata, GenomesTableMixin): + + template = 'chlamdb/group_details.html' + view_name = "group" + genome_source_object = "group" + + @property + def metadata(self): + return namedtuple("metadata", "description")(self.group) + + def get(self, request, group, *args, **kwargs): + self.group = group + taxids = self.db.get_taxids_for_groups([group]) + genome_table = self.get_genomes_table(tuple(taxids)) + return render(request, self.template, self.get_context(genome_table=genome_table)) diff --git a/webapp/views/object_type_metadata.py b/webapp/views/object_type_metadata.py index 8424f21fb..d5a697479 100644 --- a/webapp/views/object_type_metadata.py +++ b/webapp/views/object_type_metadata.py @@ -1,3 +1,5 @@ +from urllib.parse import quote + from views.utils import (format_amr, format_cog, format_ko, format_orthogroup, format_pfam, missing_mandatory, optional2status) @@ -135,6 +137,12 @@ class GroupMetadata(BaseObjectMetadata): "defined in your database.
    Groups can be used for easy selection " "of genomes in various analysis views.") + @staticmethod + def format_entry(entry, to_url=False): + if to_url: + return f"{entry}" + return entry + class MetadataGetter(): diff --git a/webapp/views/utils.py b/webapp/views/utils.py index 651bf05bd..8d59b840e 100644 --- a/webapp/views/utils.py +++ b/webapp/views/utils.py @@ -22,10 +22,11 @@ def safe_replace(string, search_string, replace_string): 'Comparisons: PFAM domains': ['pfam_comparison'], 'Comparisons: orthologous groups': ['orthogroup_comparison'], 'Comparisons: Virulence Factors': ['vf_comparison'], + 'Detail view of a group of genomes': ['group'], 'Genome alignments: Circos plot': ['circos'], 'Genome alignments: Plot region': ['plot_region'], 'Genomes: table of contents': ['extract_contigs'], - 'Overview of defined groups of genomes.':['groups'], + 'Overview of defined groups of genomes.': ['groups'], 'Homology search: Blast': ['blast'], 'Kegg Ortholog': ['fam_ko'], 'Kegg metabolic pathways': ['kegg_genomes'], From b790695cc6f47ff709c1ff5a8d981492b83b1d6a Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Mon, 6 May 2024 12:29:59 +0200 Subject: [PATCH 09/13] Add group add form and view. --- webapp/assets/css/style_FILE_zDB.css | 2 + webapp/chlamdb/forms.py | 48 +++++++++++++++++++ webapp/chlamdb/urls.py | 3 +- webapp/lib/db_utils.py | 4 ++ webapp/templates/chlamdb/group_add.html | 36 ++++++++++++++ webapp/templates/chlamdb/groups_overview.html | 7 +++ webapp/views/groups.py | 40 +++++++++++++++- webapp/views/object_type_metadata.py | 5 +- webapp/views/utils.py | 3 +- 9 files changed, 139 insertions(+), 9 deletions(-) create mode 100644 webapp/templates/chlamdb/group_add.html diff --git a/webapp/assets/css/style_FILE_zDB.css b/webapp/assets/css/style_FILE_zDB.css index fb5fd61f1..9291f06d9 100644 --- a/webapp/assets/css/style_FILE_zDB.css +++ b/webapp/assets/css/style_FILE_zDB.css @@ -1781,3 +1781,5 @@ form.example input[type=text] { .bootstrap-select.form-control.input-group-btn { z-index: auto; } + +a.button_link { text-decoration: none; color: rgb(255, 255, 255);} diff --git a/webapp/chlamdb/forms.py b/webapp/chlamdb/forms.py index fc3c37497..3ff7152e8 100644 --- a/webapp/chlamdb/forms.py +++ b/webapp/chlamdb/forms.py @@ -712,3 +712,51 @@ def clean_entries(self): code="invalid") entries.append(self.Entry(entry_id, label, object_type)) return entries + + +def make_group_add_form(db): + + accession_choices = AccessionFieldHandler().get_choices( + with_plasmids=False) + + class GroupAddForm(forms.Form): + group_name = forms.CharField(max_length=50, + required=True) + genomes = forms.MultipleChoiceField( + choices=accession_choices, + widget=Select2Multiple( + url="autocomplete_taxid", + forward=(forward.Field("genomes", "exclude_taxids_in_groups"),), + attrs={"data-close-on-select": "false", + "data-placeholder": "Nothing selected"}), + required=True) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.db = db + self.helper = FormHelper() + self.helper.form_method = "post" + + self.helper.layout = Layout( + Fieldset( + " ", + Row('group_name'), + Row('genomes', style="margin-top:1em"), + Submit('submit', 'Submit', + style="padding-left:15px; margin-top:15px; margin-bottom:15px "), + css_class="col-lg-5 col-md-6 col-sm-6") + ) + + def clean_group_name(self): + group_name = self.cleaned_data["group_name"] + if self.db.get_group(group_name): + raise ValidationError(f'Group "{group_name}" already exists.') + return group_name + + def get_taxids(self): + indices = self.cleaned_data["genomes"] + taxids, plasmids = AccessionFieldHandler().extract_choices( + indices, False) + return taxids + + return GroupAddForm diff --git a/webapp/chlamdb/urls.py b/webapp/chlamdb/urls.py index d44bc978f..42c5d74a3 100644 --- a/webapp/chlamdb/urls.py +++ b/webapp/chlamdb/urls.py @@ -52,8 +52,9 @@ re_path(r'^gwas_ko/', gwas.KoGwasView.as_view(), name="gwas_ko"), re_path(r'^gwas_cog/', gwas.CogGwasView.as_view(), name="gwas_cog"), re_path(r'^gwas_amr/', gwas.AmrGwasView.as_view(), name="gwas_amr"), + re_path(r'^groups/add/$', groups.GroupAdd.as_view(), name=groups.GroupAdd.view_name), # noqa re_path(r'^groups/([a-zA-Z0-9_\.\(\)\-\'\s]+)', groups.GroupDetails.as_view(), name=groups.GroupDetails.view_name), # noqa - re_path(r'^groups$', groups.GroupsOverview.as_view(), name=groups.GroupsOverview.view_name), # noqa + re_path(r'^groups/$', groups.GroupsOverview.as_view(), name=groups.GroupsOverview.view_name), # noqa re_path(r'^get_cog/([a-zA-Z0-9_\.]+)/([a-zA-Z0-9_\.\%]+)$', views.get_cog, name="get_cog"), # noqa re_path(r'^genomes', views.Genomes.as_view(), name=views.Genomes.view_name), re_path(r'^favicon\.ico$', favicon_view), diff --git a/webapp/lib/db_utils.py b/webapp/lib/db_utils.py index 0293b4a24..427ef3e2e 100644 --- a/webapp/lib/db_utils.py +++ b/webapp/lib/db_utils.py @@ -1485,6 +1485,10 @@ def load_groups(self, groups, group_taxons): self.load_data_into_table("groups", groups) self.load_data_into_table("taxon_in_group", group_taxons) + def get_group(self, group_name): + query = f"SELECT * FROM groups WHERE group_name='{group_name}' LIMIT 1;" + return self.server.adaptor.execute_and_fetchall(query) + def get_groups(self): query = "SELECT * FROM groups;" return self.server.adaptor.execute_and_fetchall(query) diff --git a/webapp/templates/chlamdb/group_add.html b/webapp/templates/chlamdb/group_add.html new file mode 100644 index 000000000..8510f4a6e --- /dev/null +++ b/webapp/templates/chlamdb/group_add.html @@ -0,0 +1,36 @@ + + + + + +{% include "chlamdb/header.html" %} +{% load custom_tags %} + + + +
    +
    +
    +
    +
    +
    + {% include "chlamdb/menu.html" %} +

    + {{description|safe}} +

    + {% include "chlamdb/error.html" %} + {% load crispy_forms_tags %} + {% block content %} + {% csrf_token %} + {% crispy form %} + {% endblock %} +
    +
    +
    +
    +
    +
    + + +{% include "chlamdb/style_menu.html" %} + diff --git a/webapp/templates/chlamdb/groups_overview.html b/webapp/templates/chlamdb/groups_overview.html index 1e6c17348..9b9376441 100644 --- a/webapp/templates/chlamdb/groups_overview.html +++ b/webapp/templates/chlamdb/groups_overview.html @@ -34,6 +34,13 @@

    {{object_name_plural}}

    +
    diff --git a/webapp/views/groups.py b/webapp/views/groups.py index 14418c602..d187615c8 100644 --- a/webapp/views/groups.py +++ b/webapp/views/groups.py @@ -1,6 +1,8 @@ from collections import namedtuple -from django.shortcuts import render +from chlamdb.forms import make_group_add_form +from django.shortcuts import redirect, render +from django.urls import reverse from django.views import View from views.mixins import BaseViewMixin, GenomesTableMixin @@ -25,7 +27,7 @@ def get(self, request, *args, **kwargs): class GroupDetails(BaseViewMixin, View, GroupMetadata, GenomesTableMixin): template = 'chlamdb/group_details.html' - view_name = "group" + view_name = "groups" genome_source_object = "group" @property @@ -37,3 +39,37 @@ def get(self, request, group, *args, **kwargs): taxids = self.db.get_taxids_for_groups([group]) genome_table = self.get_genomes_table(tuple(taxids)) return render(request, self.template, self.get_context(genome_table=genome_table)) + + +class GroupAdd(BaseViewMixin, View, GroupMetadata): + + template = 'chlamdb/group_add.html' + view_name = "groups" + genome_source_object = "group" + description = "Add form" + + @property + def metadata(self): + return namedtuple("metadata", "description")(self.description) + + def dispatch(self, request, *args, **kwargs): + self.form_class = make_group_add_form(self.db) + return super(GroupAdd, self).dispatch(request, *args, **kwargs) + + def get(self, request, *args, **kwargs): + self.form = self.form_class() + return render(request, self.template, self.get_context()) + + def post(self, request, *args, **kwargs): + self.form = self.form_class(request.POST) + if not self.form.is_valid(): + return render(request, self.template, self.get_context()) + + group_name = self.form.cleaned_data["group_name"] + genomes = self.form.get_taxids() + + self.db.load_data_into_table("groups", [(group_name,)]) + self.db.load_data_into_table( + "taxon_in_group", [(group_name, taxid) for taxid in genomes]) + self.db.commit() + return redirect(reverse("groups", args=[group_name])) diff --git a/webapp/views/object_type_metadata.py b/webapp/views/object_type_metadata.py index d5a697479..562ad8fd8 100644 --- a/webapp/views/object_type_metadata.py +++ b/webapp/views/object_type_metadata.py @@ -132,10 +132,7 @@ class GroupMetadata(BaseObjectMetadata): object_type = "group" object_name = "Genome Group" - overview_description = ( - "This page gives an overview of the groups of genomes that were " - "defined in your database.
    Groups can be used for easy selection " - "of genomes in various analysis views.") + overview_description = "Overview of defined groups of genomes." @staticmethod def format_entry(entry, to_url=False): diff --git a/webapp/views/utils.py b/webapp/views/utils.py index 8d59b840e..445e4376a 100644 --- a/webapp/views/utils.py +++ b/webapp/views/utils.py @@ -22,11 +22,10 @@ def safe_replace(string, search_string, replace_string): 'Comparisons: PFAM domains': ['pfam_comparison'], 'Comparisons: orthologous groups': ['orthogroup_comparison'], 'Comparisons: Virulence Factors': ['vf_comparison'], - 'Detail view of a group of genomes': ['group'], 'Genome alignments: Circos plot': ['circos'], 'Genome alignments: Plot region': ['plot_region'], 'Genomes: table of contents': ['extract_contigs'], - 'Overview of defined groups of genomes.': ['groups'], + 'Genome groups': ['groups'], 'Homology search: Blast': ['blast'], 'Kegg Ortholog': ['fam_ko'], 'Kegg metabolic pathways': ['kegg_genomes'], From ba2fb7c956e7f547a0136f522ec68d5b9d379af3 Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Mon, 6 May 2024 15:32:12 +0200 Subject: [PATCH 10/13] Add delete group view. --- webapp/chlamdb/urls.py | 1 + webapp/lib/db_utils.py | 8 ++++++ webapp/templates/chlamdb/group_details.html | 27 +++++++++++++++++++++ webapp/views/groups.py | 13 +++++++++- 4 files changed, 48 insertions(+), 1 deletion(-) diff --git a/webapp/chlamdb/urls.py b/webapp/chlamdb/urls.py index 42c5d74a3..2237b5048 100644 --- a/webapp/chlamdb/urls.py +++ b/webapp/chlamdb/urls.py @@ -53,6 +53,7 @@ re_path(r'^gwas_cog/', gwas.CogGwasView.as_view(), name="gwas_cog"), re_path(r'^gwas_amr/', gwas.AmrGwasView.as_view(), name="gwas_amr"), re_path(r'^groups/add/$', groups.GroupAdd.as_view(), name=groups.GroupAdd.view_name), # noqa + re_path(r'^groups/([a-zA-Z0-9_\.\(\)\-\'\s]+)/delete/$', groups.GroupDelete.as_view(), name=groups.GroupDelete.view_name), # noqa re_path(r'^groups/([a-zA-Z0-9_\.\(\)\-\'\s]+)', groups.GroupDetails.as_view(), name=groups.GroupDetails.view_name), # noqa re_path(r'^groups/$', groups.GroupsOverview.as_view(), name=groups.GroupsOverview.view_name), # noqa re_path(r'^get_cog/([a-zA-Z0-9_\.]+)/([a-zA-Z0-9_\.\%]+)$', views.get_cog, name="get_cog"), # noqa diff --git a/webapp/lib/db_utils.py b/webapp/lib/db_utils.py index 427ef3e2e..fe9415b94 100644 --- a/webapp/lib/db_utils.py +++ b/webapp/lib/db_utils.py @@ -1485,6 +1485,14 @@ def load_groups(self, groups, group_taxons): self.load_data_into_table("groups", groups) self.load_data_into_table("taxon_in_group", group_taxons) + def delete_group(self, group_name): + sql = f"DELETE FROM taxon_in_group WHERE group_name='{group_name}';" + self.server.adaptor.execute(sql) + sql = f"DELETE FROM groups WHERE group_name='{group_name}';" + self.server.adaptor.execute(sql) + self.commit() + return + def get_group(self, group_name): query = f"SELECT * FROM groups WHERE group_name='{group_name}' LIMIT 1;" return self.server.adaptor.execute_and_fetchall(query) diff --git a/webapp/templates/chlamdb/group_details.html b/webapp/templates/chlamdb/group_details.html index 8d485a3ea..a6b699f82 100644 --- a/webapp/templates/chlamdb/group_details.html +++ b/webapp/templates/chlamdb/group_details.html @@ -42,7 +42,34 @@

    Help

    +
    + + + + +
    diff --git a/webapp/views/groups.py b/webapp/views/groups.py index d187615c8..03509cf65 100644 --- a/webapp/views/groups.py +++ b/webapp/views/groups.py @@ -38,7 +38,18 @@ def get(self, request, group, *args, **kwargs): self.group = group taxids = self.db.get_taxids_for_groups([group]) genome_table = self.get_genomes_table(tuple(taxids)) - return render(request, self.template, self.get_context(genome_table=genome_table)) + return render(request, self.template, self.get_context(group=self.group, genome_table=genome_table)) + + +class GroupDelete(BaseViewMixin, View, GroupMetadata, GenomesTableMixin): + + view_name = "groups_delete" + genome_source_object = "group" + + def post(self, request, group, *args, **kwargs): + self.group = group + self.db.delete_group(group) + return redirect(reverse("groups")) class GroupAdd(BaseViewMixin, View, GroupMetadata): From df1b0035ff1d08ffdb474898a762c03c277941e1 Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Tue, 7 May 2024 11:40:17 +0200 Subject: [PATCH 11/13] Update existing tests and add new tests for group views. --- testing/webapp/test_groups.py | 53 +++++++++++++++++++++ testing/webapp/test_views.py | 4 ++ webapp/templates/chlamdb/group_details.html | 1 - 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 testing/webapp/test_groups.py diff --git a/testing/webapp/test_groups.py b/testing/webapp/test_groups.py new file mode 100644 index 000000000..717177bdf --- /dev/null +++ b/testing/webapp/test_groups.py @@ -0,0 +1,53 @@ +from django.conf import settings +from django.test import SimpleTestCase +from lib.db_utils import DB + + +class TestGroupsViews(SimpleTestCase): + + def test_groups_overview(self): + resp = self.client.get("/groups/") + self.assertEqual(200, resp.status_code) + self.assertTemplateUsed(resp, 'chlamdb/groups_overview.html') + + groups = [ + 'positive', + 'negative', + 'all'] + self.assertEqual(groups, resp.context["groups"]) + for group in groups: + self.assertContains(resp, group) + + self.assertContains(resp, 'Add new group') + + def test_add_and_delete_group_view(self): + resp = self.client.get("/groups/add/") + self.assertEqual(200, resp.status_code) + self.assertTemplateUsed(resp, 'chlamdb/group_add.html') + self.assertEqual(list(resp.context.get("form").fields.keys()), + ['group_name', 'genomes']) + + db = DB.load_db_from_name(settings.BIODB_DB_PATH) + data = {"group_name": "test_group", "genomes": ["group:negative", "1"]} + resp = self.client.post("/groups/add/", data=data) + self.assertEqual(302, resp.status_code) + self.assertEqual('/groups/test_group', resp.url) + self.assertTrue(db.get_group("test_group")) + + resp = self.client.post("/groups/test_group/delete/") + self.assertEqual(302, resp.status_code) + self.assertEqual('/groups/', resp.url) + self.assertFalse(db.get_group("test_group")) + + def test_group_details_view(self): + resp = self.client.get("/groups/positive") + self.assertEqual(200, resp.status_code) + self.assertTemplateUsed(resp, 'chlamdb/group_details.html') + genomes = [ + 'Klebsiella pneumoniae R6724_16313', + 'Klebsiella pneumoniae R6726_16314'] + self.assertEqual( + [row.accession for i, row in resp.context["genome_table"]["table_data"].iterrows()], + genomes) + for genome in genomes: + self.assertContains(resp, genome, html=True) diff --git a/testing/webapp/test_views.py b/testing/webapp/test_views.py index 3a8da3cd6..7fba11b5e 100644 --- a/testing/webapp/test_views.py +++ b/testing/webapp/test_views.py @@ -13,6 +13,7 @@ untested_patterns = { '^favicon\\.ico$', + r'^groups/([a-zA-Z0-9_\.\(\)\-\'\s]+)/delete/$', '^module_cat_info/([a-zA-Z0-9_\\.]+)/([a-zA-Z0-9_\\.\\+-]+)$', '^plot_region/$', '^robots.txt$', @@ -55,6 +56,9 @@ '/FAQ', '/genomes', '/get_cog/3/L?h=1&h=2&h=3', + '/groups/', + '/groups/add/', + '/groups/positive', '/gwas_amr/', '/gwas_cog/', '/gwas_ko/', diff --git a/webapp/templates/chlamdb/group_details.html b/webapp/templates/chlamdb/group_details.html index a6b699f82..37490cca8 100644 --- a/webapp/templates/chlamdb/group_details.html +++ b/webapp/templates/chlamdb/group_details.html @@ -37,7 +37,6 @@

    Help

    {{genome_table.table_help|safe}}

    -

    From 642ae448bcffe618fbd1bc05d101cdac990cf00b Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Tue, 7 May 2024 11:59:35 +0200 Subject: [PATCH 12/13] Fix bug in autocomplete view. --- webapp/views/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/webapp/views/utils.py b/webapp/views/utils.py index 445e4376a..258923e81 100644 --- a/webapp/views/utils.py +++ b/webapp/views/utils.py @@ -514,12 +514,13 @@ def get_choices(self, with_plasmids=True, with_groups=True, accession_choices.extend([(self.group_id_to_key(group[0]), group[0]) for group in self.db.get_groups()]) + exclude = set(exclude) # We also exclude taxids contained in the excluded groups groups_to_exclude = [self.group_key_to_id(key) for key in exclude if self.is_group(key)] if groups_to_exclude: in_groups = self.db.get_taxids_for_groups(groups_to_exclude) - exclude = set(exclude).union({str(el) for el in in_groups}) + exclude = exclude.union({str(el) for el in in_groups}) # And we exclude groups containing an excluded taxid taxids_to_exclude = list(filter(self.is_taxid, exclude)) @@ -534,7 +535,7 @@ def get_choices(self, with_plasmids=True, with_groups=True, if self.is_group(key)] if groups_to_exclude: in_groups = self.db.get_taxids_for_groups(groups_to_exclude) - exclude = set(exclude).union({str(el) for el in in_groups}) + exclude = exclude.union({str(el) for el in in_groups}) accession_choices = filter(lambda choice: choice[0] not in exclude, accession_choices) From 7a649b3c7e8a9e13db83e8170f4773d6d8056892 Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Tue, 7 May 2024 12:01:17 +0200 Subject: [PATCH 13/13] Add changelog entry. --- docs/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 16e86757b..e6f8a8157 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -25,6 +25,7 @@ to [Common Changelog](https://common-changelog.org) ### Added +- Add views to add, delete, and display groups. ([#86](https://github.com/metagenlab/zDB/pull/86)) (Niklaus Johner) - Allow defining groups of genomes in input file. ([#82](https://github.com/metagenlab/zDB/pull/82)) (Niklaus Johner) - Add view to produce custom plots (phylogenetic trees and table). ([#78](https://github.com/metagenlab/zDB/pull/78)) (Niklaus Johner) - Add AMRs and VFs to search index. ([#73](https://github.com/metagenlab/zDB/pull/73)) (Niklaus Johner)