From 3a532523ede38e5d29025ca601d92110bc826a4a Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Mon, 29 Apr 2024 15:16:16 +0200 Subject: [PATCH 01/17] Fix AMR and VF hit counts when including taxids. --- webapp/lib/db_utils.py | 2 +- webapp/lib/queries.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/webapp/lib/db_utils.py b/webapp/lib/db_utils.py index d890f1395..535108880 100644 --- a/webapp/lib/db_utils.py +++ b/webapp/lib/db_utils.py @@ -2538,7 +2538,7 @@ def get_amr_hit_counts(self, ids, indexing="taxid", search_on="taxid", ) plasmid_join = ( "INNER JOIN bioentry_qualifier_value AS is_plasmid ON " - " is_plasmid.bioentry_id=entry.bioentry_id " + " is_plasmid.bioentry_id=bioentry.bioentry_id " "INNER JOIN term AS plasmid_term ON plasmid_term.term_id=is_plasmid.term_id " " AND plasmid_term.name=\"plasmid\"" ) diff --git a/webapp/lib/queries.py b/webapp/lib/queries.py index d6df5d30f..d274fae3e 100644 --- a/webapp/lib/queries.py +++ b/webapp/lib/queries.py @@ -76,7 +76,7 @@ def get_hit_counts(self, ids, indexing="taxid", search_on="taxid", ) plasmid_join = ( "INNER JOIN bioentry_qualifier_value AS is_plasmid ON " - " is_plasmid.bioentry_id=entry.bioentry_id " + " is_plasmid.bioentry_id=bioentry.bioentry_id " "INNER JOIN term AS plasmid_term ON plasmid_term.term_id=is_plasmid.term_id " " AND plasmid_term.name=\"plasmid\"" ) @@ -93,7 +93,7 @@ def get_hit_counts(self, ids, indexing="taxid", search_on="taxid", ) all_ids = ids - if plasmids is not None: + if plasmids: all_ids += plasmids results = self.server.adaptor.execute_and_fetchall(query, all_ids) From 2053181bcaf958ae69bc7d2c732a46c47186f18f Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Mon, 29 Apr 2024 15:17:41 +0200 Subject: [PATCH 02/17] Add dependency to django autocomplete light. --- conda/webapp.yaml | 1 + webapp/settings/settings.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/conda/webapp.yaml b/conda/webapp.yaml index e2cf13ffd..6710b1000 100644 --- a/conda/webapp.yaml +++ b/conda/webapp.yaml @@ -22,5 +22,6 @@ dependencies: - bibtexparser - blast>=2.9.0 - gxx + - django-autocomplete-light - pip: - scoary-2 diff --git a/webapp/settings/settings.py b/webapp/settings/settings.py index 14edcd79f..00ab48bcf 100755 --- a/webapp/settings/settings.py +++ b/webapp/settings/settings.py @@ -41,6 +41,8 @@ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', + 'dal', + 'dal_select2', 'chlamdb', 'gunicorn', 'templatetags', From 9178f5d0722fba8f8577b84d8b3853dc6d27b08a Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Mon, 29 Apr 2024 15:43:42 +0200 Subject: [PATCH 03/17] Use dal for accession choices and n_excluded in hit extraction form. --- webapp/chlamdb/forms.py | 75 +++++++++---------- webapp/chlamdb/urls.py | 8 +- webapp/templates/chlamdb/extract_hits.html | 1 - .../chlamdb/show_hide_plasmid_accessions.html | 42 ----------- webapp/views/autocomplete.py | 20 +++++ webapp/views/utils.py | 47 ++++++++++++ 6 files changed, 107 insertions(+), 86 deletions(-) delete mode 100644 webapp/templates/chlamdb/show_hide_plasmid_accessions.html create mode 100644 webapp/views/autocomplete.py diff --git a/webapp/chlamdb/forms.py b/webapp/chlamdb/forms.py index fdd0d8d1e..a240fe92e 100644 --- a/webapp/chlamdb/forms.py +++ b/webapp/chlamdb/forms.py @@ -7,10 +7,12 @@ from Bio.SeqRecord import SeqRecord from crispy_forms.helper import FormHelper from crispy_forms.layout import Column, Fieldset, Layout, Row, Submit +from dal import forward +from dal.autocomplete import ListSelect2, Select2Multiple from django import forms from django.core.exceptions import ValidationError from django.core.validators import MaxLengthValidator, MinLengthValidator -from views.utils import EntryIdParser +from views.utils import AccessionFieldHandler, EntryIdParser def get_accessions(db, all=False, plasmid=False): @@ -307,7 +309,8 @@ def get_ref_taxid(self): def make_extract_form(db, action, plasmid=False, label="Orthologs"): - accession_choices, rev_index = get_accessions(db, plasmid=plasmid) + + accession_choices = AccessionFieldHandler().get_choices() class ExtractForm(forms.Form): checkbox_accessions = forms.BooleanField( @@ -321,52 +324,40 @@ class ExtractForm(forms.Form): orthologs_in = forms.MultipleChoiceField( label=f"{label} conserved in", choices=accession_choices, - widget=forms.SelectMultiple(attrs={'size': '%s' % "17", - "class": "selectpicker", - "data-live-search": "true"}), + widget=Select2Multiple( + url="autocomplete_taxid", + forward=(forward.Field("no_orthologs_in", "to_exclude"), + forward.Field("checkbox_accessions", "include_plasmids")), + attrs={"data-close-on-select": "false", + "data-placeholder": "Nothing selected"}), required=True) no_orthologs_in = forms.MultipleChoiceField( label="%s absent from (optional)" % label, choices=accession_choices, - widget=forms.SelectMultiple(attrs={'size': '%s' % "17", - "class": "selectpicker remove-example", - "data-live-search": "true"}), + widget=Select2Multiple( + url="autocomplete_taxid", + forward=(forward.Field("orthologs_in", "to_exclude"), + forward.Field("checkbox_accessions", "include_plasmids")), + attrs={"data-close-on-select": "false", + "data-placeholder": "Nothing selected"}), required=False) - new_choices = [['None', 'None']] + accession_choices - - frequency_choices = ((i, i) for i in range(len(accession_choices))) - frequency = forms.ChoiceField( - choices=frequency_choices, + _n_missing = forms.ChoiceField( label='Missing data (optional)', + choices=zip(range(len(accession_choices)), + range(len(accession_choices))), + widget=ListSelect2( + url="autocomplete_n_missing", + forward=(forward.Field("orthologs_in", "included"),)), required=False) - def extract_choices(self, indices): - keep_plasmids = self.cleaned_data["checkbox_accessions"] - taxids = [] - plasmids = None - if keep_plasmids: - plasmids = [] - - for index in indices: - taxid, is_plasmid = rev_index[index] - if keep_plasmids and is_plasmid: - plasmids.append(taxid) - elif is_plasmid: - continue - else: - taxids.append(taxid) - return taxids, plasmids + def extract_choices(self, indices, include_plasmids): + return AccessionFieldHandler().extract_choices( + indices, include_plasmids) def get_n_missing(self): - return int(self.cleaned_data["frequency"]) - - def get_include_choices(self): - return self.extract_choices((int(i) for i in self.cleaned_data["orthologs_in"])) - - def get_exclude_choices(self): - return self.extract_choices((int(i) for i in self.cleaned_data["no_orthologs_in"])) + return int(self.cleaned_data.get("_n_missing") or 0) def __init__(self, *args, **kwargs): self.helper = FormHelper() @@ -386,7 +377,7 @@ def __init__(self, *args, **kwargs): css_class='form-group col-lg-6 col-md-6 col-sm-12') ), Column( - Row('frequency'), + Row('_n_missing'), Submit('submit', 'Compare %s' % label, style="margin-top:15px"), @@ -401,8 +392,12 @@ def __init__(self, *args, **kwargs): def clean(self): cleaned_data = super(ExtractForm, self).clean() - self.included_taxids, self.included_plasmids = self.get_include_choices() - self.excluded_taxids, self.excluded_plasmids = self.get_exclude_choices() + self.included_taxids, self.included_plasmids = self.extract_choices( + self.cleaned_data["orthologs_in"], + self.cleaned_data["checkbox_accessions"]) + self.excluded_taxids, self.excluded_plasmids = self.extract_choices( + self.cleaned_data["no_orthologs_in"], + self.cleaned_data["checkbox_accessions"]) self.n_missing = self.get_n_missing() self.n_included = len(self.included_taxids) if self.included_plasmids is not None: @@ -411,7 +406,7 @@ def clean(self): err = ValidationError( "This must be smaller than the number of included genomes.", code="invalid") - self.add_error("frequency", err) + self.add_error("_n_missing", err) return cleaned_data return ExtractForm diff --git a/webapp/chlamdb/urls.py b/webapp/chlamdb/urls.py index 0ea75808d..31f9f4c52 100644 --- a/webapp/chlamdb/urls.py +++ b/webapp/chlamdb/urls.py @@ -2,8 +2,8 @@ from django.urls import re_path from django.views.generic import TemplateView from django.views.generic.base import RedirectView -from views import (custom_plots, entry_lists, fam, gwas, hits_extraction, - locus, tabular_comparison, venn, views) +from views import (autocomplete, custom_plots, entry_lists, fam, gwas, + hits_extraction, locus, tabular_comparison, venn, views) favicon_view = RedirectView.as_view(url='/assets/favicon.ico', permanent=True) @@ -73,7 +73,7 @@ re_path(r'^entry_list_ko$', entry_lists.KoEntryListView.as_view(), name="entry_list_ko"), # noqa re_path(r'^entry_list_cog$', entry_lists.CogEntryListView.as_view(), name="entry_list_cog"), # noqa re_path(r'^entry_list_amr$', entry_lists.AmrEntryListView.as_view(), name="entry_list_amr"), # noqa - re_path(r'^custom_plots/$', custom_plots.CusomPlotsView.as_view(), name="custom_plots"), + re_path(r'^custom_plots/$', custom_plots.CusomPlotsView.as_view(), name="custom_plots"), # noqa re_path(r'^cog_venn_subset/([A-Z])$', venn.VennCogSubsetView.as_view(), name="cog_venn_subset"), # noqa re_path(r'^cog_phylo_heatmap/([a-zA-Z0-9_\-]+)', views.CogPhyloHeatmap.as_view(), name="cog_phylo_heatmap"), # noqa re_path(r'^cog_comparison', tabular_comparison.CogComparisonView.as_view(), name="cog_comparison"), # noqa @@ -81,6 +81,8 @@ re_path(r'^circos_main/$', views.circos_main, name="circos_main"), re_path(r'^circos/$', views.circos, name="circos"), re_path(r'^blast/$', views.blast, name="blast"), + re_path(r'^autocomplete_taxid/$', autocomplete.AutocompleteTaxid.as_view(), name="autocomplete_taxid"), # noqa + re_path(r'^autocomplete_n_missing/$', autocomplete.AutocompleteNMissing.as_view(), name="autocomplete_n_missing"), # noqa re_path(r'^amr_comparison', tabular_comparison.AmrComparisonView.as_view(), name="amr_comparison"), # noqa re_path(r'^about$', views.about, name="about"), re_path(r'^.*$', views.home, name="home"), diff --git a/webapp/templates/chlamdb/extract_hits.html b/webapp/templates/chlamdb/extract_hits.html index 695562ed6..e7afeff90 100644 --- a/webapp/templates/chlamdb/extract_hits.html +++ b/webapp/templates/chlamdb/extract_hits.html @@ -97,7 +97,6 @@
Show the comparison on circular map
{% include "chlamdb/style_menu.html" %} -{% include "chlamdb/show_hide_plasmid_accessions.html" %} diff --git a/webapp/templates/chlamdb/show_hide_plasmid_accessions.html b/webapp/templates/chlamdb/show_hide_plasmid_accessions.html deleted file mode 100644 index 76adf396e..000000000 --- a/webapp/templates/chlamdb/show_hide_plasmid_accessions.html +++ /dev/null @@ -1,42 +0,0 @@ - diff --git a/webapp/views/autocomplete.py b/webapp/views/autocomplete.py new file mode 100644 index 000000000..85a9bd7e5 --- /dev/null +++ b/webapp/views/autocomplete.py @@ -0,0 +1,20 @@ +from dal.autocomplete import Select2ListView + +from views.utils import AccessionFieldHandler + + +class AutocompleteTaxid(Select2ListView): + + def get_list(self): + with_plasmids = self.forwarded["include_plasmids"] + to_exclude = self.forwarded["to_exclude"] + return AccessionFieldHandler().get_choices(with_plasmids=with_plasmids, + to_exclude=to_exclude) + + +class AutocompleteNMissing(Select2ListView): + + def get_list(self): + n_max = len(self.forwarded["included"]) + choices = [(i, i) for i in range(n_max)] + return choices diff --git a/webapp/views/utils.py b/webapp/views/utils.py index e05ac231d..bc007d9eb 100644 --- a/webapp/views/utils.py +++ b/webapp/views/utils.py @@ -453,3 +453,50 @@ def make_div(figure_or_data, include_plotlyjs=False, show_link=False, except IndexError: pass return div + + +class AccessionFieldHandler(): + + plasmid_prefix = "plasmid:" + _db = None + + @classmethod + def is_plasmid(cls, key): + return key.startswith(cls.plasmid_prefix) + + def plasmid_key_to_id(self, key): + return int(key.lstrip(self.plasmid_prefix)) + + def plasmid_id_to_key(self, identifier): + return f"{self.plasmid_prefix}{identifier}" + + @property + def db(self): + if self._db is None: + biodb_path = settings.BIODB_DB_PATH + self._db = DB.load_db_from_name(biodb_path) + return self._db + + def get_choices(self, with_plasmids=True, to_exclude=[]): + result = self.db.get_genomes_description() + result.set_index(result.index.astype(str), inplace=True) + accession_choices = [] + for taxid, data in result.iterrows(): + if taxid not in to_exclude: + accession_choices.append((taxid, data.description)) + if with_plasmids and data.has_plasmid: + # Distinguish plasmids from taxons + plasmid = self.plasmid_id_to_key(taxid) + if plasmid not in to_exclude: + accession_choices.append((plasmid, + f"{data.description} plasmid")) + return accession_choices + + def extract_choices(self, indices, include_plasmids): + if include_plasmids: + plasmids = [self.plasmid_key_to_id(key) + for key in indices if self.is_plasmid(key)] + else: + plasmids = None + taxids = [int(key) for key in indices if not self.is_plasmid(key)] + return taxids, plasmids From 166056dab796d8babf0cf6533c011b57215532c9 Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Mon, 29 Apr 2024 15:52:56 +0200 Subject: [PATCH 04/17] Remove unused lst_plasmids parameter from get_genomes_description. --- webapp/chlamdb/forms.py | 2 +- webapp/lib/db_utils.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/webapp/chlamdb/forms.py b/webapp/chlamdb/forms.py index a240fe92e..be264c79d 100644 --- a/webapp/chlamdb/forms.py +++ b/webapp/chlamdb/forms.py @@ -16,7 +16,7 @@ def get_accessions(db, all=False, plasmid=False): - result = db.get_genomes_description(lst_plasmids=plasmid) + result = db.get_genomes_description() accession_choices = [] index = 0 reverse_index = [] diff --git a/webapp/lib/db_utils.py b/webapp/lib/db_utils.py index 535108880..bb07ec958 100644 --- a/webapp/lib/db_utils.py +++ b/webapp/lib/db_utils.py @@ -1270,12 +1270,11 @@ def get_term_id(self, term, create_if_absent=False, ontology="Annotation Tags"): gc_term_id = result[0][0] return gc_term_id - def get_genomes_description(self, lst_plasmids=True): + def get_genomes_description(self): """ Returns the description of the genome as it has been read from the genbank files, indexed by taxon_id. The output also contains a flag - has_plasmid indicating whether the genome contains a plasmid or not, - if the lst_plasmid flag has been set. + has_plasmid indicating whether the genome contains a plasmid or not. """ has_plasmid_query = ( From a366de34b15d8418b4c76f11af8b1812c6b883b5 Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Wed, 1 May 2024 10:55:00 +0200 Subject: [PATCH 05/17] Add tests for AccessionFieldHandler. --- testing/webapp/test_utils.py | 86 ++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 testing/webapp/test_utils.py diff --git a/testing/webapp/test_utils.py b/testing/webapp/test_utils.py new file mode 100644 index 000000000..3fd2bea19 --- /dev/null +++ b/testing/webapp/test_utils.py @@ -0,0 +1,86 @@ +from django.test import SimpleTestCase + +from webapp.views.utils import AccessionFieldHandler + + +class TestAccessionFieldHandler(SimpleTestCase): + + taxons = [('1', 'Klebsiella pneumoniae R6724_16313'), + ('2', 'Klebsiella pneumoniae R6726_16314'), + ('3', 'Klebsiella pneumoniae R6728_16315')] + + plasmids = [('plasmid:1', 'Klebsiella pneumoniae R6724_16313 plasmid'), + ('plasmid:2', 'Klebsiella pneumoniae R6726_16314 plasmid'), + ('plasmid:3', 'Klebsiella pneumoniae R6728_16315 plasmid')] + + def setUp(self): + self.handler = AccessionFieldHandler() + # Because we will not commit, we need to make all modification + # on the handler's database + self.db = self.handler.db + + def tearDown(self): + self.db.server.close() + + def add_plasmid_for_taxids(self, taxids): + plasmid_term_id = self.db.server.adaptor.execute_one( + "SELECT term_id FROM term WHERE name='plasmid'")[0] + for taxid in taxids: + self.db.server.adaptor.execute( + f"UPDATE bioentry_qualifier_value SET value=1 " + f"WHERE bioentry_id={taxid} AND term_id={plasmid_term_id};") + + def assertItemsEqual(self, expected, choices): + self.assertEqual(sorted(expected), sorted(choices)) + + def test_get_choices_handles_plasmids(self): + self.assertItemsEqual(self.taxons, self.handler.get_choices()) + + self.add_plasmid_for_taxids([1, 3]) + self.assertItemsEqual(self.taxons + self.plasmids[::2], + self.handler.get_choices()) + + self.assertItemsEqual(self.taxons, + self.handler.get_choices(with_plasmids=False)) + + def test_get_choices_handles_exclusion(self): + to_exclude = [el[0] for el in self.taxons[1:]] + self.assertItemsEqual(self.taxons[:1], + self.handler.get_choices(to_exclude=to_exclude)) + + self.add_plasmid_for_taxids([1, 2, 3]) + self.assertItemsEqual(self.taxons + self.plasmids, + self.handler.get_choices()) + + to_exclude = [self.taxons[-1][0]] + self.assertItemsEqual(self.taxons[:-1] + self.plasmids, + self.handler.get_choices(to_exclude=to_exclude)) + self.assertItemsEqual(self.taxons[:-1], + self.handler.get_choices(to_exclude=to_exclude, + with_plasmids=False)) + + to_exclude = [self.plasmids[-1][0]] + self.assertItemsEqual(self.taxons + self.plasmids[:-1], + self.handler.get_choices(to_exclude=to_exclude)) + + to_exclude = [el[0] for el in self.plasmids] + self.assertItemsEqual(self.taxons, + self.handler.get_choices(to_exclude=to_exclude)) + + def test_extract_choices_returns_none_when_include_plasmids_is_false(self): + self.assertEqual( + ([1, 3], None), + self.handler.extract_choices(["1", "3"], False)) + + self.assertEqual( + ([1, 3], []), + self.handler.extract_choices(["1", "3"], True)) + + def test_extract_choices_handles_plasmids(self): + self.assertEqual( + ([2], [1, 3]), + self.handler.extract_choices(["plasmid:1", "2", "plasmid:3"], True)) + + self.assertEqual( + ([2], None), + self.handler.extract_choices(["plasmid:1", "2", "plasmid:3"], False)) From fb3246176a51179f92e5f5a05a8def24afd3c949 Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Wed, 1 May 2024 11:18:53 +0200 Subject: [PATCH 06/17] Better separate filtering stage in AccessionFieldHandler.get_choices. --- webapp/views/utils.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/webapp/views/utils.py b/webapp/views/utils.py index bc007d9eb..b8caaf3bc 100644 --- a/webapp/views/utils.py +++ b/webapp/views/utils.py @@ -482,15 +482,16 @@ def get_choices(self, with_plasmids=True, to_exclude=[]): result.set_index(result.index.astype(str), inplace=True) accession_choices = [] for taxid, data in result.iterrows(): - if taxid not in to_exclude: - accession_choices.append((taxid, data.description)) + accession_choices.append((taxid, data.description)) if with_plasmids and data.has_plasmid: # Distinguish plasmids from taxons plasmid = self.plasmid_id_to_key(taxid) - if plasmid not in to_exclude: - accession_choices.append((plasmid, - f"{data.description} plasmid")) - return accession_choices + accession_choices.append((plasmid, + f"{data.description} plasmid")) + + accession_choices = filter(lambda choice: choice[0] not in to_exclude, + accession_choices) + return tuple(accession_choices) def extract_choices(self, indices, include_plasmids): if include_plasmids: From 83b5be5f51f8ba59e3025409ad1c27cc282ddde4 Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Wed, 1 May 2024 12:02:21 +0200 Subject: [PATCH 07/17] Handle groups in AccessionFieldHandler. --- testing/webapp/test_utils.py | 59 ++++++++++++++++++++++++++++-------- webapp/views/utils.py | 38 ++++++++++++++++++----- 2 files changed, 78 insertions(+), 19 deletions(-) diff --git a/testing/webapp/test_utils.py b/testing/webapp/test_utils.py index 3fd2bea19..8eea4bfc5 100644 --- a/testing/webapp/test_utils.py +++ b/testing/webapp/test_utils.py @@ -13,6 +13,10 @@ class TestAccessionFieldHandler(SimpleTestCase): ('plasmid:2', 'Klebsiella pneumoniae R6726_16314 plasmid'), ('plasmid:3', 'Klebsiella pneumoniae R6728_16315 plasmid')] + groups = [('group:all', 'all'), + ('group:negative', 'negative'), + ('group:positive', 'positive')] + def setUp(self): self.handler = AccessionFieldHandler() # Because we will not commit, we need to make all modification @@ -34,37 +38,59 @@ def assertItemsEqual(self, expected, choices): self.assertEqual(sorted(expected), sorted(choices)) def test_get_choices_handles_plasmids(self): - self.assertItemsEqual(self.taxons, self.handler.get_choices()) + self.assertItemsEqual( + self.taxons, + self.handler.get_choices(with_groups=False)) self.add_plasmid_for_taxids([1, 3]) - self.assertItemsEqual(self.taxons + self.plasmids[::2], - self.handler.get_choices()) + self.assertItemsEqual( + self.taxons + self.plasmids[::2], + self.handler.get_choices(with_groups=False)) + + self.assertItemsEqual( + self.taxons, + self.handler.get_choices(with_groups=False, with_plasmids=False)) - self.assertItemsEqual(self.taxons, - self.handler.get_choices(with_plasmids=False)) + def test_get_choices_handles_groups(self): + self.assertItemsEqual( + self.taxons, + self.handler.get_choices(with_groups=False)) + + self.assertItemsEqual( + self.taxons + self.groups, + self.handler.get_choices(with_groups=True)) def test_get_choices_handles_exclusion(self): to_exclude = [el[0] for el in self.taxons[1:]] - self.assertItemsEqual(self.taxons[:1], - self.handler.get_choices(to_exclude=to_exclude)) + self.assertItemsEqual( + self.taxons[:1], + self.handler.get_choices(with_groups=False, to_exclude=to_exclude)) + + self.assertItemsEqual( + self.taxons[:1] + self.groups, + self.handler.get_choices(to_exclude=to_exclude)) self.add_plasmid_for_taxids([1, 2, 3]) - self.assertItemsEqual(self.taxons + self.plasmids, + self.assertItemsEqual(self.taxons + self.groups + self.plasmids, self.handler.get_choices()) to_exclude = [self.taxons[-1][0]] - self.assertItemsEqual(self.taxons[:-1] + self.plasmids, + self.assertItemsEqual(self.taxons[:-1] + self.groups + self.plasmids, self.handler.get_choices(to_exclude=to_exclude)) - self.assertItemsEqual(self.taxons[:-1], + self.assertItemsEqual(self.taxons[:-1] + self.groups, self.handler.get_choices(to_exclude=to_exclude, with_plasmids=False)) to_exclude = [self.plasmids[-1][0]] - self.assertItemsEqual(self.taxons + self.plasmids[:-1], + self.assertItemsEqual(self.taxons + self.groups + self.plasmids[:-1], self.handler.get_choices(to_exclude=to_exclude)) to_exclude = [el[0] for el in self.plasmids] - self.assertItemsEqual(self.taxons, + self.assertItemsEqual(self.taxons + self.groups, + self.handler.get_choices(to_exclude=to_exclude)) + + to_exclude = [el[0] for el in self.groups[1:]] + self.assertItemsEqual(self.taxons + self.groups[0:1] + self.plasmids, self.handler.get_choices(to_exclude=to_exclude)) def test_extract_choices_returns_none_when_include_plasmids_is_false(self): @@ -84,3 +110,12 @@ def test_extract_choices_handles_plasmids(self): self.assertEqual( ([2], None), self.handler.extract_choices(["plasmid:1", "2", "plasmid:3"], False)) + + def test_extract_choices_handles_groups(self): + self.assertEqual( + ([3], []), + self.handler.extract_choices(["group:negative"], True)) + + self.assertEqual( + ([2, 3], [1]), + self.handler.extract_choices(["plasmid:1", "2", "group:negative"], True)) diff --git a/webapp/views/utils.py b/webapp/views/utils.py index b8caaf3bc..e4b9781c1 100644 --- a/webapp/views/utils.py +++ b/webapp/views/utils.py @@ -458,6 +458,7 @@ def make_div(figure_or_data, include_plotlyjs=False, show_link=False, class AccessionFieldHandler(): plasmid_prefix = "plasmid:" + group_prefix = "group:" _db = None @classmethod @@ -470,6 +471,16 @@ def plasmid_key_to_id(self, key): def plasmid_id_to_key(self, identifier): return f"{self.plasmid_prefix}{identifier}" + @classmethod + def is_group(cls, key): + return key.startswith(cls.group_prefix) + + def group_key_to_id(self, key): + return key.rsplit(self.group_prefix, 1)[-1] + + def group_id_to_key(self, identifier): + return f"{self.group_prefix}{identifier}" + @property def db(self): if self._db is None: @@ -477,7 +488,7 @@ def db(self): self._db = DB.load_db_from_name(biodb_path) return self._db - def get_choices(self, with_plasmids=True, to_exclude=[]): + def get_choices(self, with_plasmids=True, with_groups=True, to_exclude=[]): result = self.db.get_genomes_description() result.set_index(result.index.astype(str), inplace=True) accession_choices = [] @@ -489,15 +500,28 @@ def get_choices(self, with_plasmids=True, to_exclude=[]): accession_choices.append((plasmid, f"{data.description} plasmid")) + if with_groups: + accession_choices.extend([(self.group_id_to_key(group[0]), group[0]) + for group in self.db.get_groups()]) + accession_choices = filter(lambda choice: choice[0] not in to_exclude, accession_choices) return tuple(accession_choices) def extract_choices(self, indices, include_plasmids): - if include_plasmids: - plasmids = [self.plasmid_key_to_id(key) - for key in indices if self.is_plasmid(key)] - else: + plasmids = [] + groups = [] + taxids = set() + for key in indices: + if self.is_plasmid(key): + plasmids.append(self.plasmid_key_to_id(key)) + elif self.is_group(key): + groups.append(self.group_key_to_id(key)) + else: + taxids.add(int(key)) + + if not include_plasmids: plasmids = None - taxids = [int(key) for key in indices if not self.is_plasmid(key)] - return taxids, plasmids + + taxids.update(self.db.get_taxids_for_groups(groups)) + return list(taxids), plasmids From 51de62e7c643bf3f196da2e63921110367860093 Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Wed, 1 May 2024 12:50:59 +0200 Subject: [PATCH 08/17] Also exclude from choices taxids selected through groups. --- testing/webapp/test_utils.py | 14 +++++++++++--- webapp/views/utils.py | 5 +++++ 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/testing/webapp/test_utils.py b/testing/webapp/test_utils.py index 8eea4bfc5..e81f81386 100644 --- a/testing/webapp/test_utils.py +++ b/testing/webapp/test_utils.py @@ -60,7 +60,7 @@ def test_get_choices_handles_groups(self): self.taxons + self.groups, self.handler.get_choices(with_groups=True)) - def test_get_choices_handles_exclusion(self): + def test_get_choices_handles_taxid_exclusion(self): to_exclude = [el[0] for el in self.taxons[1:]] self.assertItemsEqual( self.taxons[:1], @@ -81,6 +81,9 @@ def test_get_choices_handles_exclusion(self): self.handler.get_choices(to_exclude=to_exclude, with_plasmids=False)) + def test_get_choices_handles_plasmid_exclusion(self): + self.add_plasmid_for_taxids([1, 2, 3]) + to_exclude = [self.plasmids[-1][0]] self.assertItemsEqual(self.taxons + self.groups + self.plasmids[:-1], self.handler.get_choices(to_exclude=to_exclude)) @@ -89,8 +92,13 @@ def test_get_choices_handles_exclusion(self): self.assertItemsEqual(self.taxons + self.groups, self.handler.get_choices(to_exclude=to_exclude)) - to_exclude = [el[0] for el in self.groups[1:]] - self.assertItemsEqual(self.taxons + self.groups[0:1] + self.plasmids, + def test_get_choices_handles_group_exclusion(self): + to_exclude = [self.groups[1][0]] + self.assertItemsEqual(self.taxons[:-1] + [self.groups[0], self.groups[2]], + self.handler.get_choices(to_exclude=to_exclude)) + + to_exclude.append(self.groups[2][0]) + self.assertItemsEqual([self.groups[0]], self.handler.get_choices(to_exclude=to_exclude)) def test_extract_choices_returns_none_when_include_plasmids_is_false(self): diff --git a/webapp/views/utils.py b/webapp/views/utils.py index e4b9781c1..9da0c921c 100644 --- a/webapp/views/utils.py +++ b/webapp/views/utils.py @@ -504,6 +504,11 @@ def get_choices(self, with_plasmids=True, with_groups=True, to_exclude=[]): accession_choices.extend([(self.group_id_to_key(group[0]), group[0]) for group in self.db.get_groups()]) + # We also exclude taxids contained in the excluded groups + groups_to_exclude = [self.group_key_to_id(key) for key in to_exclude + if self.is_group(key)] + in_groups = self.db.get_taxids_for_groups(groups_to_exclude) + to_exclude = set(to_exclude).union({str(el) for el in in_groups}) accession_choices = filter(lambda choice: choice[0] not in to_exclude, accession_choices) return tuple(accession_choices) From c3420a17fdd3224e34767b74001c5c64cae61a8a Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Wed, 1 May 2024 12:56:25 +0200 Subject: [PATCH 09/17] Remove unnecessary with_groups parameter. --- testing/webapp/test_utils.py | 22 +++++++++------------- webapp/views/utils.py | 7 +++---- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/testing/webapp/test_utils.py b/testing/webapp/test_utils.py index e81f81386..2fb92bb36 100644 --- a/testing/webapp/test_utils.py +++ b/testing/webapp/test_utils.py @@ -39,32 +39,28 @@ def assertItemsEqual(self, expected, choices): def test_get_choices_handles_plasmids(self): self.assertItemsEqual( - self.taxons, - self.handler.get_choices(with_groups=False)) + self.taxons + self.groups, + self.handler.get_choices()) self.add_plasmid_for_taxids([1, 3]) self.assertItemsEqual( - self.taxons + self.plasmids[::2], - self.handler.get_choices(with_groups=False)) + self.taxons + self.groups + self.plasmids[::2], + self.handler.get_choices()) self.assertItemsEqual( - self.taxons, - self.handler.get_choices(with_groups=False, with_plasmids=False)) + self.taxons + self.groups, + self.handler.get_choices(with_plasmids=False)) def test_get_choices_handles_groups(self): - self.assertItemsEqual( - self.taxons, - self.handler.get_choices(with_groups=False)) - self.assertItemsEqual( self.taxons + self.groups, - self.handler.get_choices(with_groups=True)) + self.handler.get_choices()) def test_get_choices_handles_taxid_exclusion(self): to_exclude = [el[0] for el in self.taxons[1:]] self.assertItemsEqual( - self.taxons[:1], - self.handler.get_choices(with_groups=False, to_exclude=to_exclude)) + self.taxons[:1] + self.groups, + self.handler.get_choices(to_exclude=to_exclude)) self.assertItemsEqual( self.taxons[:1] + self.groups, diff --git a/webapp/views/utils.py b/webapp/views/utils.py index 9da0c921c..14043beab 100644 --- a/webapp/views/utils.py +++ b/webapp/views/utils.py @@ -488,7 +488,7 @@ def db(self): self._db = DB.load_db_from_name(biodb_path) return self._db - def get_choices(self, with_plasmids=True, with_groups=True, to_exclude=[]): + def get_choices(self, with_plasmids=True, to_exclude=[]): result = self.db.get_genomes_description() result.set_index(result.index.astype(str), inplace=True) accession_choices = [] @@ -500,9 +500,8 @@ def get_choices(self, with_plasmids=True, with_groups=True, to_exclude=[]): accession_choices.append((plasmid, f"{data.description} plasmid")) - if with_groups: - accession_choices.extend([(self.group_id_to_key(group[0]), group[0]) - for group in self.db.get_groups()]) + accession_choices.extend([(self.group_id_to_key(group[0]), group[0]) + for group in self.db.get_groups()]) # We also exclude taxids contained in the excluded groups groups_to_exclude = [self.group_key_to_id(key) for key in to_exclude From aceb7ab41feba578c67dca4a9b0a2853a0a4e159 Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Wed, 1 May 2024 14:58:11 +0200 Subject: [PATCH 10/17] Also exclude groups containing an excluded taxid --- testing/webapp/test_utils.py | 41 +++++++++++++++++------------------- webapp/lib/db_utils.py | 7 ++++++ webapp/views/utils.py | 19 +++++++++++++---- 3 files changed, 41 insertions(+), 26 deletions(-) diff --git a/testing/webapp/test_utils.py b/testing/webapp/test_utils.py index 2fb92bb36..cb4bec841 100644 --- a/testing/webapp/test_utils.py +++ b/testing/webapp/test_utils.py @@ -57,45 +57,42 @@ def test_get_choices_handles_groups(self): self.handler.get_choices()) def test_get_choices_handles_taxid_exclusion(self): - to_exclude = [el[0] for el in self.taxons[1:]] + exclude = [el[0] for el in self.taxons[1:]] self.assertItemsEqual( - self.taxons[:1] + self.groups, - self.handler.get_choices(to_exclude=to_exclude)) - - self.assertItemsEqual( - self.taxons[:1] + self.groups, - self.handler.get_choices(to_exclude=to_exclude)) + [self.taxons[0]], + self.handler.get_choices(exclude=exclude)) self.add_plasmid_for_taxids([1, 2, 3]) self.assertItemsEqual(self.taxons + self.groups + self.plasmids, self.handler.get_choices()) - to_exclude = [self.taxons[-1][0]] - self.assertItemsEqual(self.taxons[:-1] + self.groups + self.plasmids, - self.handler.get_choices(to_exclude=to_exclude)) - self.assertItemsEqual(self.taxons[:-1] + self.groups, - self.handler.get_choices(to_exclude=to_exclude, + exclude = [self.taxons[-1][0]] + self.assertItemsEqual(self.taxons[:-1] + [self.groups[2]] + self.plasmids, + self.handler.get_choices(exclude=exclude)) + + self.assertItemsEqual(self.taxons[:-1] + [self.groups[2]], + self.handler.get_choices(exclude=exclude, with_plasmids=False)) def test_get_choices_handles_plasmid_exclusion(self): self.add_plasmid_for_taxids([1, 2, 3]) - to_exclude = [self.plasmids[-1][0]] + exclude = [self.plasmids[-1][0]] self.assertItemsEqual(self.taxons + self.groups + self.plasmids[:-1], - self.handler.get_choices(to_exclude=to_exclude)) + self.handler.get_choices(exclude=exclude)) - to_exclude = [el[0] for el in self.plasmids] + exclude = [el[0] for el in self.plasmids] self.assertItemsEqual(self.taxons + self.groups, - self.handler.get_choices(to_exclude=to_exclude)) + self.handler.get_choices(exclude=exclude)) def test_get_choices_handles_group_exclusion(self): - to_exclude = [self.groups[1][0]] - self.assertItemsEqual(self.taxons[:-1] + [self.groups[0], self.groups[2]], - self.handler.get_choices(to_exclude=to_exclude)) + exclude = [self.groups[1][0]] + self.assertItemsEqual(self.taxons[:-1] + [self.groups[2]], + self.handler.get_choices(exclude=exclude)) - to_exclude.append(self.groups[2][0]) - self.assertItemsEqual([self.groups[0]], - self.handler.get_choices(to_exclude=to_exclude)) + exclude.append(self.groups[2][0]) + self.assertItemsEqual([], + self.handler.get_choices(exclude=exclude)) def test_extract_choices_returns_none_when_include_plasmids_is_false(self): self.assertEqual( diff --git a/webapp/lib/db_utils.py b/webapp/lib/db_utils.py index bb07ec958..e6cc2e105 100644 --- a/webapp/lib/db_utils.py +++ b/webapp/lib/db_utils.py @@ -1487,6 +1487,13 @@ def get_taxids_for_groups(self, group_names): results = self.server.adaptor.execute_and_fetchall(query, group_names) return (el[0] for el in results) + def get_groups_containing_taxids(self, taxids): + plchd = self.gen_placeholder_string(taxids) + query = f"SELECT DISTINCT group_name FROM taxon_in_group "\ + f"WHERE taxon_id IN ({plchd});" + results = self.server.adaptor.execute_and_fetchall(query, taxids) + return (el[0] for el in results) + def load_genomes_info(self, data): sql = ( "CREATE TABLE genome_summary (taxon_id INTEGER, completeness FLOAT, " diff --git a/webapp/views/utils.py b/webapp/views/utils.py index 14043beab..febf97c3c 100644 --- a/webapp/views/utils.py +++ b/webapp/views/utils.py @@ -481,6 +481,10 @@ def group_key_to_id(self, key): def group_id_to_key(self, identifier): return f"{self.group_prefix}{identifier}" + @classmethod + def is_taxid(cls, key): + return not (cls.is_group(key) or cls.is_plasmid(key)) + @property def db(self): if self._db is None: @@ -488,7 +492,7 @@ def db(self): self._db = DB.load_db_from_name(biodb_path) return self._db - def get_choices(self, with_plasmids=True, to_exclude=[]): + def get_choices(self, with_plasmids=True, exclude=[]): result = self.db.get_genomes_description() result.set_index(result.index.astype(str), inplace=True) accession_choices = [] @@ -504,11 +508,18 @@ def get_choices(self, with_plasmids=True, to_exclude=[]): for group in self.db.get_groups()]) # We also exclude taxids contained in the excluded groups - groups_to_exclude = [self.group_key_to_id(key) for key in to_exclude + groups_to_exclude = [self.group_key_to_id(key) for key in exclude if self.is_group(key)] in_groups = self.db.get_taxids_for_groups(groups_to_exclude) - to_exclude = set(to_exclude).union({str(el) for el in in_groups}) - accession_choices = filter(lambda choice: choice[0] not in to_exclude, + exclude = set(exclude).union({str(el) for el in in_groups}) + + # And we exclude groups containing an excluded taxid + taxids_to_exclude = list(filter(self.is_taxid, exclude)) + exclude = exclude.union( + {self.group_id_to_key(groupid) for groupid in + self.db.get_groups_containing_taxids(taxids_to_exclude)}) + + accession_choices = filter(lambda choice: choice[0] not in exclude, accession_choices) return tuple(accession_choices) From 0b8f3463a3eed240bc25c3a60cbc6af72fb87041 Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Wed, 1 May 2024 17:04:42 +0200 Subject: [PATCH 11/17] Implement just excluding taxids contained in group exclusion list. This introduces a new argument exclude_taxids_in_groups to AccessionFieldHandler.get_choices. --- testing/webapp/test_utils.py | 11 +++++++++++ webapp/views/utils.py | 9 ++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/testing/webapp/test_utils.py b/testing/webapp/test_utils.py index cb4bec841..54a588f74 100644 --- a/testing/webapp/test_utils.py +++ b/testing/webapp/test_utils.py @@ -94,6 +94,17 @@ def test_get_choices_handles_group_exclusion(self): self.assertItemsEqual([], self.handler.get_choices(exclude=exclude)) + def test_get_choices_handles_exclude_taxids_in_groups(self): + exclude = [self.groups[1][0]] + self.assertItemsEqual( + self.taxons[:-1] + self.groups, + self.handler.get_choices(exclude_taxids_in_groups=exclude)) + + exclude.append(self.groups[2][0]) + self.assertItemsEqual( + self.groups, + self.handler.get_choices(exclude_taxids_in_groups=exclude)) + def test_extract_choices_returns_none_when_include_plasmids_is_false(self): self.assertEqual( ([1, 3], None), diff --git a/webapp/views/utils.py b/webapp/views/utils.py index febf97c3c..50cb34875 100644 --- a/webapp/views/utils.py +++ b/webapp/views/utils.py @@ -492,7 +492,7 @@ def db(self): self._db = DB.load_db_from_name(biodb_path) return self._db - def get_choices(self, with_plasmids=True, exclude=[]): + def get_choices(self, with_plasmids=True, exclude=[], exclude_taxids_in_groups=[]): result = self.db.get_genomes_description() result.set_index(result.index.astype(str), inplace=True) accession_choices = [] @@ -519,6 +519,13 @@ def get_choices(self, with_plasmids=True, exclude=[]): {self.group_id_to_key(groupid) for groupid in self.db.get_groups_containing_taxids(taxids_to_exclude)}) + # Finally we exclude taxids from groups in exclude_taxids_in_groups + groups_to_exclude = [self.group_key_to_id(key) + for key in exclude_taxids_in_groups + if self.is_group(key)] + in_groups = self.db.get_taxids_for_groups(groups_to_exclude) + exclude = set(exclude).union({str(el) for el in in_groups}) + accession_choices = filter(lambda choice: choice[0] not in exclude, accession_choices) return tuple(accession_choices) From 55f5202ff92e0258b3fb26363e11ac18d3660fad Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Wed, 1 May 2024 17:08:46 +0200 Subject: [PATCH 12/17] Handle groups correctly in AutocompleteTaxid view. --- webapp/chlamdb/forms.py | 6 ++++-- webapp/views/autocomplete.py | 9 ++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/webapp/chlamdb/forms.py b/webapp/chlamdb/forms.py index be264c79d..574341dd1 100644 --- a/webapp/chlamdb/forms.py +++ b/webapp/chlamdb/forms.py @@ -326,7 +326,8 @@ class ExtractForm(forms.Form): choices=accession_choices, widget=Select2Multiple( url="autocomplete_taxid", - forward=(forward.Field("no_orthologs_in", "to_exclude"), + forward=(forward.Field("no_orthologs_in", "exclude"), + forward.Field("orthologs_in", "exclude_taxids_in_groups"), forward.Field("checkbox_accessions", "include_plasmids")), attrs={"data-close-on-select": "false", "data-placeholder": "Nothing selected"}), @@ -337,7 +338,8 @@ class ExtractForm(forms.Form): choices=accession_choices, widget=Select2Multiple( url="autocomplete_taxid", - forward=(forward.Field("orthologs_in", "to_exclude"), + forward=(forward.Field("orthologs_in", "exclude"), + forward.Field("no_orthologs_in", "exclude_taxids_in_groups"), forward.Field("checkbox_accessions", "include_plasmids")), attrs={"data-close-on-select": "false", "data-placeholder": "Nothing selected"}), diff --git a/webapp/views/autocomplete.py b/webapp/views/autocomplete.py index 85a9bd7e5..9931ab369 100644 --- a/webapp/views/autocomplete.py +++ b/webapp/views/autocomplete.py @@ -7,9 +7,12 @@ class AutocompleteTaxid(Select2ListView): def get_list(self): with_plasmids = self.forwarded["include_plasmids"] - to_exclude = self.forwarded["to_exclude"] - return AccessionFieldHandler().get_choices(with_plasmids=with_plasmids, - to_exclude=to_exclude) + exclude = self.forwarded["exclude"] + exclude_taxids_in_groups = self.forwarded["exclude_taxids_in_groups"] + return AccessionFieldHandler().get_choices( + with_plasmids=with_plasmids, + exclude=exclude, + exclude_taxids_in_groups=exclude_taxids_in_groups) class AutocompleteNMissing(Select2ListView): From fa1e592cb0abe7baba8b1344dbbb0d1d3df1a015 Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Wed, 1 May 2024 17:10:49 +0200 Subject: [PATCH 13/17] Handle groups correctly in AutocompleteNMissing view. --- webapp/chlamdb/forms.py | 5 ++++- webapp/views/autocomplete.py | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/webapp/chlamdb/forms.py b/webapp/chlamdb/forms.py index 574341dd1..2676a094b 100644 --- a/webapp/chlamdb/forms.py +++ b/webapp/chlamdb/forms.py @@ -351,7 +351,8 @@ class ExtractForm(forms.Form): range(len(accession_choices))), widget=ListSelect2( url="autocomplete_n_missing", - forward=(forward.Field("orthologs_in", "included"),)), + forward=(forward.Field("orthologs_in", "included"), + forward.Field("checkbox_accessions", "include_plasmids"))), required=False) def extract_choices(self, indices, include_plasmids): @@ -394,12 +395,14 @@ def __init__(self, *args, **kwargs): def clean(self): cleaned_data = super(ExtractForm, self).clean() + self.included_taxids, self.included_plasmids = self.extract_choices( self.cleaned_data["orthologs_in"], self.cleaned_data["checkbox_accessions"]) self.excluded_taxids, self.excluded_plasmids = self.extract_choices( self.cleaned_data["no_orthologs_in"], self.cleaned_data["checkbox_accessions"]) + self.n_missing = self.get_n_missing() self.n_included = len(self.included_taxids) if self.included_plasmids is not None: diff --git a/webapp/views/autocomplete.py b/webapp/views/autocomplete.py index 9931ab369..a5479e8e5 100644 --- a/webapp/views/autocomplete.py +++ b/webapp/views/autocomplete.py @@ -18,6 +18,8 @@ def get_list(self): class AutocompleteNMissing(Select2ListView): def get_list(self): - n_max = len(self.forwarded["included"]) + taxids, plasmids = AccessionFieldHandler().extract_choices( + self.forwarded["included"], self.forwarded["include_plasmids"]) + n_max = len(taxids) + len(plasmids) choices = [(i, i) for i in range(n_max)] return choices From 7f6810dc50da71bb698942c21fe31ee1494633c5 Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Thu, 2 May 2024 08:53:31 +0200 Subject: [PATCH 14/17] Handle missing arguments in autocomplete views. --- testing/webapp/test_views.py | 12 +++++++++++- webapp/views/autocomplete.py | 12 +++++++----- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/testing/webapp/test_views.py b/testing/webapp/test_views.py index 228e2970f..49d879e73 100644 --- a/testing/webapp/test_views.py +++ b/testing/webapp/test_views.py @@ -24,6 +24,8 @@ urls = [ '/about', '/amr_comparison', + '/autocomplete_n_missing/', + '/autocomplete_taxid/', '/blast/', '/circos/', '/circos_main/', @@ -150,9 +152,17 @@ def test_all_urlpatterns_are_tested(self): "Some patterns are not covered in the tests: please add them to " "untested_patterns or urls") - def test_all_views_render_valid_html(self): + def test_all_views_render_valid_html_or_json(self): for url in urls: resp = self.client.get(url) + if resp.get("Content-Type") == 'application/json': + try: + resp.json() + except Exception as exc: + print(f"\n\nInvalid json for {url}") + raise exc + finally: + continue try: self.assertContains(resp, "", html=True) except Exception as exc: diff --git a/webapp/views/autocomplete.py b/webapp/views/autocomplete.py index a5479e8e5..7792cbd54 100644 --- a/webapp/views/autocomplete.py +++ b/webapp/views/autocomplete.py @@ -6,9 +6,9 @@ class AutocompleteTaxid(Select2ListView): def get_list(self): - with_plasmids = self.forwarded["include_plasmids"] - exclude = self.forwarded["exclude"] - exclude_taxids_in_groups = self.forwarded["exclude_taxids_in_groups"] + with_plasmids = self.forwarded.get("include_plasmids", False) + exclude = self.forwarded.get("exclude", []) + exclude_taxids_in_groups = self.forwarded.get("exclude_taxids_in_groups", []) return AccessionFieldHandler().get_choices( with_plasmids=with_plasmids, exclude=exclude, @@ -19,7 +19,9 @@ class AutocompleteNMissing(Select2ListView): def get_list(self): taxids, plasmids = AccessionFieldHandler().extract_choices( - self.forwarded["included"], self.forwarded["include_plasmids"]) - n_max = len(taxids) + len(plasmids) + self.forwarded.get("included", []), self.forwarded.get("include_plasmids", False)) + n_max = len(taxids) + if plasmids: + n_max += len(plasmids) choices = [(i, i) for i in range(n_max)] return choices From ac8552a72321e16d0677e0bdd518d85193cbabf0 Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Thu, 2 May 2024 08:54:28 +0200 Subject: [PATCH 15/17] Fix bug in custom plots view with orthogroups. --- webapp/views/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/views/utils.py b/webapp/views/utils.py index 50cb34875..60733ac45 100644 --- a/webapp/views/utils.py +++ b/webapp/views/utils.py @@ -297,7 +297,7 @@ def __init__(self, db): def id_to_object_type(self, identifier): match = self.og_re.match(identifier) parsed_id = match and int(match.groups()[0]) - if parsed_id and self.db.check_orthogroup_entry_id(parsed_id): + if parsed_id and self.db.check_og_entry_id(parsed_id): return "orthogroup", parsed_id match = self.cog_re.match(identifier) From 3272a1ea4647dea2ee48798edfa5c8bcf1a86bad Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Thu, 2 May 2024 10:18:00 +0200 Subject: [PATCH 16/17] Add tests for the autocomplete views. --- testing/webapp/test_autocomplete_views.py | 141 ++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 testing/webapp/test_autocomplete_views.py diff --git a/testing/webapp/test_autocomplete_views.py b/testing/webapp/test_autocomplete_views.py new file mode 100644 index 000000000..38f8f9347 --- /dev/null +++ b/testing/webapp/test_autocomplete_views.py @@ -0,0 +1,141 @@ +import json +from contextlib import contextmanager + +from django.conf import settings +from django.test import SimpleTestCase +from lib.db_utils import DB + + +class BaseAutocompleteTestCase(SimpleTestCase): + + def make_request(self, **kwargs): + url = f'{self.base_url}?forward={json.dumps(kwargs)}' + return self.client.get(url) + + +class TestAutocompleteTaxid(BaseAutocompleteTestCase): + + base_url = '/autocomplete_taxid/' + + taxons = [ + {'id': '1', 'text': 'Klebsiella pneumoniae R6724_16313'}, + {'id': '2', 'text': 'Klebsiella pneumoniae R6726_16314'}, + {'id': '3', 'text': 'Klebsiella pneumoniae R6728_16315'}] + + groups = [ + {'id': 'group:positive', 'text': 'positive'}, + {'id': 'group:negative', 'text': 'negative'}, + {'id': 'group:all', 'text': 'all'}] + + plasmids = [ + {'id': 'plasmid:1', 'text': 'Klebsiella pneumoniae R6724_16313 plasmid'}, + {'id': 'plasmid:2', 'text': 'Klebsiella pneumoniae R6726_16314 plasmid'}, + {'id': 'plasmid:3', 'text': 'Klebsiella pneumoniae R6728_16315 plasmid'}] + + @contextmanager + def add_plasmid_for_taxids(self, taxids): + """We need to commit this change so that it is picked up + by the autocomplete view, so we need to cleanup afterwards. + I guess we could also have isolated the DB by making a backup in setUp + and restoring it in tearDown, but seemed like overkill for now. + """ + try: + plasmid_term_id = self.db.server.adaptor.execute_one( + "SELECT term_id FROM term WHERE name='plasmid'")[0] + for taxid in taxids: + self.db.server.adaptor.execute( + f"UPDATE bioentry_qualifier_value SET value=1 " + f"WHERE bioentry_id={taxid} AND term_id={plasmid_term_id};") + self.db.server.commit() + yield + finally: + for taxid in taxids: + self.db.server.adaptor.execute( + f"UPDATE bioentry_qualifier_value SET value=0 " + f"WHERE bioentry_id={taxid} AND term_id={plasmid_term_id};") + self.db.server.commit() + + def assertItemsEqual(self, expected, actual): + self.assertEqual(sorted(expected, key=lambda x: x["id"]), + sorted(actual, key=lambda x: x["id"])) + + def setUp(self): + biodb_path = settings.BIODB_DB_PATH + self.db = DB.load_db_from_name(biodb_path) + + def tearDown(self): + self.db.server.close() + + def test_handles_include_plasmids(self): + with self.add_plasmid_for_taxids([1, 2]): + resp = self.make_request() + self.assertItemsEqual( + self.taxons + self.groups, + resp.json()["results"]) + + resp = self.make_request(include_plasmids=True) + self.assertItemsEqual( + self.taxons + self.groups + self.plasmids[:2], + resp.json()["results"]) + + def test_handles_exclude(self): + resp = self.make_request(exclude=["3"]) + self.assertItemsEqual( + [self.taxons[0], self.taxons[1], self.groups[0]], + resp.json()["results"]) + + resp = self.make_request(exclude=["group:positive"]) + self.assertItemsEqual( + [self.taxons[2], self.groups[1]], + resp.json()["results"]) + + resp = self.make_request(exclude=["3", "group:positive"]) + self.assertItemsEqual( + [], + resp.json()["results"]) + + def test_handles_exclude_taxids_in_groups(self): + # ignored because these are not groups + resp = self.make_request(exclude_taxids_in_groups=["1", "3"]) + self.assertItemsEqual( + self.taxons + self.groups, + resp.json()["results"]) + + resp = self.make_request(exclude_taxids_in_groups=["group:positive"]) + self.assertItemsEqual( + [self.taxons[2]] + self.groups, + resp.json()["results"]) + + resp = self.make_request(exclude_taxids_in_groups=["group:positive", + "group:negative"]) + self.assertItemsEqual( + self.groups, + resp.json()["results"]) + + +class TestAutocompleteNMissing(BaseAutocompleteTestCase): + + base_url = '/autocomplete_n_missing/' + + @staticmethod + def get_expected_response(n): + return [{"id": i, "text": i} for i in range(n)] + + def test_handles_include_plasmids(self): + included = ["1", "2", "plasmid:2", "plasmid:3"] + resp = self.make_request(included=included) + self.assertEqual(self.get_expected_response(2), + resp.json()["results"]) + + resp = self.make_request(included=included, include_plasmids=True) + self.assertEqual(self.get_expected_response(4), + resp.json()["results"]) + + def test_handles_groups(self): + resp = self.make_request(included=["group:positive"]) + self.assertEqual(self.get_expected_response(2), + resp.json()["results"]) + + resp = self.make_request(included=["group:positive", "group:negative"]) + self.assertEqual(self.get_expected_response(3), + resp.json()["results"]) From 0e8836df91efe1ad962b6282451c3fd57f2a5dab Mon Sep 17 00:00:00 2001 From: Niklaus Johner Date: Thu, 2 May 2024 11:49:50 +0200 Subject: [PATCH 17/17] Add changelog. --- docs/CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index e07da51c3..75d69bd36 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -9,6 +9,8 @@ to [Common Changelog](https://common-changelog.org) ### Changed +- Handle groups in hit extraction view. ([#84](https://github.com/metagenlab/zDB/pull/84)) (Niklaus Johner) +- Allow using groups to define phenotype in GWAS view. ([#82](https://github.com/metagenlab/zDB/pull/82)) (Niklaus Johner) - Display form validation errors next to the corresponding fields. ([#83](https://github.com/metagenlab/zDB/pull/83)) (Niklaus Johner) - Filter VF hits by SeqID and coverage and keep one hit per locus. ([#77](https://github.com/metagenlab/zDB/pull/77)) (Niklaus Johner) - Improve layout for various views, making better use of available space. ([#70](https://github.com/metagenlab/zDB/pull/70)) (Niklaus Johner)