diff --git a/ckanext/bulk/assets/scripts/bulk-manager-form.js b/ckanext/bulk/assets/scripts/bulk-manager-form.js index b515bad..43ae8ee 100644 --- a/ckanext/bulk/assets/scripts/bulk-manager-form.js +++ b/ckanext/bulk/assets/scripts/bulk-manager-form.js @@ -17,7 +17,10 @@ ckan.module("bulk-manager-form", function () { addFilter: "/bulk/htmx/create_filter_item", addUpdate: "/bulk/htmx/create_update_item", }, - options: {}, + options: { + resultMaxEntries: 50, + logsMaxEntries: 50, + }, initialize() { $.proxyAll(this, /_/); @@ -205,7 +208,6 @@ ckan.module("bulk-manager-form", function () { } this._toggleLoadSpinner(true); - // window.bulkUpdateOn = this._getUpdateOn(); if (!data.filters.length) { this.infoBlock.find(".counter").html("There will be information about how many entities will be changed."); @@ -217,7 +219,7 @@ ckan.module("bulk-manager-form", function () { "bulk_get_entities_by_filters", data, (data) => { - if (!data.result || data.result.error || data.result.fields.length === 0) { + if (!data.result || data.result.error || data.result.entities.length === 0) { if (data.result.error) { iziToast.error({ message: data.result.error }); } @@ -231,14 +233,14 @@ ckan.module("bulk-manager-form", function () { this.bulkModalResult.iziModal( "setContent", "
"
-                        + JSON.stringify(data.result.fields, null, 2)
+                        + JSON.stringify(this._limitResultEntries(data.result.entities), null, 2)
                         + "
" ); Prism.highlightElement(this.bulkModalResult.find("pre")[0]); - this.bulkEntitiesToUpdate = data.result.fields; - this.infoBlock.find(".counter").html("Found " + data.result.fields.length + " entities"); + this.bulkEntitiesToUpdate = data.result.entities; + this.infoBlock.find(".counter").html("Found " + data.result.entities.length + " entities"); this._toggleLoadSpinner(false); }, (resp) => { @@ -248,6 +250,58 @@ ckan.module("bulk-manager-form", function () { ); }, + /** + * Limit the number of entries in the result to avoid performance issues. + * + * Also remove some fields that are not needed for the user. + * + * @param {Array} entities + * + * @returns {Array} + */ + _limitResultEntries: function (entities) { + entities = entities.slice(0, this.options.resultMaxEntries); + + entities.forEach(entity => { + delete entity.resources; + delete entity.organization; + delete entity.groups; + + delete entity.relationships_as_subject, + delete entity.relationships_as_object; + }); + + return entities; + }, + + /** + * Limit the number of entries in the logs to avoid performance issues. + * + * Also remove some fields that are not needed for the user. + * + * @param {Array} logs + * + * @returns {Array} + */ + _limitLogsEntries: function (logs) { + logs = logs.slice(logs.length - this.options.logsMaxEntries, logs.length); + + logs.forEach(log => { + if (!log.result) { + return; + } + + delete log.result.resources; + delete log.result.organization; + delete log.result.groups; + + delete log.result.relationships_as_subject, + delete log.result.relationships_as_object; + }); + + return logs; + }, + _initFieldSelectors: function (selectItems, reinit = false) { let prevValue = ""; @@ -335,7 +389,7 @@ ckan.module("bulk-manager-form", function () { "setContent", { content: "
"
-                        + JSON.stringify(this.bulkLogs, null, 2)
+                        + JSON.stringify(this._limitLogsEntries(this.bulkLogs), null, 2)
                         + "
", } ); diff --git a/ckanext/bulk/entity_manager.py b/ckanext/bulk/entity_manager.py index aeabc77..46a0892 100644 --- a/ckanext/bulk/entity_manager.py +++ b/ckanext/bulk/entity_manager.py @@ -1,10 +1,14 @@ from __future__ import annotations +import json from abc import abstractmethod from typing import Any, TypedDict +from attr import field + import ckan.plugins.toolkit as tk from ckan import model +from ckan.lib.redis import connect_to_redis from ckanext.bulk import const @@ -67,10 +71,10 @@ def get_entity_by_id(cls, entity_id: str) -> dict[str, Any] | None: @classmethod def combine_filters(cls, filters: list[FilterItem]) -> list[CombinedFilter]: - combined_filters = [] + combined_filters: list[CombinedFilter] = [] - current_field = None - current_operator = None + current_field = "" + current_operator = "" current_values = [] for filter_item in filters: @@ -82,11 +86,11 @@ def combine_filters(cls, filters: list[FilterItem]) -> list[CombinedFilter]: else: if current_field and current_operator: combined_filters.append( - { - "field": current_field, - "operator": current_operator, - "values": current_values, - } + CombinedFilter( + field=current_field, + operator=current_operator, + value=current_values, + ) ) current_field = filter_item["field"] @@ -95,11 +99,11 @@ def combine_filters(cls, filters: list[FilterItem]) -> list[CombinedFilter]: if current_field and current_operator: combined_filters.append( - { - "field": current_field, - "operator": current_operator, - "values": current_values, - } + CombinedFilter( + field=current_field, + operator=current_operator, + value=current_values, + ) ) return combined_filters @@ -111,7 +115,7 @@ def update_entity( entity = cls.get_entity_by_id(entity_id) if not entity: - raise tk.ObjectNotFound(f"Entity {entity_id} not found") + raise tk.ObjectNotFound(f"Entity <{entity_id}> not found") return tk.get_action(cls.patch_action)( {"ignore_auth": True}, @@ -131,6 +135,22 @@ def delete_entity(cls, entity_id: str) -> bool: return True + @classmethod + def cache_fields_to_redis(cls, fields: list[FieldItem], ttl: int = 3600): + conn = connect_to_redis() + conn.set(f"ckanext-bulk:fields:{cls.entity_type}", json.dumps(fields), ex=ttl) + + @classmethod + def get_fields_from_redis(cls) -> list[FieldItem]: + conn = connect_to_redis() + + fields = conn.get(f"ckanext-bulk:fields:{cls.entity_type}") + + if not fields: + return [] + + return json.loads(fields) + class DatasetEntityManager(EntityManager): entity_type = "dataset" @@ -140,6 +160,9 @@ class DatasetEntityManager(EntityManager): @classmethod def get_fields(cls) -> list[FieldItem]: + if fields := cls.get_fields_from_redis(): + return fields + result = tk.get_action("package_search")( {"ignore_auth": True}, {"rows": 1, "include_private": True, "q": f'type:"{cls.entity_type}"'}, @@ -148,7 +171,11 @@ def get_fields(cls) -> list[FieldItem]: if not result["results"]: return [] - return [FieldItem(value=field, text=field) for field in result["results"][0]] + fields = [FieldItem(value=field, text=field) for field in result["results"][0]] + + cls.cache_fields_to_redis(fields) + + return fields @classmethod def search_entities_by_filters( @@ -190,7 +217,7 @@ def search_entities_by_filters( query = f" {global_operator} ".join(q_list) start = 0 - rows = 100 + rows = 1000 results = [] @@ -225,6 +252,9 @@ class DatasetResourceEntityManager(EntityManager): @classmethod def get_fields(cls) -> list[FieldItem]: + if fields := cls.get_fields_from_redis(): + return fields + resource: model.Resource | None = ( model.Session.query(model.Resource) .join(model.Package) @@ -235,7 +265,13 @@ def get_fields(cls) -> list[FieldItem]: if not resource: return [] - return [FieldItem(value=field, text=field) for field in resource.get_columns()] + fields = [ + FieldItem(value=field, text=field) for field in resource.get_columns() + ] + + cls.cache_fields_to_redis(fields) + + return fields @classmethod def search_entities_by_filters( @@ -275,20 +311,21 @@ class GroupEntityManager(EntityManager): @classmethod def get_fields(cls) -> list[FieldItem]: - item_list = tk.get_action(cls.show_action)( - {"ignore_auth": True}, {"all_fields": True} + if fields := cls.get_fields_from_redis(): + return fields + + item_list: list[dict[str, Any]] = tk.get_action(cls.show_action)( + {"ignore_auth": True}, {"all_fields": True, "rows": 1} ) if not item_list: return [] - return [ - { - "value": field, - "text": field, - } - for field in item_list[0] - ] + fields = [FieldItem(value=field, text=field) for field in item_list[0]] + + cls.cache_fields_to_redis(fields) + + return fields @classmethod def search_entities_by_filters( diff --git a/ckanext/bulk/logic/action.py b/ckanext/bulk/logic/action.py index fb9ac0a..160a732 100644 --- a/ckanext/bulk/logic/action.py +++ b/ckanext/bulk/logic/action.py @@ -55,16 +55,16 @@ def bulk_get_entities_by_filters(context: Context, data_dict: dict[str, Any]): ) except (ValueError, tk.ValidationError) as e: return { - "fields": [], + "entities": [], "error": str(e), } except DatabaseError as e: return { - "fields": [], + "entities": [], "error": f"Database error: {e.statement}", } - return {"fields": result} + return {"entities": result} @tk.side_effect_free diff --git a/ckanext/bulk/tests/test_entity_managers.py b/ckanext/bulk/tests/test_entity_managers.py index 5398d5b..f470541 100644 --- a/ckanext/bulk/tests/test_entity_managers.py +++ b/ckanext/bulk/tests/test_entity_managers.py @@ -488,17 +488,6 @@ def test_update_group_invalid_field(self, group_entity_manager, group_factory): assert "new_field" not in result - def test_update_group_empty_field(self, group_entity_manager, group_factory): - group = group_factory() - - result = group_entity_manager.update_entity( - group["id"], [{"field": "name", "value": ""}] - ) - - # TODO: this looks like a CKAN bug, we shouldn't be able to nullify - # the name field - assert not result["name"] - def test_update_id_field(self, group_entity_manager, group_factory): group_factory() @@ -536,19 +525,6 @@ def test_update_group_invalid_field( assert "new_field" not in result - def test_update_group_empty_field( - self, organization_entity_manager, organization_factory - ): - organization = organization_factory() - - result = organization_entity_manager.update_entity( - organization["id"], [{"field": "name", "value": ""}] - ) - - # TODO: this looks like a CKAN bug, we shouldn't be able to nullify - # the name field - assert not result["name"] - def test_update_id_field(self, organization_entity_manager, organization_factory): organization_factory()