From e44293508741cfd3f26ee6d18a9608baf970f9a9 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 29 Jul 2022 13:30:25 +1000 Subject: [PATCH 1/2] Use parameterised logging to reduce unnecessary string concatenation --- ckanext/validation/logic.py | 14 +++++++------- ckanext/validation/plugin/__init__.py | 8 ++++---- ckanext/validation/utils.py | 2 +- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/ckanext/validation/logic.py b/ckanext/validation/logic.py index bc6db3ee..7378e85b 100644 --- a/ckanext/validation/logic.py +++ b/ckanext/validation/logic.py @@ -310,9 +310,9 @@ def resource_validation_run_batch(context, data_dict): except t.ValidationError as e: log.warning( - u'Could not run validation for resource {} ' + - u'from dataset {}: {}'.format( - resource['id'], dataset['name'], str(e))) + u'Could not run validation for resource %s ' + + u'from dataset %s: %s', + resource['id'], dataset['name'], e) if len(query['results']) < page_size: break @@ -487,7 +487,7 @@ def resource_create(context, data_dict): for plugin in plugins.PluginImplementations(IDataValidation): if not plugin.can_validate(context, data_dict): - log.debug('Skipping validation for resource {}'.format(resource_id)) + log.debug('Skipping validation for resource %s', resource_id) run_validation = False if run_validation: @@ -603,7 +603,7 @@ def resource_update(context, data_dict): run_validation = True for plugin in plugins.PluginImplementations(IDataValidation): if not plugin.can_validate(context, data_dict): - log.debug('Skipping validation for resource {}'.format(id)) + log.debug('Skipping validation for resource %s', id) run_validation = False if run_validation: @@ -642,8 +642,8 @@ def _run_sync_validation(resource_id, local_upload=False, new_resource=True): u'async': False}) except t.ValidationError as e: log.info( - u'Could not run validation for resource {}: {}'.format( - resource_id, str(e))) + u'Could not run validation for resource %s: %s', + resource_id, e) return validation = t.get_action(u'resource_validation_show')( diff --git a/ckanext/validation/plugin/__init__.py b/ckanext/validation/plugin/__init__.py index bbca635e..88ac0838 100644 --- a/ckanext/validation/plugin/__init__.py +++ b/ckanext/validation/plugin/__init__.py @@ -188,7 +188,7 @@ def _handle_validation_for_resource(self, context, resource): for plugin in p.PluginImplementations(IDataValidation): if not plugin.can_validate(context, resource): - log.debug('Skipping validation for resource {}'.format(resource['id'])) + log.debug('Skipping validation for resource %s', resource['id']) return _run_async_validation(resource[u'id']) @@ -259,7 +259,7 @@ def after_update(self, context, data_dict): if resource_id in self.resources_to_validate: for plugin in p.PluginImplementations(IDataValidation): if not plugin.can_validate(context, data_dict): - log.debug('Skipping validation for resource {}'.format(data_dict['id'])) + log.debug('Skipping validation for resource %s', data_dict['id']) return del self.resources_to_validate[resource_id] @@ -299,8 +299,8 @@ def _run_async_validation(resource_id): u'async': True}) except t.ValidationError as e: log.warning( - u'Could not run validation for resource {}: {}'.format( - resource_id, str(e))) + u'Could not run validation for resource %s: %s', + resource_id, e) def _get_underlying_file(wrapper): if isinstance(wrapper, FlaskFileStorage): diff --git a/ckanext/validation/utils.py b/ckanext/validation/utils.py index cd89176b..060b216d 100644 --- a/ckanext/validation/utils.py +++ b/ckanext/validation/utils.py @@ -70,4 +70,4 @@ def delete_local_uploaded_file(resource_id): os.rmdir(second_directory) except OSError as e: - log.warning(u'Error deleting uploaded file: {}'.format(e)) + log.warning(u'Error deleting uploaded file: %s', e) From 636aab98e4e25673509605fc0b80c07c94aafab2 Mon Sep 17 00:00:00 2001 From: ThrawnCA Date: Fri, 29 Jul 2022 13:30:45 +1000 Subject: [PATCH 2/2] Don't validate the entire package when updating a single resource --- ckanext/validation/plugin/__init__.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ckanext/validation/plugin/__init__.py b/ckanext/validation/plugin/__init__.py index 88ac0838..c291b247 100644 --- a/ckanext/validation/plugin/__init__.py +++ b/ckanext/validation/plugin/__init__.py @@ -146,6 +146,7 @@ def before_create(self, context, data_dict): return self._process_schema_fields(data_dict) resources_to_validate = {} + packages_to_skip = {} def after_create(self, context, data_dict): @@ -197,6 +198,15 @@ def before_update(self, context, current_resource, updated_resource): updated_resource = self._process_schema_fields(updated_resource) + # the call originates from a resource API, so don't validate the entire package + package_id = updated_resource.get('package_id') + if not package_id: + existing_resource = t.get_action('resource_show')( + context={'ignore_auth': True}, data_dict={'id': updated_resource['id']}) + if existing_resource: + package_id = existing_resource['package_id'] + self.packages_to_skip[package_id] = True + if not get_update_mode_from_config() == u'async': return updated_resource @@ -242,6 +252,13 @@ def after_update(self, context, data_dict): return if is_dataset: + package_id = data_dict.get('id') + if self.packages_to_skip.pop(package_id, None) or context.get('save', False): + # Either we're updating an individual resource, + # or we're updating the package metadata via the web form; + # in both cases, we don't need to validate every resource. + return + for resource in data_dict.get(u'resources', []): if resource[u'id'] in self.resources_to_validate: # This is part of a resource_update call, it will be