Merge pull request #67 from qld-gov-au/qgov-cleanup

QGOV cleanup as per #57
ckan · Jul 29, 2022 · f151aa5 · f151aa5
2 parents cd06189 + 636aab9
commit f151aa5
Show file tree

Hide file tree

Showing 3 changed files with 29 additions and 12 deletions.
diff --git a/ckanext/validation/logic.py b/ckanext/validation/logic.py
@@ -310,9 +310,9 @@ def resource_validation_run_batch(context, data_dict):
 
                     except t.ValidationError as e:
                         log.warning(
-                            u'Could not run validation for resource {} ' +
-                            u'from dataset {}: {}'.format(
-                                resource['id'], dataset['name'], str(e)))
+                            u'Could not run validation for resource %s ' +
+                            u'from dataset %s: %s',
+                                resource['id'], dataset['name'], e)
 
             if len(query['results']) < page_size:
                 break
@@ -487,7 +487,7 @@ def resource_create(context, data_dict):
 
         for plugin in plugins.PluginImplementations(IDataValidation):
             if not plugin.can_validate(context, data_dict):
-                log.debug('Skipping validation for resource {}'.format(resource_id))
+                log.debug('Skipping validation for resource %s', resource_id)
                 run_validation = False
 
         if run_validation:
@@ -603,7 +603,7 @@ def resource_update(context, data_dict):
         run_validation = True
         for plugin in plugins.PluginImplementations(IDataValidation):
             if not plugin.can_validate(context, data_dict):
-                log.debug('Skipping validation for resource {}'.format(id))
+                log.debug('Skipping validation for resource %s', id)
                 run_validation = False
 
         if run_validation:
@@ -642,8 +642,8 @@ def _run_sync_validation(resource_id, local_upload=False, new_resource=True):
              u'async': False})
     except t.ValidationError as e:
         log.info(
-            u'Could not run validation for resource {}: {}'.format(
-                resource_id, str(e)))
+            u'Could not run validation for resource %s: %s',
+                resource_id, e)
         return
 
     validation = t.get_action(u'resource_validation_show')(

diff --git a/ckanext/validation/plugin/__init__.py b/ckanext/validation/plugin/__init__.py
@@ -146,6 +146,7 @@ def before_create(self, context, data_dict):
         return self._process_schema_fields(data_dict)
 
     resources_to_validate = {}
+    packages_to_skip = {}
 
     def after_create(self, context, data_dict):
 
@@ -188,7 +189,7 @@ def _handle_validation_for_resource(self, context, resource):
 
             for plugin in p.PluginImplementations(IDataValidation):
                 if not plugin.can_validate(context, resource):
-                    log.debug('Skipping validation for resource {}'.format(resource['id']))
+                    log.debug('Skipping validation for resource %s', resource['id'])
                     return
 
             _run_async_validation(resource[u'id'])
@@ -197,6 +198,15 @@ def before_update(self, context, current_resource, updated_resource):
 
         updated_resource = self._process_schema_fields(updated_resource)
 
+        # the call originates from a resource API, so don't validate the entire package
+        package_id = updated_resource.get('package_id')
+        if not package_id:
+            existing_resource = t.get_action('resource_show')(
+                context={'ignore_auth': True}, data_dict={'id': updated_resource['id']})
+            if existing_resource:
+                package_id = existing_resource['package_id']
+        self.packages_to_skip[package_id] = True
+
         if not get_update_mode_from_config() == u'async':
             return updated_resource
 
@@ -242,6 +252,13 @@ def after_update(self, context, data_dict):
             return
 
         if is_dataset:
+            package_id = data_dict.get('id')
+            if self.packages_to_skip.pop(package_id, None) or context.get('save', False):
+                # Either we're updating an individual resource,
+                # or we're updating the package metadata via the web form;
+                # in both cases, we don't need to validate every resource.
+                return
+
             for resource in data_dict.get(u'resources', []):
                 if resource[u'id'] in self.resources_to_validate:
                     # This is part of a resource_update call, it will be
@@ -259,7 +276,7 @@ def after_update(self, context, data_dict):
             if resource_id in self.resources_to_validate:
                 for plugin in p.PluginImplementations(IDataValidation):
                     if not plugin.can_validate(context, data_dict):
-                        log.debug('Skipping validation for resource {}'.format(data_dict['id']))
+                        log.debug('Skipping validation for resource %s', data_dict['id'])
                         return
 
                 del self.resources_to_validate[resource_id]
@@ -299,8 +316,8 @@ def _run_async_validation(resource_id):
              u'async': True})
     except t.ValidationError as e:
         log.warning(
-            u'Could not run validation for resource {}: {}'.format(
-                resource_id, str(e)))
+            u'Could not run validation for resource %s: %s',
+                resource_id, e)
 
 def _get_underlying_file(wrapper):
     if isinstance(wrapper, FlaskFileStorage):

diff --git a/ckanext/validation/utils.py b/ckanext/validation/utils.py
@@ -70,4 +70,4 @@ def delete_local_uploaded_file(resource_id):
             os.rmdir(second_directory)
 
     except OSError as e:
-        log.warning(u'Error deleting uploaded file: {}'.format(e))
+        log.warning(u'Error deleting uploaded file: %s', e)