Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

QGOV cleanup as per #57 #67

Merged
merged 2 commits into from
Jul 29, 2022
Merged

QGOV cleanup as per #57 #67

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
e442935
Use parameterised logging to reduce unnecessary string concatenation
ThrawnCA Jul 29, 2022
636aab9
Don't validate the entire package when updating a single resource
ThrawnCA Jul 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 7 additions & 7 deletions ckanext/validation/logic.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -310,9 +310,9 @@ def resource_validation_run_batch(context, data_dict):

except t.ValidationError as e:
log.warning(
u'Could not run validation for resource {} ' +
u'from dataset {}: {}'.format(
resource['id'], dataset['name'], str(e)))
u'Could not run validation for resource %s ' +
u'from dataset %s: %s',
resource['id'], dataset['name'], e)

if len(query['results']) < page_size:
break
Expand Down Expand Up @@ -487,7 +487,7 @@ def resource_create(context, data_dict):

for plugin in plugins.PluginImplementations(IDataValidation):
if not plugin.can_validate(context, data_dict):
log.debug('Skipping validation for resource {}'.format(resource_id))
log.debug('Skipping validation for resource %s', resource_id)
run_validation = False

if run_validation:
Expand Down Expand Up @@ -603,7 +603,7 @@ def resource_update(context, data_dict):
run_validation = True
for plugin in plugins.PluginImplementations(IDataValidation):
if not plugin.can_validate(context, data_dict):
log.debug('Skipping validation for resource {}'.format(id))
log.debug('Skipping validation for resource %s', id)
run_validation = False

if run_validation:
Expand Down Expand Up @@ -642,8 +642,8 @@ def _run_sync_validation(resource_id, local_upload=False, new_resource=True):
u'async': False})
except t.ValidationError as e:
log.info(
u'Could not run validation for resource {}: {}'.format(
resource_id, str(e)))
u'Could not run validation for resource %s: %s',
resource_id, e)
return

validation = t.get_action(u'resource_validation_show')(
Expand Down
25 changes: 21 additions & 4 deletions ckanext/validation/plugin/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ def before_create(self, context, data_dict):
return self._process_schema_fields(data_dict)

resources_to_validate = {}
packages_to_skip = {}

def after_create(self, context, data_dict):

Expand Down Expand Up @@ -188,7 +189,7 @@ def _handle_validation_for_resource(self, context, resource):

for plugin in p.PluginImplementations(IDataValidation):
if not plugin.can_validate(context, resource):
log.debug('Skipping validation for resource {}'.format(resource['id']))
log.debug('Skipping validation for resource %s', resource['id'])
return

_run_async_validation(resource[u'id'])
Expand All @@ -197,6 +198,15 @@ def before_update(self, context, current_resource, updated_resource):

updated_resource = self._process_schema_fields(updated_resource)

# the call originates from a resource API, so don't validate the entire package
package_id = updated_resource.get('package_id')
if not package_id:
existing_resource = t.get_action('resource_show')(
context={'ignore_auth': True}, data_dict={'id': updated_resource['id']})
if existing_resource:
package_id = existing_resource['package_id']
self.packages_to_skip[package_id] = True

if not get_update_mode_from_config() == u'async':
return updated_resource

Expand Down Expand Up @@ -242,6 +252,13 @@ def after_update(self, context, data_dict):
return

if is_dataset:
package_id = data_dict.get('id')
if self.packages_to_skip.pop(package_id, None) or context.get('save', False):
# Either we're updating an individual resource,
# or we're updating the package metadata via the web form;
# in both cases, we don't need to validate every resource.
return

for resource in data_dict.get(u'resources', []):
if resource[u'id'] in self.resources_to_validate:
# This is part of a resource_update call, it will be
Expand All @@ -259,7 +276,7 @@ def after_update(self, context, data_dict):
if resource_id in self.resources_to_validate:
for plugin in p.PluginImplementations(IDataValidation):
if not plugin.can_validate(context, data_dict):
log.debug('Skipping validation for resource {}'.format(data_dict['id']))
log.debug('Skipping validation for resource %s', data_dict['id'])
return

del self.resources_to_validate[resource_id]
Expand Down Expand Up @@ -299,8 +316,8 @@ def _run_async_validation(resource_id):
u'async': True})
except t.ValidationError as e:
log.warning(
u'Could not run validation for resource {}: {}'.format(
resource_id, str(e)))
u'Could not run validation for resource %s: %s',
resource_id, e)

def _get_underlying_file(wrapper):
if isinstance(wrapper, FlaskFileStorage):
Expand Down
2 changes: 1 addition & 1 deletion ckanext/validation/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,4 +70,4 @@ def delete_local_uploaded_file(resource_id):
os.rmdir(second_directory)

except OSError as e:
log.warning(u'Error deleting uploaded file: {}'.format(e))
log.warning(u'Error deleting uploaded file: %s', e)