Use frictionless framework v5

.github/workflows/test.yml

@@ -49,6 +49,8 @@ jobs:
       run: |
         pip install -r dev-requirements.txt
         pip install -r requirements.txt
+        pip install --no-warn-conflicts jinja2==2.10.1
+        pip install --no-warn-conflicts markupsafe==2.0.1
         pip install -e .
         # Replace default path to CKAN core config file with the one on the container
         sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini

README.md

@@ -36,7 +36,7 @@ Data description and validation for CKAN with [Frictionless Data](https://fricti
 
 ## Overview
 
-This extension brings data validation powered by the [Goodtables](https://github.com/frictionlessdata/goodtables-py) library to CKAN. It provides out of the box features to validate tabular data and integrate validation reports to the CKAN interface.
+This extension brings data validation powered by the [Frictionless Framework](https://github.com/frictionlessdata/framework) library to CKAN. It provides out of the box features to validate tabular data and integrate validation reports to the CKAN interface.
 
 Data validation can be performed automatically on the background or during dataset creation, and the results are stored against each resource.
 
@@ -51,7 +51,7 @@ If you are eager to get started, jump to the [Installation](#installation) and [
 
 ## Versions supported and requirements
 
-This extension is currently tested in CKAN 2.8 and CKAN 2.9.
+This extension is currently tested in CKAN 2.9.
 
 It is strongly recommended to use it alongside [ckanext-scheming](https://github.com/ckan/ckanext-scheming) to define the necessary extra fields in the default CKAN schema.
 
@@ -76,7 +76,6 @@ Once installed, add the `validation` plugin to the `ckan.plugins` configuration
 
     ckan.plugins = ... validation
 
-*Note:* if using CKAN 2.6 or lower and [asynchronous validation](#asynchronous-validation), also add the `rq` plugin ([see Versions supported and requirements](#versions-supported-and-requirements)) to `ckan.plugins`.
 
 ### Adding schema fields to the Resource metadata
 
@@ -108,7 +107,7 @@ You can also provide [validation options](#validation-options) that will be used
 
 	ckanext.validation.default_validation_options={
 	    "skip_checks": ["blank-rows", "duplicate-headers"],
-    	"headers": 3}
+    	}
 
 Make sure to use indentation if the value spans multiple lines otherwise it won't be parsed.
 
@@ -238,9 +237,6 @@ The following validation options would make validation pass:
 
 ```json
 {
-    "headers": 3,
-    "delimiter": ";",
-    "skip_rows": ["#"],
     "skip_checks": ["blank-rows"]
 }
 
@@ -327,7 +323,7 @@ The extension requires changes in the default CKAN resource metadata schema to a
 Here's more detail on the fields added:
 
 * `schema`: This can be a [Table Schema](http://frictionlessdata.io/specs/table-schema/) JSON object or an URL pointing to one. In the UI form you can upload a JSON file, link to one providing a URL or enter it directly. If uploaded, the file contents will be read and stored in the `schema` field. In all three cases the contents will be validated against the Table Schema specification.
-* `validation_options`: A JSON object with validation options that will be passed to [Goodtables](https://github.com/frictionlessdata/goodtables-py#validatesource-options).
+* `validation_options`: A JSON object with validation options that will be passed to Frictionless Framework [validate](https://framework.frictionlessdata.io/docs/guides/validating-data.html) function.
 
 ![Form fields](https://i.imgur.com/ixKOCij.png)
 

ckanext/validation/helpers.py

@@ -51,8 +51,12 @@ def validation_extract_report_from_errors(errors):
         if error == 'validation':
             report = errors[error][0]
             # Remove full path from table source
-            source = report['tables'][0]['source']
-            report['tables'][0]['source'] = source.split('/')[-1]
+            if 'tasks' in report:
+                source = report['tasks'][0]['place']
+                report['tasks'][0]['place'] = source.split('/')[-1]
+            elif 'tables' in report:
+                source = report['tables'][0]['source']
+                report['tables'][0]['source'] = source.split('/')[-1]
             msg = _('''
 There are validation issues with this file, please see the
 <a {params}>report</a> for details. Once you have resolved the issues,

ckanext/validation/jobs.py

@@ -7,7 +7,8 @@
 
 import requests
 from sqlalchemy.orm.exc import NoResultFound
-from goodtables import validate
+# from goodtables import validate
+from frictionless import validate, system, Report
 
 from ckan.model import Session
 import ckan.lib.uploader as uploader
@@ -22,7 +23,7 @@
 
 def run_validation_job(resource):
 
-    log.debug(u'Validating resource %s', resource['id'])
+    log.debug('Validating resource %s', resource['id'])
 
     try:
         validation = Session.query(Validation).filter(
@@ -33,18 +34,18 @@ def run_validation_job(resource):
     if not validation:
         validation = Validation(resource_id=resource['id'])
 
-    validation.status = u'running'
+    validation.status = 'running'
     Session.add(validation)
     Session.commit()
 
     options = t.config.get(
-        u'ckanext.validation.default_validation_options')
+        'ckanext.validation.default_validation_options')
     if options:
         options = json.loads(options)
     else:
         options = {}
 
-    resource_options = resource.get(u'validation_options')
+    resource_options = resource.get('validation_options')
     if resource_options and isinstance(resource_options, str):
         resource_options = json.loads(resource_options)
     if resource_options:
@@ -54,54 +55,62 @@ def run_validation_job(resource):
         {'ignore_auth': True}, {'id': resource['package_id']})
 
     source = None
-    if resource.get(u'url_type') == u'upload':
+    if resource.get('url_type') == 'upload':
         upload = uploader.get_resource_uploader(resource)
         if isinstance(upload, uploader.ResourceUpload):
-            source = upload.get_path(resource[u'id'])
+            source = upload.get_path(resource['id'])
         else:
             # Upload is not the default implementation (ie it's a cloud storage
             # implementation)
             pass_auth_header = t.asbool(
-                t.config.get(u'ckanext.validation.pass_auth_header', True))
-            if dataset[u'private'] and pass_auth_header:
+                t.config.get('ckanext.validation.pass_auth_header', True))
+            if dataset['private'] and pass_auth_header:
                 s = requests.Session()
                 s.headers.update({
-                    u'Authorization': t.config.get(
-                        u'ckanext.validation.pass_auth_header_value',
+                    'Authorization': t.config.get(
+                        'ckanext.validation.pass_auth_header_value',
                         _get_site_user_api_key())
                 })
 
-                options[u'http_session'] = s
+                options['http_session'] = s
 
     if not source:
-        source = resource[u'url']
-
-    schema = resource.get(u'schema')
-    if schema and isinstance(schema, str):
-        if schema.startswith('http'):
-            r = requests.get(schema)
-            schema = r.json()
+        source = resource['url']
+
+    schema = resource.get('schema')
+    if schema:
+        if isinstance(schema, str):
+            if schema.startswith('http'):
+                r = requests.get(schema)
+                schema = r.json()
         else:
-            schema = json.loads(schema)
+            schema = json.dumps(schema)
 
-    _format = resource[u'format'].lower()
+    _format = resource['format'].lower()
 
     report = _validate_table(source, _format=_format, schema=schema, **options)
 
     # Hide uploaded files
-    for table in report.get('tables', []):
-        if table['source'].startswith('/'):
-            table['source'] = resource['url']
-    for index, warning in enumerate(report.get('warnings', [])):
-        report['warnings'][index] = re.sub(r'Table ".*"', 'Table', warning)
-
-    if report['table-count'] > 0:
-        validation.status = u'success' if report[u'valid'] else u'failure'
-        validation.report = report
+    if type(report) == Report:
+        report = report.to_dict()
+    if 'tables' in report:
+        for table in report['tables']:
+            if table['source'].startswith('/'):
+                table['source'] = resource['url']
+    if 'warnings' in report:
+        for index, warning in enumerate(report['warnings']):
+            report['warnings'][index] = re.sub(r'Table ".*"', 'Table', warning)
+    if 'valid' in report and report['valid']:
+        validation.status = 'success' if report['valid'] else 'failure'
+        validation.report = json.dumps(report)
     else:
-        validation.status = u'error'
-        validation.error = {
-            'message': '\n'.join(report['warnings']) or u'No tables found'}
+        validation.status = 'error'
+        validation.report = json.dumps(report)
+        if 'tables' in report: 
+            validation.error = {
+                'message': [str(err) for err in report['tables'][0]['errors']] if len(report['tables'][0]['errors']) > 0 else 'No tables found'}
+        else:
+            validation.error = {'message': []}
     validation.finished = datetime.datetime.utcnow()
 
     Session.add(validation)
@@ -117,18 +126,21 @@ def run_validation_job(resource):
          'validation_timestamp': validation.finished.isoformat()})
 
 
-def _validate_table(source, _format=u'csv', schema=None, **options):
+def _validate_table(source, _format='csv', schema=None, **options):
 
+    frictionless_context = { 'trusted': True }
     http_session = options.pop('http_session', None) or requests.Session()
 
     use_proxy = 'ckan.download_proxy' in t.config
     if use_proxy:
         proxy = t.config.get('ckan.download_proxy')
-        log.debug(u'Download resource for validation via proxy: %s', proxy)
+        log.debug('Download resource for validation via proxy: %s', proxy)
         http_session.proxies.update({'http': proxy, 'https': proxy})
-    report = validate(source, format=_format, schema=schema, http_session=http_session, **options)
+        frictionless_context['http_session'] = http_session
 
-    log.debug(u'Validating source: %s', source)
+    with system.use_context(**frictionless_context):
+        report = validate(source, format=_format, schema=schema, **options)
+        log.debug('Validating source: %s', source)
 
     return report
 

ckanext/validation/logic.py

@@ -656,26 +656,31 @@ def _run_sync_validation(resource_id, local_upload=False, new_resource=True):
         {u'ignore_auth': True},
         {u'resource_id': resource_id})
 
-    report = validation['report']
+    if validation['report']:
+        report = json.loads(validation['report'])
 
-    if not report['valid']:
+        if not report['valid']:
 
-        # Delete validation object
-        t.get_action(u'resource_validation_delete')(
-            {u'ignore_auth': True},
-            {u'resource_id': resource_id}
-        )
+            # Delete validation object
+            t.get_action(u'resource_validation_delete')(
+                {u'ignore_auth': True},
+                {u'resource_id': resource_id}
+            )
 
-        # Delete uploaded file
-        if local_upload:
-            delete_local_uploaded_file(resource_id)
+            # Delete uploaded file
+            if local_upload:
+                delete_local_uploaded_file(resource_id)
 
-        if new_resource:
-            # Delete resource
-            t.get_action(u'resource_delete')(
-                {u'ignore_auth': True, 'user': None},
-                {u'id': resource_id}
-            )
+            if new_resource:
+                # Delete resource
+                t.get_action(u'resource_delete')(
+                    {u'ignore_auth': True, 'user': None},
+                    {u'id': resource_id}
+                )
 
+            raise t.ValidationError({
+                u'validation': [report]})
+    else:
         raise t.ValidationError({
-            u'validation': [report]})
+            'validation': []
+        })

ckanext/validation/templates/scheming/form_snippets/resource_schema.html

@@ -1,7 +1,6 @@
 {% import 'macros/form.html' as form %}
 
   {% set value = data[field.field_name] %}
-  {% set is_url = value and value[4:]|lower == 'http' %}
   {% set is_json = not is_url and value %}
 
   <div class="image-upload"

ckanext/validation/tests/test_form.py

@@ -80,7 +80,7 @@ def test_resource_form_create(self, app):
 
         dataset = call_action("package_show", id=dataset["id"])
 
-        assert json.loads(dataset["resources"][0]["schema"]) == value
+        assert dataset["resources"][0]["schema"] == value
 
     def test_resource_form_create_json(self, app):
         dataset = Dataset()
@@ -105,7 +105,7 @@ def test_resource_form_create_json(self, app):
 
         dataset = call_action("package_show", id=dataset["id"])
 
-        assert json.loads(dataset["resources"][0]["schema"]) == value
+        assert dataset["resources"][0]["schema"] == value
 
     def test_resource_form_create_upload(self, app):
         dataset = Dataset()
@@ -131,7 +131,7 @@ def test_resource_form_create_upload(self, app):
 
         dataset = call_action("package_show", id=dataset["id"])
 
-        assert json.loads(dataset["resources"][0]["schema"]) == value
+        assert dataset["resources"][0]["schema"] == value
 
     def test_resource_form_create_url(self, app):
         dataset = Dataset()
@@ -187,7 +187,7 @@ def test_resource_form_update(self, app):
 
         dataset = call_action("package_show", id=dataset["id"])
 
-        assert json.loads(dataset["resources"][0]["schema"]) == value
+        assert dataset["resources"][0]["schema"] == value
 
     def test_resource_form_update_json(self, app):
         value = {"fields": [{"name": "code"}, {"name": "department"}]}
@@ -216,7 +216,7 @@ def test_resource_form_update_json(self, app):
 
         dataset = call_action("package_show", id=dataset["id"])
 
-        assert json.loads(dataset["resources"][0]["schema"]) == value
+        assert dataset["resources"][0]["schema"] == value
 
     def test_resource_form_update_url(self, app):
         value = {"fields": [{"name": "code"}, {"name": "department"}]}
@@ -274,7 +274,7 @@ def test_resource_form_update_upload(self, app):
 
         dataset = call_action("package_show", id=dataset["id"])
 
-        assert json.loads(dataset["resources"][0]["schema"]) == value
+        assert dataset["resources"][0]["schema"] == value
 
 
 @pytest.mark.usefixtures("clean_db", "validation_setup")
@@ -312,7 +312,7 @@ def test_resource_form_create(self, app):
 
         dataset = call_action("package_show", id=dataset["id"])
 
-        assert json.loads(dataset["resources"][0]["validation_options"]) == value
+        assert dataset["resources"][0]["validation_options"] == value
 
     def test_resource_form_update(self, app):
         value = {
@@ -353,7 +353,7 @@ def test_resource_form_update(self, app):
 
         dataset = call_action("package_show", id=dataset["id"])
 
-        assert json.loads(dataset["resources"][0]["validation_options"]) == value
+        assert dataset["resources"][0]["validation_options"] == value
 
 
 @pytest.mark.usefixtures("clean_db", "validation_setup", "mock_uploads")
@@ -405,8 +405,8 @@ def test_resource_form_create_invalid(self, app):
         )
 
         assert "validation" in response.body
-        assert "missing-value" in response.body
-        assert "Row 2 has a missing value in column 4" in response.body
+        assert "missing-cell" in response.body
+        assert "This row has less values compared to the header row" in response.body
 
 
 @pytest.mark.usefixtures("clean_db", "validation_setup", "mock_uploads")
@@ -458,8 +458,8 @@ def test_resource_form_update_invalid(self, app):
         )
 
         assert "validation" in response.body
-        assert "missing-value" in response.body
-        assert "Row 2 has a missing value in column 4" in response.body
+        assert "missing-cell" in response.body
+        assert "This row has less values compared to the header row" in response.body
 
 
 @pytest.mark.usefixtures("clean_db", "validation_setup")