Merge branch 'master' into canada-v2.10

# Conflicts:
#	.circleci/config.yml
#	ckanext/recombinant/views.py
### RESOLVED.

.circleci/config.yml

@@ -61,6 +61,14 @@ jobs:
             python3 -m ensurepip
             pip3 install --no-cache --upgrade pip setuptools
             python3 --version
+      - run:
+          name: Set CKAN to open-data/ckan@canada-v2.10
+          command: |
+            pip uninstall -y ckan
+            git --git-dir=\/srv\/app\/src\/ckan\/.git --work-tree=\/srv\/app\/src\/ckan\/ remote add canada https://github.com/open-data/ckan.git
+            git --git-dir=\/srv\/app\/src\/ckan\/.git --work-tree=\/srv\/app\/src\/ckan\/ fetch canada canada-v2.10
+            git --git-dir=\/srv\/app\/src\/ckan\/.git --work-tree=\/srv\/app\/src\/ckan\/ checkout -b canada-v2.10 canada/canada-2.10
+            git --git-dir=\/srv\/app\/src\/ckan\/.git --work-tree=\/srv\/app\/src\/ckan\/ pull
       - run:
           name: Install requirements
           command: |

.gitignore

@@ -1,3 +1,4 @@
 ckan.log
 *.pyc
 *.egg-info/
+*.mo

changes/0004.bugfix

@@ -0,0 +1 @@
+Excel reference sheets now display the correct field number for the resource fields. Excel DataDictionary reference sheets now display the correct resource titles.

changes/0005.feature

@@ -0,0 +1 @@
+Added new published DataDictionary XLSX and Schema JSON routes to include published fields.

changes/0006.bugfix

@@ -0,0 +1 @@
+Fixed an issue with Activity Streams and purged packages.

changes/127.a.changes

@@ -0,0 +1 @@
+Added `format_trigger_error` method to format Database errors in a pythonian way to support language translations in the CKAN framework.

changes/127.b.changes

@@ -0,0 +1 @@
+The DataDictionary form will now show the Recombinant resource's schema as readonly fields. The DataDictionary front-end will now show non-schema fields such as any additional fields added by filter scripts, allowing normal usage of DataDictionary for these fields.

changes/127.c.changes

@@ -0,0 +1 @@
+The Occurence from field schemas will now display in the Excel reference sheet and in the DataDictionary front-end.

changes/127.d.bugfix

@@ -0,0 +1 @@
+Fixed an issue with full text choices not spanning 2 columns in the Excel reference sheet.

changes/127.e.feature

@@ -0,0 +1 @@
+Recombinant field schemas now support `max_chars`, limitting the field value to a character count with an excel formula. Use the `max_char_error` Database trigger/function from the Canada extension.

changes/127.f.feature

@@ -0,0 +1 @@
+The `datastore_info` action method now shows the field schema in the field info for Recombinant resources and Published resources.

changes/127.feature

@@ -0,0 +1 @@
+Recombinant chromos now support DataStore foreign key constraints. The Recombinant plugin limits all foreign key usages to Recombinant resources.

changes/127.g.changes

@@ -0,0 +1 @@
+Uploading an Excel file now requires all worksheets to pass validation for the records to be committed to the database.

changes/128.bugfix

@@ -0,0 +1 @@
+Fixed language switching in Flask requests that was affecting the XLSX DataDictionary and JSON Schema downloads.

ckanext/recombinant/cli.py

@@ -373,6 +373,8 @@ def _delete(dataset_types, all_types=False, verbose=False):
                         resource_id=r['id'])
                 except NotFound:
                     pass
+                except ValidationError as e:
+                    click.echo('failed %s %s -- %s' % (dtype, p['owner_org'], str(e.error_dict)), err=True)
             lc.action.package_delete(id=p['id'])
 
 
@@ -430,7 +432,7 @@ def _load_one_csv_file(name):
 
         # convert list values to lists
         list_fields = [f['datastore_id']
-            for f in chromo['fields'] if f['datastore_type'] == '_text']
+            for f in chromo['fields'] if f['datastore_type'] == '_text' and not f.get('published_resource_computed_field')]
         if list_fields:
             for r in records:
                 for k in list_fields:
@@ -510,7 +512,7 @@ def _combine_csv(target_dir, resource_names, all_types=False, verbose=False):
 
 def _write_one_csv(lc, pkgs, chromo, outfile):
     out = csv.writer(outfile)
-    column_ids = [f['datastore_id'] for f in chromo['fields']
+    column_ids = [f['datastore_id'] for f in chromo['fields'] if not f.get('published_resource_computed_field')
         ] + chromo.get('csv_org_extras', []) + [
         'owner_org', 'owner_org_title']
     out.writerow(column_ids)

ckanext/recombinant/errors.py

@@ -1,3 +1,4 @@
+from ckan.plugins.toolkit import _
 
 class RecombinantException(Exception):
     pass
@@ -8,3 +9,17 @@ class RecombinantConfigurationError(Exception):
 class BadExcelData(Exception):
     def __init__(self, message):
         self.message = message
+
+
+def format_trigger_error(error_values: list):
+    """
+    Format PSQL function errors from raised ValidationError exceptions.
+
+    This method will split the error messages on a unicode private code point (\\uF8FF)
+    in order to do string replacements, allowing i18n support in the framework.
+    """
+    for e in error_values:
+        if '\uF8FF' in e:
+            yield _(e.split('\uF8FF')[0]).format(e.split('\uF8FF')[1])
+        else:
+            yield _(e)