Add support for nested repeating_subfields

ckanext/scheming/assets/js/scheming-repeating-subfields.js

@@ -6,9 +6,15 @@ ckan.module('scheming-repeating-subfields', function($) {
       var $template = this.el.children('div[name="repeating-template"]');
       this.template = $template.html();
       $template.remove();
+      this._findClosestDescendants('a[name="repeating-add"]').on("click", this._onCreateGroup);
+      this._findClosestDescendants('a[name="repeating-remove"]').on('click', this._onRemoveGroup);
+    },
 
-      this.el.find('a[name="repeating-add"]').on("click", this._onCreateGroup);
-      this.el.on('click', 'a[name="repeating-remove"]', this._onRemoveGroup);
+    _findClosestDescendants: function(selector) {
+      const thisEl = this.el;
+      return thisEl.find(selector).filter(function(index) {
+        return this.closest('[data-module="scheming-repeating-subfields"]') === thisEl[0]
+      })
     },
 
     /**
@@ -26,19 +32,21 @@ ckan.module('scheming-repeating-subfields', function($) {
      *  ...
      */
     _onCreateGroup: function(e) {
-        var $last = this.el.find('.scheming-subfield-group').last();
-        var group = ($last.data('groupIndex') + 1) || 0;
-        var $copy = $(
+      var $last = this.el.find('.scheming-subfield-group').last();
+      var group = ($last.data('groupIndex') + 1) || 0;
+      var $copy = $(
 	    this.template.replace(/REPEATING-INDEX0/g, group)
-          .replace(/REPEATING-INDEX1/g, group + 1));
-        this.el.find('.scheming-repeating-subfields-group').append($copy);
+        .replace(/REPEATING-INDEX1/g, group + 1));
 
-	this.initializeModules($copy);
-        $copy.hide().show(100);
-        $copy.find('input').first().focus();
-        // hook for late init when required for rendering polyfills
-        this.el.trigger('scheming.subfield-group-init');
-        e.preventDefault();
+      this._findClosestDescendants('.scheming-repeating-subfields-group').append($copy);
+
+	  this.initializeModules($copy);
+      $copy.hide().show(100);
+      $copy.find('input').first().focus();
+      // hook for late init when required for rendering polyfills
+      this.el.trigger('scheming.subfield-group-init');
+      e.preventDefault();
+      e.stopPropagation();
     },
 
     /**

ckanext/scheming/helpers.py

@@ -5,6 +5,7 @@
 import pytz
 import json
 import six
+import copy
 
 from jinja2 import Environment
 from ckantoolkit import config, _
@@ -445,3 +446,42 @@ def scheming_flatten_subfield(subfield, data):
         for k in record:
             flat[prefix + k] = record[k]
     return flat
+
+def get_at_depth(data, path):
+    if len(path) == 0:
+        return data
+    if isinstance(data, dict):
+        found_data = data.get(path[0])
+    elif isinstance(data, list):
+        if isinstance(path[0], int) and path[0] < len(data):
+            found_data = data[path[0]]
+        else:
+            found_data = None
+    else:
+        found_data = None
+    if found_data is None:
+        return None
+    if len(path) > 1:
+        return get_at_depth(found_data, path[1:])
+    else:
+        return found_data
+
+def set_at_depth(data, path, value):
+    if len(path) == 0:
+        raise ValueError("Cannot set a value at an empty path")
+    data_to_modify = get_at_depth(data, path[0:len(path) - 1])
+    last_path = path[len(path)-1]
+    data_to_modify[last_path] = value
+    return data
+
+@helper
+def get_subfield_group_data(data, subfield_data_path):
+    subfield_data = get_at_depth(data, subfield_data_path)
+    if subfield_data is None:
+        return []
+    else:
+        return subfield_data
+
+@helper
+def deep_copy(data):
+    return copy.deepcopy(data)

ckanext/scheming/plugins.py

@@ -394,27 +394,50 @@ def expand_form_composite(data, fieldnames):
     fieldnames -= set(data)
     if not fieldnames:
         return
-    indexes = {}
-    for key in sorted(data):
-        if '-' not in key:
-            continue
-        parts = key.split('-')
-        if parts[0] not in fieldnames:
-            continue
-        if parts[1] not in indexes:
-            indexes[parts[1]] = len(indexes)
-        comp = data.setdefault(parts[0], [])
-        parts[1] = indexes[parts[1]]
-        try:
-            try:
-                comp[int(parts[1])]['-'.join(parts[2:])] = data[key]
-                del data[key]
-            except IndexError:
-                comp.append({})
-                comp[int(parts[1])]['-'.join(parts[2:])] = data[key]
-                del data[key]
-        except (IndexError, ValueError):
-            pass  # best-effort only
+
+    IDX_KEY = '__scheming_idx'
+
+    for fieldname in fieldnames:
+        keys_from_data = [key for key in data.keys() if key.startswith(fieldname)]
+        indexes = {}
+        field_data = []
+        for fieldname_key in sorted(keys_from_data):
+            if '-' not in fieldname_key:
+                continue
+            parts = fieldname_key.split('-')
+            path_to_field_data = parts[1:]
+            comp = field_data
+            parts_grouped = [[]]
+            for part in path_to_field_data:
+                last_part = parts_grouped[len(parts_grouped)-1]
+                if len(last_part) < 2:
+                    last_part.append(part)
+                    continue
+                else:
+                    parts_grouped.append([part])
+            stacked_keys = [fieldname]
+            data_to_set = comp
+
+            for parts_idx, part in enumerate(parts_grouped):
+                marked_index, part_key = part
+                if idx_map := helpers.get_at_depth(indexes, stacked_keys):
+                    marked_idx_map = idx_map.setdefault(marked_index, {IDX_KEY: len(idx_map)})
+                    idx = marked_idx_map[IDX_KEY]
+                else:
+                    idx = 0
+                    helpers.set_at_depth(indexes, stacked_keys, {marked_index: {IDX_KEY: idx}})
+                path_to_field_data[parts_idx*2] = idx
+                stacked_keys.append(marked_index)
+                stacked_keys.append(part_key)
+                if len(data_to_set) > idx:
+                    data_at_idx = data_to_set[idx]
+                else:
+                    data_at_idx = {}
+                    data_to_set.append(data_at_idx)
+                data_to_set = data_at_idx.setdefault(part_key, [])
+            helpers.set_at_depth(field_data, path_to_field_data, data[fieldname_key])
+            del data[fieldname_key]
+        data[fieldname] = field_data
 
 
 

ckanext/scheming/templates/scheming/form_snippets/repeating_subfields.html

@@ -17,9 +17,15 @@
       {% endblock %}
       <div class="panel-body fields-content">
         {% for subfield in field.repeating_subfields %}
+          {%- set new_subfield_data_path = h.deep_copy(subfield_data_path) -%}
+          {%- set _ = new_subfield_data_path.append(index) -%}
+          {%- set _ = new_subfield_data_path.append(subfield.field_name) -%}
+          {%- set is_template = 'REPEATING-INDEX' in index|string() -%}
           {% set sf = dict(
             subfield,
-            field_name=field.field_name ~ '-' ~ index ~ '-' ~ subfield.field_name)
+            field_name=field.field_name ~ '-' ~ index ~ '-' ~ subfield.field_name,
+            subfield_data_path=new_subfield_data_path,
+            is_repeating_template=is_template)
           %}
           {%- snippet 'scheming/snippets/form_field.html',
             field=sf,
@@ -46,6 +52,7 @@
 
 {% set flat = h.scheming_flatten_subfield(field, data) %}
 {% set flaterr = h.scheming_flatten_subfield(field, errors) %}
+{% set subfield_data_path = field.subfield_data_path or [field.field_name] %}
 
 {% call form.input_block(
   'field-' + field.field_name,
@@ -62,9 +69,11 @@
           </section>
         {% endif %}
 
-        {%- set group_data = data[field.field_name] -%}
-        {%- set group_count = group_data|length -%}
-        {%- if not group_count and 'id' not in data -%}
+        {%- if not field.is_repeating_template -%}
+          {%- set group_data = h.get_subfield_group_data(data, subfield_data_path) -%}
+          {%- set group_count = group_data|length -%}
+        {%- endif -%}
+        {%- if not group_count -%}
           {%- set group_count = field.form_blanks|default(1) -%}
         {%- endif -%}
 

ckanext/scheming/tests/test_form.py

@@ -390,6 +390,100 @@ def test_dataset_form_update(self, app, sysadmin_env):
         assert dataset["contact_address"] == [{'address': 'home'}]
 
 
+@pytest.mark.usefixtures("clean_db")
+class TestNestedSubfieldDatasetForm(object):
+    def test_dataset_form_includes_nested_subfields(self, app, sysadmin_env):
+        response = _get_package_new_page(app, sysadmin_env, 'test-subfields')
+        form = BeautifulSoup(response.body).select("#dataset-edit")[0]
+        assert form.select("fieldset[name=scheming-repeating-subfields] fieldset[name=scheming-repeating-subfields]")
+
+    def test_dataset_form_create(self, app, sysadmin_env):
+        data = {"save": "", "_ckan_phase": 1}
+
+        contact_points = [
+            {
+                'name': 'representative',
+                'ways_of_contact': [
+                    {
+                        'email': 'email@example.com',
+                        'by_letter': [
+                            {
+                                'name': 'some office',
+                                'address': 'some office address'
+                            },
+                            {
+                                'name': 'other office',
+                                'address': 'other office address'
+                            }
+                        ]
+                    }
+                ]
+            },
+            {
+                'name': 'second representative',
+                'ways_of_contact': [
+                    {
+                        'phone': '01234567'
+                    }
+                ]
+            }
+        ]
+
+        data["name"] = "nested_subfield_dataset_1"
+        data["contact_points-0-name"] = contact_points[0]['name']
+        data["contact_points-0-ways_of_contact-0-email"] = contact_points[0]['ways_of_contact'][0]['email']
+        data["contact_points-0-ways_of_contact-0-by_letter-0-name"] = contact_points[0]['ways_of_contact'][0]['by_letter'][0]['name']
+        data["contact_points-0-ways_of_contact-0-by_letter-0-address"] = contact_points[0]['ways_of_contact'][0]['by_letter'][0]['address']
+        data["contact_points-0-ways_of_contact-0-by_letter-1-name"] = contact_points[0]['ways_of_contact'][0]['by_letter'][1]['name']
+        data["contact_points-0-ways_of_contact-0-by_letter-1-address"] = contact_points[0]['ways_of_contact'][0]['by_letter'][1]['address']
+
+        data["contact_points-1-name"] = contact_points[1]['name']
+        data["contact_points-1-ways_of_contact-0-phone"] = contact_points[1]['ways_of_contact'][0]['phone']
+
+        url = '/test-subfields/new'
+
+        _post_data(app, url, data, sysadmin_env)
+
+        dataset = call_action("package_show", id="nested_subfield_dataset_1")
+        assert dataset["contact_points"] == contact_points
+
+    def test_dataset_form_update(self, app, sysadmin_env):
+        dataset = Dataset(
+            type="test-subfields",
+            contact_points=[
+                {'name': 'representative', 'ways_of_contact': [{'email': 'representative@example.com'}]},
+                {'name': 'second representative', 'ways_of_contact': [{'email': 'second.representative@example.com'}]}
+            ])
+
+        response = _get_package_update_page(
+            app, dataset["id"], sysadmin_env
+        )
+        form = BeautifulSoup(response.body).select_one("#dataset-edit")
+        assert form.select_one(
+            "input[name=contact_points-1-ways_of_contact-0-email]"
+        ).attrs['value'] == 'second.representative@example.com'
+
+        data = {"save": ""}
+        data["contact_points-0-name"] = 'representative'
+        data["contact_points-0-ways_of_contact-0-email"] = 'modified.representative@example.com'
+        data["contact_points-0-ways_of_contact-1-email"] = 'added.representative@example.com'
+        data["contact_points-1-name"] = 'second representative'
+        data["contact_points-1-ways_of_contact-0-email"] = 'second.representative@example.com'
+        data["name"] = dataset["name"]
+
+        url = '/test-subfields/edit/' + dataset["id"]
+
+        _post_data(app, url, data, sysadmin_env)
+
+        dataset = call_action("package_show", id=dataset["id"])
+
+        assert dataset["contact_points"] == [
+            {'name': 'representative', 'ways_of_contact': [{'email': 'modified.representative@example.com'},
+                                                           {'email': 'added.representative@example.com'}]},
+            {'name': 'second representative', 'ways_of_contact': [{'email': 'second.representative@example.com'}]}
+        ]
+
+
 
 @pytest.mark.usefixtures("clean_db")
 class TestSubfieldResourceForm(object):

ckanext/scheming/tests/test_subfields.yaml

@@ -44,6 +44,27 @@ dataset_fields:
   - field_name: country
     label: Country
 
+- field_name: contact_points
+  label: Contact Points
+  repeating_subfields:
+  - field_name: name
+    label: Name
+    required: true
+  - field_name: ways_of_contact
+    label: Ways of contact
+    repeating_subfields:
+    - field_name: email
+      label: Email
+    - field_name: phone
+      label: Phone
+    - field_name: by_letter
+      label: By Letter
+      repeating_subfields:
+      - field_name: name
+        label: name
+      - field_name: address
+        label: Address
+
 
 resource_fields: