Merge branch 'develop' into fix/schema-issues

aether-odk-module/aether/odk/api/tests/__init__.py

@@ -38,6 +38,8 @@
 XML_DATA_FILE_ERR = PATH_DIR + 'demo-data--error.xml'
 XML_DATA_FILE_ERR_MISSING_INSTANCE_ID = PATH_DIR + 'demo-data--error--missing-instance-id.xml'
 
+JSON_DATA_FILE = PATH_DIR + 'demo-data.json'
+
 XML_DATA = '''
     <h:html
         xmlns="http://www.w3.org/2002/xforms"
@@ -96,6 +98,7 @@ def setUp(self):
                 'file-ok': XML_DATA_FILE,
                 'file-err': XML_DATA_FILE_ERR,
                 'file-err-missing-instance-id': XML_DATA_FILE_ERR_MISSING_INSTANCE_ID,
+                'file-ok-json': JSON_DATA_FILE,
             },
 
             # sample collection for xForm objects

aether-odk-module/aether/odk/api/tests/files/demo-data.json

@@ -0,0 +1,46 @@
+{
+  "@id": "my-test-form",
+  "starttime": "2017-07-14T16:37:08.966+02",
+  "endtime": "2017-07-14T16:38:47.151+02",
+  "deviceid": "355217062209730",
+  "name": "Name",
+  "country": "CM",
+  "location": {
+    "type": "Point",
+    "coordinates": [
+      52.52469543,
+      13.39282687
+    ],
+    "altitude": 108.0,
+    "accuracy": 22.0
+  },
+  "number": 3,
+  "number2": 3.56,
+  "date": "2017-07-14T00:00:00",
+  "datetime": "2017-07-14T16:38:47.151000+02:00",
+  "choice": "a",
+  "a_choice": "A",
+  "b_choice": null,
+  "image": null,
+  "lang": "EN,FR",
+  "iterate": [
+    {
+      "index": "1"
+    },
+    {
+      "index": "2"
+    },
+    {
+      "index": "3"
+    }
+  ],
+  "iterate_one": [
+    {
+      "index": "one"
+    }
+  ],
+  "meta": {
+    "instanceID": "uuid:cef69d9d-ebd9-408f-8bc6-9d418bb083d9"
+  },
+  "not_in_the_definition": "Oops!"
+}

aether-odk-module/aether/odk/api/tests/files/demo-data.xml

@@ -25,6 +25,10 @@
   <iterate>
     <index>3</index>
   </iterate>
+  <iterate_one>
+    <index>one</index>
+  </iterate_one>
+  <not_in_the_definition>Oops!</not_in_the_definition>
   <meta>
     <instanceID>uuid:cef69d9d-ebd9-408f-8bc6-9d418bb083d9</instanceID>
   </meta>

aether-odk-module/aether/odk/api/tests/files/demo-xform.xml

@@ -71,6 +71,10 @@
           <iterate jr:template="">
             <index/>
           </iterate>
+          <iterate_one_count/>
+          <iterate_one jr:template="">
+            <index/>
+          </iterate_one>
           <meta>
             <instanceID/>
           </meta>
@@ -178,6 +182,8 @@
       <bind nodeset="/Something_that_is_not_None/lang" type="select"/>
       <bind calculate="3" nodeset="/Something_that_is_not_None/iterate_count" readonly="true()" type="string"/>
       <bind nodeset="/Something_that_is_not_None/iterate/index" type="string"/>
+      <bind calculate="1" nodeset="/Something_that_is_not_None/iterate_one_count" readonly="true()" type="string"/>
+      <bind nodeset="/Something_that_is_not_None/iterate_one/index" type="string"/>
       <bind
             calculate="concat('uuid:', uuid())"
             nodeset="/Something_that_is_not_None/meta/instanceID"
@@ -271,5 +277,13 @@
         </input>
       </repeat>
     </group>
+    <group ref="/Something_that_is_not_None/iterate_one">
+      <label></label>
+      <repeat jr:count=" /Something_that_is_not_None/iterate_one_count " nodeset="/Something_that_is_not_None/iterate_one">
+        <input ref="/Something_that_is_not_None/iterate_one/index">
+          <label>Index</label>
+        </input>
+      </repeat>
+    </group>
   </h:body>
 </h:html>

aether-odk-module/aether/odk/api/tests/test_xform_utils.py

@@ -16,6 +16,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+import json
+
 from django.core.exceptions import ValidationError
 from django.core.files.uploadedfile import SimpleUploadedFile
 
@@ -120,15 +122,19 @@ def test__parse_submission(self):
         with open(self.samples['submission']['file-ok'], 'rb') as xml:
             data, form_id, version = extract_data_from_xml(xml)
 
+        with open(self.samples['submission']['file-ok-json'], 'rb') as content:
+            expected = json.load(content)
+
         self.assertEqual(form_id, 'my-test-form')
         self.assertEqual(version, '0')
         self.assertEqual(len(list(data.keys())), 1)
         self.assertEqual(list(data.keys())[0], 'Something_that_is_not_None')
 
         data = parse_submission(data, self.samples['xform']['raw-xml'])
-
         self.assertNotEqual(list(data.keys())[0], 'Something_that_is_not_None')
 
+        self.assertEqual(data, expected)
+
     def test__get_instance_id(self):
         instance_id = 'abc'
         valid_data = {'meta': {'instanceID': instance_id}}

aether-odk-module/aether/odk/api/xform_utils.py

@@ -20,7 +20,6 @@
 import xmltodict
 
 from dateutil import parser
-from geojson import Point
 
 from pyxform import xls2json, builder
 from pyxform.xls2json_backends import xls_to_dict
@@ -68,7 +67,7 @@ def get_xml_title(data):
     try:
         # data is an `OrderedDict` object
         return data['h:html']['h:head']['h:title']
-    except Exception as e:
+    except Exception:
         return None
 
 
@@ -124,7 +123,7 @@ def get_xml_instance_attr(data, attr):
             key = list(instance.keys())[0]
             return instance[key][attr]
 
-    except Exception as e:
+    except Exception:
         pass
 
     return None
@@ -167,6 +166,17 @@ def extract_data_from_xml(xml):
     return data, form_id, version
 
 
+def get_instance_id(data):
+    '''
+    Extracts device instance id from xml data
+    '''
+
+    try:
+        return data['meta']['instanceID']
+    except Exception:
+        return None
+
+
 def parse_submission(data, xml_definition):
     '''
     Transforms and cleans the dictionary submission.
@@ -202,25 +212,42 @@ def walk(obj, parent_keys, coerce_dict):
 
         for k, v in obj.items():
             keys = parent_keys + [k]
+            xpath = '/' + '/'.join(keys)
+            _type = coerce_dict.get(xpath)
+
+            if _type == 'list' and not isinstance(v, list):
+                # list of one item but not presented as a list
+                # transform it back into a list
+                obj[k] = [v]
+
             if isinstance(v, dict):
                 walk(v, keys, coerce_dict)
+
             elif isinstance(v, list):
                 for i in v:
                     # indices are not important
                     walk(i, keys, coerce_dict)
+
             elif v is not None:
-                xpath = '/' + '/'.join(keys)
-                _type = coerce_dict.get(xpath)
+
                 if _type in ('int', 'integer'):
                     obj[k] = int(v)
+
                 if _type == 'decimal':
                     obj[k] = float(v)
+
                 if _type in ('date', 'dateTime'):
                     obj[k] = parser.parse(v).isoformat()
+
                 if _type == 'geopoint':
                     lat, lng, altitude, accuracy = v.split()
-                    # {"coordinates": [<<lat>>, <<lng>>], "type": "Point"}
-                    obj[k] = Point((float(lat), float(lng)))
+                    obj[k] = {
+                        'coordinates': [float(lat), float(lng)],
+                        'altitude': float(altitude),
+                        'accuracy': float(accuracy),
+                        'type': 'Point',
+                    }
+
             else:
                 obj[k] = None
 
@@ -232,11 +259,16 @@ def walk(obj, parent_keys, coerce_dict):
 
         try:
             coerce_dict[re_nodeset[0]] = re_type[0]
-        except Exception as e:
+        except Exception:
             # ignore, sometimes there is no "type"
             # <bind nodeset="/ZZZ/some_field" relevant=" /ZZZ/some_choice ='value'"/>
             pass
 
+    # repeat entries define the "list" fields
+    for repeat_entry in re.findall(r'<repeat.*>', xml_definition):
+        re_nodeset = re.findall(r'nodeset="([^"]*)"', repeat_entry)
+        coerce_dict[re_nodeset[0]] = 'list'
+
     walk(data, None, coerce_dict)  # modifies inplace
 
     # assumption: there is only one child that represents the form content
@@ -246,10 +278,3 @@ def walk(obj, parent_keys, coerce_dict):
         data = data[list(data.keys())[0]]
 
     return data
-
-
-def get_instance_id(data):
-    try:
-        return data['meta']['instanceID']
-    except Exception:
-        return None

aether-odk-module/conf/pip/primary-requirements.txt

@@ -8,27 +8,42 @@
 #                                                                              #
 ################################################################################
 
-django<2
-django-cas-ng
-django-cors-headers
-django-debug-toolbar
-django-ums-client
-djangorestframework
+# Main libraries
+# Aether common module, will also install:
+#   django                   < 2
+#   djangorestframework     >= 3.8   < 4
+#   djangorestframework-csv >= 2.0.0 < 3
+#   django-cors-headers     >= 2.0.0 < 3
+aether.common
+
+
+# Django specific
 drf-dynamic-fields
-geojson
-psycopg2-binary
-python-dateutil
-pytz
-pyxform
-raven
+psycopg2-binary        # Postgres library
 requests
 uwsgi
+
+
+# xForm and data manipulation
+python-dateutil
+pyxform
 xmltodict
 
-# Aether common module
-aether.common
 
-# test libraries
+# UMS libraries
+django-cas-ng
+django-ums-client
+
+
+# Development
+django-debug-toolbar
+
+
+# Server side
+raven                  # Sentry
+
+
+# Test libraries
 coverage
 flake8
 flake8-quotes

aether-odk-module/conf/pip/requirements.txt

@@ -10,10 +10,10 @@
 ################################################################################
 
 aether.common==0.0.0
-certifi==2018.1.18
+certifi==2018.4.16
 chardet==3.0.4
 coverage==4.5.1
-Django==1.11.12
+Django==1.11.13
 django-cas-ng==3.5.9
 django-cors-headers==2.2.0
 django-debug-toolbar==1.9.1
@@ -24,20 +24,19 @@ drf-dynamic-fields==0.3.0
 flake8==3.5.0
 flake8-quotes==1.0.0
 FormEncode==1.3.1
-geojson==2.3.0
 idna==2.6
 linecache2==1.0.0
 mccabe==0.6.1
 mock==2.0.0
-pbr==4.0.1
+pbr==4.0.2
 psycopg2-binary==2.7.4
 pycodestyle==2.3.1
 pyflakes==1.6.0
 python-cas==1.2.0
 python-dateutil==2.7.2
-pytz==2018.3
-pyxform==0.11.1
-raven==6.6.0
+pytz==2018.4
+pyxform==0.11.2
+raven==6.7.0
 requests==2.18.4
 six==1.11.0
 sqlparse==0.2.4