Merge pull request #24405 from dimagi/revert-24393-sk/api-indexed-on

Revert "standardize case and form API"

corehq/apps/api/es.py

@@ -674,8 +674,6 @@ def consume_params(self, raw_params):
     TermParam('xmlns', 'xmlns.exact'),
     DateRangeParams('received_on'),
     DateRangeParams('server_modified_on'),
-    DateRangeParams('date_modified'),
-    DateRangeParams('server_date_modified'),
     DateRangeParams('indexed_on'),
 ]
 

corehq/apps/api/resources/v0_3.py

@@ -1,22 +1,45 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
-
-import six
 from tastypie import fields
-from tastypie.exceptions import BadRequest
 
 from casexml.apps.case.models import CommCareCase
-from corehq.apps.api.es import es_search, ElasticAPIQuerySet
-from corehq.apps.api.models import ESCase
 from corehq.apps.api.resources import DomainSpecificResourceMixin
 from corehq.apps.api.resources import HqBaseResource
 from corehq.apps.api.resources.auth import RequirePermissionAuthentication
 from corehq.apps.api.resources.meta import CustomResourceMeta
 from corehq.apps.api.util import object_does_not_exist, get_obj
+from corehq.apps.cloudcare.api import es_filter_cases
+from corehq.apps.data_interfaces.forms import is_valid_case_property_name
 from corehq.apps.users.models import Permissions
 from corehq.form_processor.exceptions import CaseNotFound
 from corehq.form_processor.interfaces.dbaccessors import CaseAccessors
-from no_exceptions.exceptions import Http400
+
+
+class CaseListFilters(object):
+    format = 'json'
+
+    def __init__(self, params):
+
+        self.filters = dict((k, v) for k, v in params.items() if k and is_valid_case_property_name(k))
+
+        #hacky hack for v0.3.
+        #for v0.4, the API will explicitly require name and type
+        #for this version, magically behind the scenes override the query for case_name and case_type to be name, type
+        #note, on return output, the name will return as case_name, and type will return as case_type
+
+        if 'case_name' in self.filters:
+            self.filters['name'] = self.filters['case_name']
+            del(self.filters['case_name'])
+        if 'case_type' in self.filters:
+            self.filters['type'] = self.filters['case_type']
+            del(self.filters['case_type'])
+
+        if 'format' in self.filters:
+            self.format = self.filters['format']
+            del self.filters['format']
+
+        if 'order_by' in self.filters:
+            del self.filters['order_by']
 
 
 class CommCareCaseResource(HqBaseResource, DomainSpecificResourceMixin):
@@ -56,16 +79,8 @@ def obj_get(self, bundle, **kwargs):
             raise object_does_not_exist("CommCareCase", case_id)
 
     def obj_get_list(self, bundle, domain, **kwargs):
-        try:
-            es_query = es_search(bundle.request, domain)
-        except Http400 as e:
-            raise BadRequest(six.text_type(e))
-
-        return ElasticAPIQuerySet(
-            payload=es_query,
-            model=ESCase,
-            es_client=self.case_es(domain)
-        ).order_by('server_modified_on')
+        filters = CaseListFilters(bundle.request.GET)
+        return es_filter_cases(domain, filters=filters.filters)
 
     class Meta(CustomResourceMeta):
         authentication = RequirePermissionAuthentication(Permissions.edit_data)

corehq/apps/api/resources/v0_4.py

@@ -28,6 +28,7 @@
 from corehq.apps.app_manager.app_schemas.case_properties import get_case_properties
 from corehq.apps.app_manager.dbaccessors import get_apps_in_domain, get_all_built_app_results
 from corehq.apps.app_manager.models import Application, RemoteApp
+from corehq.apps.cloudcare.api import ElasticCaseQuery
 from corehq.apps.groups.models import Group
 from corehq.apps.users.models import CouchUser, Permissions
 from corehq.apps.users.util import format_username
@@ -261,9 +262,29 @@ def obj_get(self, bundle, **kwargs):
         return self.case_es(domain).get_document(case_id)
 
     def case_es(self, domain):
-        # Note that CaseES is used only as an ES client, for `run_query` against the proper index
         return MOCK_CASE_ES or CaseES(domain)
 
+    def obj_get_list(self, bundle, domain, **kwargs):
+        request_filters = {k: v for k, v in bundle.request.GET.items()}
+        request_filters.update(kwargs)
+        filters = v0_3.CaseListFilters(request_filters).filters
+
+        # Since tastypie handles the "from" and "size" via slicing, we have to wipe them out here
+        # since ElasticCaseQuery adds them. I believe other APIs depend on the behavior of ElasticCaseQuery
+        # hence I am not modifying that
+        query = ElasticCaseQuery(domain, filters).get_query()
+        if 'from' in query:
+            del query['from']
+        if 'size' in query:
+            del query['size']
+
+        # Note that CaseES is used only as an ES client, for `run_query` against the proper index
+        return ElasticAPIQuerySet(
+            payload=query,
+            model=ESCase,
+            es_client=self.case_es(domain)
+        ).order_by('server_modified_on')
+
     class Meta(v0_3.CommCareCaseResource.Meta):
         max_limit = 1000
         serializer = CommCareCaseSerializer()
@@ -475,9 +496,23 @@ def obj_get_list(self, bundle, domain, **kwargs):
         """
         Overridden to wrap the case JSON from ElasticSearch with the custom.hope.case.HOPECase class
         """
-        queryset = super(HOPECaseResource, self).obj_get_list(bundle, domain, **kwargs)
-        queryset.model = HOPECase
-        return queryset
+        filters = v0_3.CaseListFilters(bundle.request.GET).filters
+
+        # Since tastypie handles the "from" and "size" via slicing, we have to wipe them out here
+        # since ElasticCaseQuery adds them. I believe other APIs depend on the behavior of ElasticCaseQuery
+        # hence I am not modifying that
+        query = ElasticCaseQuery(domain, filters).get_query()
+        if 'from' in query:
+            del query['from']
+        if 'size' in query:
+            del query['size']
+
+        # Note that CaseES is used only as an ES client, for `run_query` against the proper index
+        return ElasticAPIQuerySet(
+            payload=query,
+            model=HOPECase,
+            es_client=self.case_es(domain),
+        ).order_by('server_modified_on')
 
     def alter_list_data_to_serialize(self, request, data):
 

corehq/apps/cleanup/management/commands/local_commcare_export.py

@@ -15,8 +15,10 @@
 
 from corehq.apps.api.es import ElasticAPIQuerySet, CaseES, es_search_by_params, XFormES
 from corehq.apps.api.models import ESCase, ESXFormInstance
+from corehq.apps.api.resources.v0_3 import CaseListFilters
 from corehq.apps.api.resources.v0_4 import CommCareCaseResource, XFormInstanceResource
 from corehq.apps.api.serializers import CommCareCaseSerializer, XFormInstanceSerializer
+from corehq.apps.cloudcare.api import ElasticCaseQuery
 from corehq.elastic import ESError
 
 
@@ -28,9 +30,15 @@ def serialize(self, obj):
 
 def _get_case_mock(project, params):
     # this is mostly copy/paste/modified from CommCareCaseResource
-    es_query = es_search_by_params(params, project)
+    filters = CaseListFilters(params).filters
+    query = ElasticCaseQuery(project, filters).get_query()
+    if 'from' in query:
+        del query['from']
+    if 'size' in query:
+        del query['size']
+
     query_set = ElasticAPIQuerySet(
-        payload=es_query,
+        payload=query,
         model=ESCase,
         es_client=CaseES(project),
     ).order_by('server_modified_on')

corehq/apps/cloudcare/api.py

@@ -1,12 +1,129 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
+from datetime import datetime
+from django.utils.translation import ugettext as _
 
 from couchdbkit.exceptions import ResourceNotFound
-from django.utils.translation import ugettext as _
+
+from casexml.apps.case.models import CommCareCase
+from dimagi.utils.parsing import json_format_date
 
 from corehq.apps.app_manager.dbaccessors import get_app
 from corehq.apps.cloudcare.dbaccessors import get_cloudcare_apps
 from corehq.apps.cloudcare.exceptions import RemoteAppError
+from corehq.elastic import get_es_new, ES_META
+
+
+CLOUDCARE_API_DATETIME_FORMAT = '%Y-%m-%dT%H:%M:%S'  # todo: add '.%fZ'?
+
+
+class ElasticCaseQuery(object):
+    # this class is currently pretty customized to serve exactly
+    # this API. one day it may be worth reconciling our ES interfaces
+    # but today is not that day.
+    # To be replaced by CaseES framework.
+    RESERVED_KEYS = ('date_modified_start', 'date_modified_end', 
+                     'server_date_modified_start', 'server_date_modified_end', 
+                     'limit', 'offset')
+
+    def __init__(self, domain, filters):
+        self.domain = domain
+        self.filters = filters
+        self.offset = int(filters.get('offset', 0))
+        self.limit = int(filters.get('limit', 50))
+        self._date_modified_start = filters.get("date_modified_start", None)
+        self._date_modified_end = filters.get("date_modified_end", None)
+        self._server_date_modified_start = filters.get("server_date_modified_start", None)
+        self._server_date_modified_end = filters.get("server_date_modified_end", None)
+
+    @property
+    def uses_modified(self):
+        return bool(self._date_modified_start or self._date_modified_end)
+
+    @property
+    def uses_server_modified(self):
+        return bool(self._server_date_modified_start or self._server_date_modified_end)
+
+    @property
+    def date_modified_start(self):
+        return self._date_modified_start or json_format_date(datetime(1970, 1, 1))
+
+    @property
+    def date_modified_end(self):
+        return self._date_modified_end or json_format_date(datetime.max)
+
+    @property
+    def server_date_modified_start(self):
+        return self._server_date_modified_start or json_format_date(datetime(1970, 1, 1))
+
+    @property
+    def server_date_modified_end(self):
+        return self._server_date_modified_end or json_format_date(datetime.max)
+
+    @property
+    def scrubbed_filters(self):
+        return dict( (k, v) for k, v in self.filters.items()
+                     if k not in self.RESERVED_KEYS and not k.endswith('__full') )
+
+    def _modified_params(self, key, start, end):
+        return {
+            'range': {
+                key: {
+                    'from': start,
+                    'to': end
+                }
+            }
+        }
+
+    @property
+    def modified_params(self, ):
+        return self._modified_params('modified_on',
+                                     self.date_modified_start,
+                                     self.date_modified_end)
+
+    @property
+    def server_modified_params(self):
+        return self._modified_params('server_modified_on',
+                                     self.server_date_modified_start,
+                                     self.server_date_modified_end)
+
+    def get_terms(self):
+        yield {'term': {'domain.exact': self.domain}}
+        if self.uses_modified:
+            yield self.modified_params
+        if self.uses_modified:
+            yield self.modified_params
+        if self.uses_server_modified:
+            yield self.server_modified_params
+        for k, v in self.scrubbed_filters.items():
+            yield {'term': {k: v.lower()}}
+
+    def get_query(self):
+        return {
+            'query': {
+                'bool': {
+                    'must': list(self.get_terms())
+                }
+            },
+            'sort': {
+                'modified_on': {'order': 'asc'}
+            },
+            'from': self.offset,
+            'size': self.offset + self.limit,
+        }
+
+
+def es_filter_cases(domain, filters=None):
+    """
+    Filter cases using elastic search
+    (Domain, Filters?) -> [CommCareCase]
+    """
+    q = ElasticCaseQuery(domain, filters)
+    meta = ES_META['cases']
+    res = get_es_new().search(meta.index, body=q.get_query())
+    # this is ugly, but for consistency / ease of deployment just
+    # use this to return everything in the expected format for now
+    return [CommCareCase.wrap(r["_source"]) for r in res['hits']['hits'] if r["_source"]]
 
 
 def get_app_json(app):