Merge pull request #19 from prio-data/integrate_xform

Integrate xform

Dockerfile

@@ -1,4 +1,4 @@
-FROM views3/uvicorn-deployment:2.1.0
+FROM views3/uvicorn-deployment:p39
 
 COPY ./requirements.txt /
 RUN pip install -r requirements.txt 

alembic.ini

@@ -41,7 +41,6 @@ prepend_sys_path = .
 
 sqlalchemy.url = driver://user:pass@localhost/dbname
 
-
 [post_write_hooks]
 # post_write_hooks defines scripts or Python functions that are run
 # on newly generated revision scripts.  See the documentation for further

queryset_manager/app.py

@@ -8,6 +8,7 @@
 import fastapi
 import views_schema as schema
 import aiohttp
+import requests
 
 from . import crud, models, db, remotes, settings, data_retriever
 
@@ -32,10 +33,6 @@ def get_session():
     finally:
         sess.close()
 
-remotes_api = remotes.Api(
-        source_url = os.path.join(settings.JOB_MANAGER_URL,"job")
-        )
-
 @app.get("/")
 def handshake():
     """
@@ -58,12 +55,39 @@ async def queryset_data(
     if queryset is None:
         return Response(status_code=404)
 
-    async with aiohttp.ClientSession() as http_session:
-        retriever = data_retriever.DataRetriever(
-                settings.JOB_MANAGER_URL+"/job",
-                http_session)
-        status_code, content = await retriever.queryset_data_response(queryset)
-        return Response(content, status_code = status_code)
+    full_qs_dict = queryset.dict()
+
+    qs_dict={}
+    qs_dict['name'] = full_qs_dict['name']
+    qs_dict['to_loa'] = full_qs_dict['loa']
+    qs_paths = []
+    operations = full_qs_dict['operations']
+    for operation_path in operations:
+        path = ''
+        for operation in operation_path:
+            path = path + '/' + operation['namespace']
+            path = path + '/' + operation['name']
+            args = operation['arguments']
+            if len(args) == 0:
+                path = path + '/_'
+#            elif len(args) == 1:
+#                path = path + '/' + args[0]
+            else:
+                path = path + '/'
+                for arg in args:
+                    path = path + arg + '__'
+                path = path[:-2]
+        qs_paths.append(path)
+    qs_dict['paths'] = qs_paths
+
+    logger.debug("dict %s", qs_dict)
+
+    response = requests.get(settings.DATA_SERVICE_URL+'/queryset/', json=qs_dict)
+
+    content = response.content
+    status_code = response.status_code
+
+    return Response(content, status_code=status_code)
 
 @app.get("/querysets/{queryset}")
 def queryset_detail(queryset:str, session = Depends(get_session)):

queryset_manager/data_retriever.py

@@ -72,7 +72,8 @@ async def fetch_dataframe(self, queryset: models.Queryset)-> Either[List[respons
 
     def _url_from_path(self, path: str) -> str:
         """
-        _url
+        _url_from_path
+
         ====
 
         parameters:

queryset_manager/models.py

@@ -189,7 +189,7 @@ def operation_path(self):
             args = ["_"]
         else:
             args = self.arguments
-        components.append("_".join([str(a) for a in args]))
+        components.append("__".join([str(a) for a in args]))
 
         return os.path.join(*components)
 

queryset_manager/settings.py

@@ -6,12 +6,12 @@
 
 DB_HOST                    = env.str("QUERYSET_MANAGER_DB_HOST", "127.0.0.1")
 DB_PORT                    = env.int("QUERYSET_MANAGER_DB_PORT", 5432)
-DB_NAME                    = env.str("QUERYSET_MANAGER_DB_NAME", "postgres")
-DB_USER                    = env.str("QUERYSET_MANAGER_DB_USER", "postgres")
+DB_NAME                    = env.str("QUERYSET_MANAGER_DB_NAME", "fallback3")
+DB_USER                    = env.str("QUERYSET_MANAGER_DB_USER", "views_data")
 DB_SCHEMA: Optional[str]   = env.str("QUERYSET_MANAGER_DB_SCHEMA", None)
 DB_PASSWORD: Optional[str] = env.str("QUERYSET_MANAGER_DB_PASSWORD", None)
 DB_SSL                     = env.str("QUERYSET_MANAGER_DB_SSL", "allow")
 
 LOG_LEVEL                  = env.str("LOG_LEVEL", "WARNING")
 
-JOB_MANAGER_URL            = env.str("JOB_MANAGER_URL", "http://job-manager")
+DATA_SERVICE_URL           = env.str("DATA_SERVICE_URL", "http://data-service")

requirements.txt

@@ -11,5 +11,4 @@ httpretty>=1.0.5
 toolz>=0.11.1
 views-schema==2.1.0
 aiohttp==3.7.4.post0
-fast-views~=0.1.0
 PyMonad==2.4.0