Merge pull request #371 from hackforla/IngestTweaks

Tweaked ingest endpoint to expose limit at the api level
hackforla · Mar 6, 2020 · 56a9012 · 56a9012
2 parents bb966bf + de109b2
commit 56a9012
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 6 deletions.
diff --git a/server/src/app.py b/server/src/app.py
@@ -83,6 +83,7 @@ async def ingest(request):
         Ex. '/ingest?years=2015,2016,2017'
     """
     current_year = datetime.now().year
+    limit = request.args.get("limit", None)
     ALLOWED_YEARS = [year for year in range(2015, current_year+1)]
     if not request.args.get("years"):
         return json({"error": "'years' parameter is required."})
@@ -91,7 +92,7 @@ async def ingest(request):
         return json({"error":
                     f"'years' param values must be one of {ALLOWED_YEARS}"})
     loader = DataHandler(app.config['Settings'])
-    loader.populateFullDatabase(yearRange=years)
+    loader.populateFullDatabase(yearRange=years, limit=limit)
     return_data = {'response': 'ingest ok'}
     return json(return_data)
 

diff --git a/server/src/services/databaseOrm.py b/server/src/services/databaseOrm.py
@@ -59,23 +59,23 @@ class Ingest(Base):
                 'closeddate': String(30),
                 'addressverified': String(16),
                 'approximateaddress': String(20),
-                'address': String(100),
+                'address': String(250),
                 'housenumber': String(10),
                 'direction': String(10),
                 'streetname': String(50),
                 'suffix': String(10),
                 'zipcode': Integer,
                 'latitude': Float,
                 'longitude': Float,
-                'location': String(100),
+                'location': String(250),
                 'tbmpage': Integer,
                 'tbmcolumn': String(10),
                 'tbmrow': Float,
                 'apc': String(30),
                 'cd': Float,
                 'cdmember': String(30),
                 'nc': Float,
-                'ncname': String(100),
+                'ncname': String(30),
                 'policeprecinct': String(30)}
 
 

diff --git a/server/src/services/sqlIngest.py b/server/src/services/sqlIngest.py
@@ -93,6 +93,7 @@ def ingestData(self, ingestMethod='replace',
                     schema='public',
                     index=False,
                     chunksize=10,
+                    method='multi',
                     dtype=self.insertParams)
         print('\tIngest Complete: %.1f minutes' %
               self.elapsedTimer(ingestTimer))
@@ -162,7 +163,7 @@ def fetchSocrataFull(self, year=2019, limit=10**7):
         print('\tDownload Complete: %.1f minutes' %
               self.elapsedTimer(downloadTimer))
 
-    def populateFullDatabase(self, yearRange=range(2015, 2021)):
+    def populateFullDatabase(self, yearRange=range(2015, 2021), limit=None):
         '''Fetches all data from Socrata to populate database
            Default operation is to fetch data from 2015-2020
            !!! Be aware that each fresh import will wipe the
@@ -171,7 +172,7 @@ def populateFullDatabase(self, yearRange=range(2015, 2021)):
         tableInit = False
         globalTimer = time.time()
         for y in yearRange:
-            self.fetchSocrataFull(year=y)
+            self.fetchSocrataFull(year=y, limit=limit)
             self.cleanData()
             if not tableInit:
                 self.ingestData(ingestMethod='replace')