From 910e8ade5dc4f963445f9f982985d7551c959910 Mon Sep 17 00:00:00 2001 From: Jake Mensch Date: Thu, 30 Apr 2020 14:18:51 -0700 Subject: [PATCH 1/3] /pin-clusters with redis --- Orchestration/docker-compose-example.yml | 16 ++ redis/Dockerfile | 3 + redis/redis.conf | 43 +++ server/Dockerfile | 2 +- .../PinsComparisonTest.jmx | 270 ++++++++++++++++++ .../pins-comparison/pins-comparison.jtl | 181 ++++++++++++ server/requirements.txt | 2 + server/src/app.py | 22 ++ server/src/services/pinClusterService.py | 81 ++++++ server/src/settings.example.cfg | 4 + server/src/utils/redis.py | 34 +++ server/src/utils/sanic.py | 11 +- 12 files changed, 664 insertions(+), 5 deletions(-) create mode 100644 redis/Dockerfile create mode 100644 redis/redis.conf create mode 100644 server/performanceStatistics/PinsComparisonTest.jmx create mode 100644 server/performanceStatistics/pins-comparison/pins-comparison.jtl create mode 100644 server/src/services/pinClusterService.py create mode 100644 server/src/utils/redis.py diff --git a/Orchestration/docker-compose-example.yml b/Orchestration/docker-compose-example.yml index 8d740d56a..f74ab7761 100644 --- a/Orchestration/docker-compose-example.yml +++ b/Orchestration/docker-compose-example.yml @@ -47,5 +47,21 @@ services: container_name: "postgres-dashboard" ports: - 8080:8080 + + redis: + build: + context: ../redis + container_name: "311-redis" + expose: + - 6379 + + rebrow: + image: marian/rebrow + links: + - redis:redis + container_name: "redis-dashboard" + ports: + - 5001:5001 + volumes: backend_data: diff --git a/redis/Dockerfile b/redis/Dockerfile new file mode 100644 index 000000000..bd09a8b85 --- /dev/null +++ b/redis/Dockerfile @@ -0,0 +1,3 @@ +FROM redis +COPY redis.conf /usr/local/etc/redis/redis.conf +CMD [ "redis-server", "/usr/local/etc/redis/redis.conf" ] diff --git a/redis/redis.conf b/redis/redis.conf new file mode 100644 index 000000000..1d89a281b --- /dev/null +++ b/redis/redis.conf @@ -0,0 +1,43 @@ +# full example config here: +# http://download.redis.io/redis-stable/redis.conf + +# discussion of memory policies here: +# https://redis.io/topics/lru-cache + +# how to check that this config is working inside docker: +# 1. login to container: `docker exec -it 311-redis /bin/bash` +# 2. start the redis cli: `redis-cli` +# 3. check the config: `config get maxmemory-policy` + +# MAXMEMORY POLICY: how Redis will select what to remove when maxmemory +# is reached. You can select one from the following behaviors: +# +# volatile-lru -> Evict using approximated LRU, only keys with an expire set. +# allkeys-lru -> Evict any key using approximated LRU. +# volatile-lfu -> Evict using approximated LFU, only keys with an expire set. +# allkeys-lfu -> Evict any key using approximated LFU. +# volatile-random -> Remove a random key having an expire set. +# allkeys-random -> Remove a random key, any key. +# volatile-ttl -> Remove the key with the nearest expire time (minor TTL) +# noeviction -> Don't evict anything, just return an error on write operations. +# +# LRU means Least Recently Used +# LFU means Least Frequently Used +# +# Both LRU, LFU and volatile-ttl are implemented using approximated +# randomized algorithms. +# +# Note: with any of the above policies, Redis will return an error on write +# operations, when there are no suitable keys for eviction. +# +# At the date of writing these commands are: set setnx setex append +# incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd +# sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby +# zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby +# getset mset msetnx exec sort +# +# The default is: +# +# maxmemory-policy noeviction + +maxmemory-policy volatile-lfu diff --git a/server/Dockerfile b/server/Dockerfile index 4c322d588..e968da4ea 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.7-slim RUN apt-get update && apt-get install -yq \ - python3 python3-dev gcc \ + python3 python3-dev gcc g++ \ gfortran musl-dev ENV DB_CONNECTION_STRING=REDACTED diff --git a/server/performanceStatistics/PinsComparisonTest.jmx b/server/performanceStatistics/PinsComparisonTest.jmx new file mode 100644 index 000000000..1b6cbbc81 --- /dev/null +++ b/server/performanceStatistics/PinsComparisonTest.jmx @@ -0,0 +1,270 @@ + + + + + Compares /pins with /pin-clusters for 3 different request sizes + false + false + + + + server + 0529ed2e.ngrok.io + = + + + port + 80 + = + + + concurrentUsers + 3 + = + + + iterations + 10 + = + + + startDate + 2019-01-01 + = + + + endDate + 2020-01-01 + = + + + requestTypes + ["Bulky Items","Graffiti Removal","Homeless Encampment","Illegal Dumping Pickup","Dead Animal Removal","Electronic Waste","Single Streetlight Issue","Multiple Streetlight Issue","Metal/Household Appliances","Feedback","Other"] + = + + + ncs_small + [4] + = + + + ncs_medium + [4, 10, 20, 24, 38, 39, 50, 53, 58, 62] + = + + + ncs_large + [4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 32, 33, 34, 36, 37, 38, 39, 40, 41, 42, 43, 44, 46, 47, 48, 50, 52, 53, 54, 55, 58, 60, 61, 62] + = + + + + + + + + continue + + false + ${iterations} + + ${concurrentUsers} + 1 + 1587331449000 + 1587331449000 + false + + + true + + + + true + + + + false + { + "startDate":"${startDate}", + "endDate":"${endDate}", + "ncList": ${ncs_small}, + "requestTypes": ${requestTypes} +} + = + + + + ${server} + ${port} + + + /pins + POST + true + false + true + false + + + + + + + true + + + + false + { + "startDate":"${startDate}", + "endDate":"${endDate}", + "ncList": ${ncs_small}, + "requestTypes": ${requestTypes} +} + = + + + + ${server} + ${port} + + + /pin-clusters + POST + true + false + true + false + + + + + + + true + + + + false + { + "startDate":"${startDate}", + "endDate":"${endDate}", + "ncList": ${ncs_medium}, + "requestTypes": ${requestTypes} +} + = + + + + ${server} + ${port} + + + /pins + POST + true + false + true + false + + + + + + + true + + + + false + { + "startDate":"${startDate}", + "endDate":"${endDate}", + "ncList": ${ncs_medium}, + "requestTypes": ${requestTypes} +} + = + + + + ${server} + ${port} + + + /pin-clusters + POST + true + false + true + false + + + + + + + true + + + + false + { + "startDate":"${startDate}", + "endDate":"${endDate}", + "ncList": ${ncs_large}, + "requestTypes": ${requestTypes} +} + = + + + + ${server} + ${port} + + + /pins + POST + true + false + true + false + + + + + + + true + + + + false + { + "startDate":"${startDate}", + "endDate":"${endDate}", + "ncList": ${ncs_large}, + "requestTypes": ${requestTypes} +} + = + + + + ${server} + ${port} + + + /pin-clusters + POST + true + false + true + false + + + + + + + + + diff --git a/server/performanceStatistics/pins-comparison/pins-comparison.jtl b/server/performanceStatistics/pins-comparison/pins-comparison.jtl new file mode 100644 index 000000000..52d4ad819 --- /dev/null +++ b/server/performanceStatistics/pins-comparison/pins-comparison.jtl @@ -0,0 +1,181 @@ +timeStamp,elapsed,label,responseCode,responseMessage,threadName,dataType,success,failureMessage,bytes,sentBytes,grpThreads,allThreads,URL,Latency,IdleTime,Connect +1588341062311,4267,pins (small),200,OK,test environment 1-2,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,479,0,78 +1588341062070,4784,pins (small),200,OK,test environment 1-1,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,742,0,143 +1588341066580,461,pin-clusters (small),200,OK,test environment 1-2,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,461,0,76 +1588341066854,324,pin-clusters (small),200,OK,test environment 1-1,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,324,0,75 +1588341062641,4670,pins (small),200,OK,test environment 1-3,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,1881,0,76 +1588341067311,402,pin-clusters (small),200,OK,test environment 1-3,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,402,0,82 +1588341067042,36237,pins (medium),200,OK,test environment 1-2,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,5536,0,0 +1588341067179,36246,pins (medium),200,OK,test environment 1-1,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,5797,0,0 +1588341067713,35809,pins (medium),200,OK,test environment 1-3,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,6706,0,0 +1588341103280,5215,pin-clusters (medium),200,OK,test environment 1-2,text,true,,263,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,5215,0,77 +1588341103426,5170,pin-clusters (medium),200,OK,test environment 1-1,text,true,,263,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,5169,0,86 +1588341103523,5183,pin-clusters (medium),200,OK,test environment 1-3,text,true,,263,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,5183,0,80 +1588341108495,106582,pins (large),200,OK,test environment 1-2,text,true,,80300564,682,3,3,http://0529ed2e.ngrok.io/pins,9478,0,82 +1588341108596,107234,pins (large),200,OK,test environment 1-1,text,true,,80300564,682,3,3,http://0529ed2e.ngrok.io/pins,9906,0,83 +1588341108707,107180,pins (large),200,OK,test environment 1-3,text,true,,80300564,682,3,3,http://0529ed2e.ngrok.io/pins,9992,0,81 +1588341215059,9688,pin-clusters (large),200,OK,test environment 1-2,text,true,,263,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,9688,0,119 +1588341215811,9034,pin-clusters (large),200,OK,test environment 1-1,text,true,,263,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,9034,0,85 +1588341215867,9283,pin-clusters (large),200,OK,test environment 1-3,text,true,,264,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,9283,0,77 +1588341224747,1520,pins (small),200,OK,test environment 1-2,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,433,0,80 +1588341224846,1519,pins (small),200,OK,test environment 1-1,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,422,0,78 +1588341226268,238,pin-clusters (small),200,OK,test environment 1-2,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,238,0,0 +1588341225150,1420,pins (small),200,OK,test environment 1-3,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,707,0,92 +1588341226366,249,pin-clusters (small),200,OK,test environment 1-1,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,249,0,0 +1588341226571,251,pin-clusters (small),200,OK,test environment 1-3,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,250,0,0 +1588341226506,29329,pins (medium),200,OK,test environment 1-2,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,5258,0,0 +1588341226616,30831,pins (medium),200,OK,test environment 1-1,text,true,,19369721,527,3,3,http://0529ed2e.ngrok.io/pins,5137,0,0 +1588341226822,32010,pins (medium),200,OK,test environment 1-3,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,9554,0,0 +1588341255836,5131,pin-clusters (medium),200,OK,test environment 1-2,text,true,,263,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,5131,0,76 +1588341257447,6251,pin-clusters (medium),200,OK,test environment 1-1,text,true,,262,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,6251,0,80 +1588341258832,5886,pin-clusters (medium),200,OK,test environment 1-3,text,true,,262,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,5886,0,80 +1588341260967,104283,pins (large),200,OK,test environment 1-2,text,true,,80300564,682,3,3,http://0529ed2e.ngrok.io/pins,9703,0,81 +1588341263698,106168,pins (large),200,OK,test environment 1-1,text,true,,80300564,682,3,3,http://0529ed2e.ngrok.io/pins,9690,0,82 +1588341264718,105567,pins (large),200,OK,test environment 1-3,text,true,,80300564,682,3,3,http://0529ed2e.ngrok.io/pins,9824,0,82 +1588341365257,8606,pin-clusters (large),200,OK,test environment 1-2,text,true,,263,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,8605,0,114 +1588341373863,1409,pins (small),200,OK,test environment 1-2,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,379,0,82 +1588341375272,302,pin-clusters (small),200,OK,test environment 1-2,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,302,0,0 +1588341370288,9594,pin-clusters (large),200,OK,test environment 1-3,text,true,,262,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,9594,0,92 +1588341369869,10013,pin-clusters (large),200,OK,test environment 1-1,text,true,,263,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,10013,0,74 +1588341379882,1044,pins (small),200,OK,test environment 1-3,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,484,0,83 +1588341379882,1044,pins (small),200,OK,test environment 1-1,text,true,,791940,491,3,3,http://0529ed2e.ngrok.io/pins,483,0,84 +1588341380926,286,pin-clusters (small),200,OK,test environment 1-3,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,286,0,0 +1588341380926,292,pin-clusters (small),200,OK,test environment 1-1,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,292,0,0 +1588341375575,20057,pins (medium),200,OK,test environment 1-2,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,5555,0,0 +1588341395634,4906,pin-clusters (medium),200,OK,test environment 1-2,text,true,,263,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,4906,0,74 +1588341381212,25296,pins (medium),200,OK,test environment 1-3,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,6074,0,0 +1588341381218,25327,pins (medium),200,OK,test environment 1-1,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,6225,0,0 +1588341406509,5654,pin-clusters (medium),200,OK,test environment 1-3,text,true,,263,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,5654,0,77 +1588341406546,5617,pin-clusters (medium),200,OK,test environment 1-1,text,true,,263,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,5617,0,78 +1588341400540,88688,pins (large),200,OK,test environment 1-2,text,true,,80300564,682,3,3,http://0529ed2e.ngrok.io/pins,9177,0,75 +1588341489230,7621,pin-clusters (large),200,OK,test environment 1-2,text,true,,264,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,7621,0,103 +1588341496851,1193,pins (small),200,OK,test environment 1-2,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,431,0,79 +1588341498044,327,pin-clusters (small),200,OK,test environment 1-2,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,327,0,0 +1588341412164,102763,pins (large),200,OK,test environment 1-1,text,true,,80300564,682,3,3,http://0529ed2e.ngrok.io/pins,9706,0,74 +1588341412164,103148,pins (large),200,OK,test environment 1-3,text,true,,80300564,682,3,3,http://0529ed2e.ngrok.io/pins,10093,0,73 +1588341498372,21694,pins (medium),200,OK,test environment 1-2,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,5211,0,0 +1588341514932,8427,pin-clusters (large),200,OK,test environment 1-1,text,true,,264,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,8427,0,74 +1588341523360,1029,pins (small),200,OK,test environment 1-1,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,437,0,138 +1588341524389,355,pin-clusters (small),200,OK,test environment 1-1,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,355,0,0 +1588341515316,10631,pin-clusters (large),200,OK,test environment 1-3,text,true,,265,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,10631,0,80 +1588341520067,6440,pin-clusters (medium),200,OK,test environment 1-2,text,true,,262,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,6440,0,82 +1588341525947,1012,pins (small),200,OK,test environment 1-3,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,433,0,81 +1588341526960,289,pin-clusters (small),200,OK,test environment 1-3,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,289,0,0 +1588341524744,27124,pins (medium),200,OK,test environment 1-1,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,6158,0,0 +1588341527250,28448,pins (medium),200,OK,test environment 1-3,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,5861,0,0 +1588341551869,4715,pin-clusters (medium),200,OK,test environment 1-1,text,true,,262,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,4715,0,78 +1588341555699,5107,pin-clusters (medium),200,OK,test environment 1-3,text,true,,262,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,5107,0,76 +1588341526508,89795,pins (large),200,OK,test environment 1-2,text,true,,80300564,682,3,3,http://0529ed2e.ngrok.io/pins,9248,0,78 +1588341616307,8298,pin-clusters (large),200,OK,test environment 1-2,text,true,,264,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,8298,0,151 +1588341624605,1378,pins (small),200,OK,test environment 1-2,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,546,0,84 +1588341625984,353,pin-clusters (small),200,OK,test environment 1-2,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,353,0,0 +1588341626337,29308,pins (medium),200,OK,test environment 1-2,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,4867,0,0 +1588341556585,100493,pins (large),200,OK,test environment 1-1,text,true,,80300564,682,3,3,http://0529ed2e.ngrok.io/pins,10203,0,73 +1588341560806,97383,pins (large),200,OK,test environment 1-3,text,true,,80300564,682,3,3,http://0529ed2e.ngrok.io/pins,8751,0,76 +1588341655647,6095,pin-clusters (medium),200,OK,test environment 1-2,text,true,,263,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,6095,0,72 +1588341657082,9664,pin-clusters (large),200,OK,test environment 1-1,text,true,,263,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,9664,0,76 +1588341658192,8848,pin-clusters (large),200,OK,test environment 1-3,text,true,,264,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,8848,0,83 +1588341666746,898,pins (small),200,OK,test environment 1-1,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,431,0,85 +1588341667644,228,pin-clusters (small),200,OK,test environment 1-1,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,228,0,0 +1588341667042,942,pins (small),200,OK,test environment 1-3,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,379,0,83 +1588341667984,5050,pin-clusters (small),200,OK,test environment 1-3,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,5050,0,0 +1588341667872,29589,pins (medium),200,OK,test environment 1-1,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,5072,0,0 +1588341673035,28044,pins (medium),200,OK,test environment 1-3,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,5612,0,198 +1588341697463,4914,pin-clusters (medium),200,OK,test environment 1-1,text,true,,263,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,4914,0,151 +1588341701080,5344,pin-clusters (medium),200,OK,test environment 1-3,text,true,,263,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,5344,0,74 +1588341661743,116409,pins (large),Non HTTP response code: org.apache.http.ConnectionClosedException,"Non HTTP response message: Premature end of Content-Length delimited message body (expected: 80,300,398; received: 68,550,465)",test environment 1-2,text,false,,1335,0,3,3,http://0529ed2e.ngrok.io/pins,11344,0,82 +1588341706424,71828,pins (large),Non HTTP response code: org.apache.http.ConnectionClosedException,"Non HTTP response message: Premature end of Content-Length delimited message body (expected: 80,300,398; received: 19,627,842)",test environment 1-3,text,false,,1335,0,3,3,http://0529ed2e.ngrok.io/pins,8558,0,86 +1588341702377,75883,pins (large),Non HTTP response code: org.apache.http.ConnectionClosedException,"Non HTTP response message: Premature end of Content-Length delimited message body (expected: 80,300,398; received: 22,904,642)",test environment 1-1,text,false,,1335,0,3,3,http://0529ed2e.ngrok.io/pins,9189,0,73 +1588341829520,9486,pin-clusters (large),200,OK,test environment 1-2,text,true,,263,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,9486,0,103 +1588341829625,9389,pin-clusters (large),200,OK,test environment 1-1,text,true,,264,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,9389,0,80 +1588341829617,9513,pin-clusters (large),200,OK,test environment 1-3,text,true,,263,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,9513,0,71 +1588341839015,8704,pins (small),200,OK,test environment 1-1,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,1664,0,78 +1588341839007,8723,pins (small),200,OK,test environment 1-2,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,1500,0,81 +1588341839132,8661,pins (small),200,OK,test environment 1-3,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,2331,0,84 +1588341847720,368,pin-clusters (small),200,OK,test environment 1-1,text,true,,258,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,368,0,80 +1588341847730,377,pin-clusters (small),200,OK,test environment 1-2,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,377,0,91 +1588341847793,356,pin-clusters (small),200,OK,test environment 1-3,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,356,0,81 +1588341848149,30348,pins (medium),200,OK,test environment 1-3,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,5767,0,0 +1588341848107,30580,pins (medium),200,OK,test environment 1-2,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,5912,0,0 +1588341848088,30601,pins (medium),200,OK,test environment 1-1,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,6082,0,0 +1588341878498,5233,pin-clusters (medium),200,OK,test environment 1-3,text,true,,263,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,5233,0,76 +1588341878688,5212,pin-clusters (medium),200,OK,test environment 1-2,text,true,,263,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,5212,0,79 +1588341878690,5239,pin-clusters (medium),200,OK,test environment 1-1,text,true,,263,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,5239,0,78 +1588341883731,95085,pins (large),200,OK,test environment 1-3,text,true,,80300564,682,3,3,http://0529ed2e.ngrok.io/pins,9260,0,84 +1588341883900,95101,pins (large),200,OK,test environment 1-2,text,true,,80300564,682,3,3,http://0529ed2e.ngrok.io/pins,9306,0,82 +1588341883930,95187,pins (large),200,OK,test environment 1-1,text,true,,80300564,682,3,3,http://0529ed2e.ngrok.io/pins,9426,0,86 +1588341978818,11519,pin-clusters (large),200,OK,test environment 1-3,text,true,,264,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,11519,0,140 +1588341979120,11217,pin-clusters (large),200,OK,test environment 1-1,text,true,,264,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,11217,0,84 +1588341979004,11394,pin-clusters (large),200,OK,test environment 1-2,text,true,,264,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,11394,0,84 +1588341990338,1691,pins (small),200,OK,test environment 1-1,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,678,0,79 +1588341990399,1714,pins (small),200,OK,test environment 1-2,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,631,0,81 +1588341990340,1776,pins (small),200,OK,test environment 1-3,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,808,0,77 +1588341992029,240,pin-clusters (small),200,OK,test environment 1-1,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,240,0,0 +1588341992114,243,pin-clusters (small),200,OK,test environment 1-2,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,243,0,0 +1588341992116,241,pin-clusters (small),200,OK,test environment 1-3,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,241,0,0 +1588341992358,33246,pins (medium),200,OK,test environment 1-2,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,6453,0,0 +1588341992270,33414,pins (medium),200,OK,test environment 1-1,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,7871,0,0 +1588341992358,33377,pins (medium),200,OK,test environment 1-3,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,8136,0,85 +1588342025607,5432,pin-clusters (medium),200,OK,test environment 1-2,text,true,,262,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,5432,0,75 +1588342025737,5346,pin-clusters (medium),200,OK,test environment 1-3,text,true,,263,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,5346,0,82 +1588342025686,5440,pin-clusters (medium),200,OK,test environment 1-1,text,true,,262,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,5440,0,84 +1588342031127,102553,pins (large),200,OK,test environment 1-1,text,true,,80300564,682,3,3,http://0529ed2e.ngrok.io/pins,10028,0,82 +1588342031084,102705,pins (large),200,OK,test environment 1-3,text,true,,80300564,682,3,3,http://0529ed2e.ngrok.io/pins,10090,0,82 +1588342031039,102983,pins (large),200,OK,test environment 1-2,text,true,,80300565,682,3,3,http://0529ed2e.ngrok.io/pins,11788,0,86 +1588342133791,10959,pin-clusters (large),200,OK,test environment 1-3,text,true,,264,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,10959,0,79 +1588342134024,10841,pin-clusters (large),200,OK,test environment 1-2,text,true,,265,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,10841,0,78 +1588342133683,11379,pin-clusters (large),200,OK,test environment 1-1,text,true,,264,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,11379,0,123 +1588342145063,3064,pins (small),200,OK,test environment 1-1,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,381,0,85 +1588342144867,3328,pins (small),200,OK,test environment 1-2,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,600,0,82 +1588342148196,344,pin-clusters (small),200,OK,test environment 1-2,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,344,0,73 +1588342148127,413,pin-clusters (small),200,OK,test environment 1-1,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,413,0,74 +1588342144753,3870,pins (small),200,OK,test environment 1-3,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,1076,0,82 +1588342148623,344,pin-clusters (small),200,OK,test environment 1-3,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,344,0,74 +1588342148541,28239,pins (medium),200,OK,test environment 1-1,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,6318,0,0 +1588342148541,28391,pins (medium),200,OK,test environment 1-2,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,6592,0,0 +1588342148968,27999,pins (medium),200,OK,test environment 1-3,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,6248,0,0 +1588342176780,5486,pin-clusters (medium),200,OK,test environment 1-1,text,true,,263,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,5486,0,98 +1588342176932,5442,pin-clusters (medium),200,OK,test environment 1-2,text,true,,263,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,5442,0,79 +1588342176967,5464,pin-clusters (medium),200,OK,test environment 1-3,text,true,,262,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,5464,0,78 +1588342182267,100309,pins (large),200,OK,test environment 1-1,text,true,,80300565,682,3,3,http://0529ed2e.ngrok.io/pins,10777,0,81 +1588342182432,100184,pins (large),200,OK,test environment 1-3,text,true,,80300565,682,3,3,http://0529ed2e.ngrok.io/pins,10631,0,74 +1588342182374,100301,pins (large),200,OK,test environment 1-2,text,true,,80300565,682,3,3,http://0529ed2e.ngrok.io/pins,11040,0,84 +1588342282580,9849,pin-clusters (large),200,OK,test environment 1-1,text,true,,264,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,9849,0,103 +1588342282678,9986,pin-clusters (large),200,OK,test environment 1-2,text,true,,264,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,9985,0,82 +1588342282619,10637,pin-clusters (large),200,OK,test environment 1-3,text,true,,265,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,10637,0,78 +1588342292431,1361,pins (small),200,OK,test environment 1-1,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,571,0,82 +1588342292664,1371,pins (small),200,OK,test environment 1-2,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,436,0,85 +1588342293793,256,pin-clusters (small),200,OK,test environment 1-1,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,256,0,0 +1588342294035,262,pin-clusters (small),200,OK,test environment 1-2,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,262,0,0 +1588342293257,1274,pins (small),200,OK,test environment 1-3,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,734,0,156 +1588342294532,5498,pin-clusters (small),200,OK,test environment 1-3,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,5498,0,0 +1588342294049,24323,pins (medium),200,OK,test environment 1-1,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,5780,0,0 +1588342294297,24106,pins (medium),200,OK,test environment 1-2,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,5629,0,0 +1588342300031,21733,pins (medium),200,OK,test environment 1-3,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,5393,0,157 +1588342318373,5569,pin-clusters (medium),200,OK,test environment 1-1,text,true,,262,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,5569,0,79 +1588342318404,5547,pin-clusters (medium),200,OK,test environment 1-2,text,true,,263,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,5547,0,78 +1588342321765,5782,pin-clusters (medium),200,OK,test environment 1-3,text,true,,262,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,5782,0,85 +1588342323942,104679,pins (large),200,OK,test environment 1-1,text,true,,80300564,682,3,3,http://0529ed2e.ngrok.io/pins,9318,0,84 +1588342327547,106782,pins (large),200,OK,test environment 1-3,text,true,,80300563,682,3,3,http://0529ed2e.ngrok.io/pins,9664,0,81 +1588342323951,112279,pins (large),200,OK,test environment 1-2,text,true,,80300564,682,3,3,http://0529ed2e.ngrok.io/pins,17552,0,84 +1588342428624,8877,pin-clusters (large),200,OK,test environment 1-1,text,true,,264,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,8877,0,392 +1588342437502,851,pins (small),200,OK,test environment 1-1,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,371,0,84 +1588342438353,235,pin-clusters (small),200,OK,test environment 1-1,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,235,0,0 +1588342434333,9657,pin-clusters (large),200,OK,test environment 1-3,text,true,,264,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,9657,0,81 +1588342443992,1750,pins (small),200,OK,test environment 1-3,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,390,0,82 +1588342445743,785,pin-clusters (small),200,OK,test environment 1-3,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,785,0,0 +1588342436234,11494,pin-clusters (large),200,OK,test environment 1-2,text,true,,264,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,11494,0,85 +1588342447729,2344,pins (small),200,OK,test environment 1-2,text,true,,791941,491,3,3,http://0529ed2e.ngrok.io/pins,527,0,76 +1588342450073,367,pin-clusters (small),200,OK,test environment 1-2,text,true,,259,499,3,3,http://0529ed2e.ngrok.io/pin-clusters,366,0,76 +1588342438588,29371,pins (medium),200,OK,test environment 1-1,text,true,,19369721,527,3,3,http://0529ed2e.ngrok.io/pins,5928,0,0 +1588342446528,24495,pins (medium),200,OK,test environment 1-3,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,5291,0,81 +1588342450440,22224,pins (medium),200,OK,test environment 1-2,text,true,,19369722,527,3,3,http://0529ed2e.ngrok.io/pins,4923,0,0 +1588342467960,5083,pin-clusters (medium),200,OK,test environment 1-1,text,true,,262,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,5083,0,73 +1588342471026,6550,pin-clusters (medium),200,OK,test environment 1-3,text,true,,262,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,6550,0,76 +1588342472665,6033,pin-clusters (medium),200,OK,test environment 1-2,text,true,,263,535,3,3,http://0529ed2e.ngrok.io/pin-clusters,6033,0,83 +1588342473043,113254,pins (large),200,OK,test environment 1-1,text,true,,80300564,682,3,3,http://0529ed2e.ngrok.io/pins,11012,0,82 +1588342477576,110314,pins (large),200,OK,test environment 1-3,text,true,,80300563,682,3,3,http://0529ed2e.ngrok.io/pins,9950,0,84 +1588342478698,109407,pins (large),200,OK,test environment 1-2,text,true,,80300564,682,3,3,http://0529ed2e.ngrok.io/pins,9583,0,83 +1588342586300,9222,pin-clusters (large),200,OK,test environment 1-1,text,true,,264,690,3,3,http://0529ed2e.ngrok.io/pin-clusters,9222,0,141 +1588342587893,9381,pin-clusters (large),200,OK,test environment 1-3,text,true,,263,690,2,2,http://0529ed2e.ngrok.io/pin-clusters,9381,0,73 +1588342588108,9714,pin-clusters (large),200,OK,test environment 1-2,text,true,,264,690,1,1,http://0529ed2e.ngrok.io/pin-clusters,9714,0,84 diff --git a/server/requirements.txt b/server/requirements.txt index 55f85d080..969e4214b 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -24,9 +24,11 @@ py==1.8.1 pycodestyle==2.5.0 pyflakes==2.1.1 pyparsing==2.4.6 +pysupercluster==0.7.6 pytest==5.3.3 python-dateutil==2.8.1 pytz==2019.3 +redis==3.5.0 requests==2.23.0 requests-async==0.5.0 rfc3986==1.3.2 diff --git a/server/src/app.py b/server/src/app.py index 012430fa3..6bc6bd386 100644 --- a/server/src/app.py +++ b/server/src/app.py @@ -11,6 +11,7 @@ from services.timeToCloseService import TimeToCloseService from services.frequencyService import FrequencyService from services.pinService import PinService +from services.pinClusterService import PinClusterService from services.requestCountsService import RequestCountsService from services.requestDetailService import RequestDetailService from services.sqlIngest import DataHandler @@ -18,6 +19,7 @@ from services.dataService import DataService from utils.sanic import add_performance_header +from utils.redis import cache app = Sanic(__name__) CORS(app) @@ -44,6 +46,7 @@ def configure_app(): os.makedirs(os.path.join(app.config["STATIC_DIR"], "temp"), exist_ok=True) if app.config['Settings']['Server']['Debug']: add_performance_header(app) + cache.config(app.config['Settings']['Redis']) @app.route('/apistatus') @@ -222,6 +225,25 @@ async def pinMap(request): return json(return_data) +@app.route('/pin-clusters', methods=["POST"]) +@compress.compress() +async def pinClusters(request): + worker = PinClusterService(app.config['Settings']) + + postArgs = request.json + zoom = int(postArgs.get('zoom', 0)) + bounds = postArgs.get('bounds', {}) + filters = { + 'startDate': postArgs.get('startDate', None), + 'endDate': postArgs.get('endDate', None), + 'requestTypes': postArgs.get('requestTypes', []), + 'ncList': postArgs.get('ncList', []) + } + + clusters = await worker.get_pin_clusters(filters, zoom, bounds) + return json(clusters) + + @app.route('/requestcounts', methods=["POST"]) @compress.compress() async def requestCounts(request): diff --git a/server/src/services/pinClusterService.py b/server/src/services/pinClusterService.py new file mode 100644 index 000000000..cd2c3a30c --- /dev/null +++ b/server/src/services/pinClusterService.py @@ -0,0 +1,81 @@ +import pysupercluster +import pandas as pd +import hashlib +import json +from utils.redis import cache +from .dataService import DataService + + +class PinClusterService(object): + def __init__(self, config=None): + self.config = config + + def pins_key(self, filters): + filters_json = json.dumps(filters, sort_keys=True).encode('utf-8') + hashed_json = hashlib.md5(filters_json).hexdigest() + return 'filters:{}:pins'.format(hashed_json) + + def get_pins(self, filters): + key = self.pins_key(filters) + pins = cache.get(key) + + if pins is None: + dataAccess = DataService(self.config) + + fields = [ + 'srnumber', + 'requesttype', + 'latitude', + 'longitude'] + + filters = dataAccess.standardFilters( + filters['startDate'], + filters['endDate'], + filters['requestTypes'], + filters['ncList']) + + pins = dataAccess.query(fields, filters) + pins = pd.DataFrame(pins, columns=fields) + + cache.set(key, pins) + + return pins + + def pin_clusters(self, pins, zoom, bounds, options={}): + if len(pins) == 0: + return [] + + min_zoom = options.get('min_zoom', 0) + max_zoom = options.get('max_zoom', 17) + radius = options.get('radius', 200) + extent = options.get('extent', 512) + + index = pysupercluster.SuperCluster( + pins[['longitude', 'latitude']].to_numpy(), + min_zoom=min_zoom, + max_zoom=max_zoom, + radius=radius, + extent=extent) + + north = bounds.get('north', 90) + south = bounds.get('south', -90) + west = bounds.get('west', -180) + east = bounds.get('east', 180) + + clusters = index.getClusters( + top_left=(west, north), + bottom_right=(east, south), + zoom=zoom) + + for cluster in clusters: + if cluster['count'] == 1: + pin = pins.iloc[cluster['id']] + cluster['srnumber'] = pin['srnumber'] + cluster['requesttype'] = pin['requesttype'] + del cluster['expansion_zoom'] + + return clusters + + async def get_pin_clusters(self, filters, zoom, bounds): + pins = self.get_pins(filters) + return self.pin_clusters(pins, zoom, bounds) diff --git a/server/src/settings.example.cfg b/server/src/settings.example.cfg index 32375eeac..64a09cf53 100644 --- a/server/src/settings.example.cfg +++ b/server/src/settings.example.cfg @@ -37,3 +37,7 @@ GITHUB_TOKEN = REDACTED ISSUES_URL = https://api.github.com/repos/hackforla/311-data/issues PROJECT_URL = REDACTED GITHUB_SHA = DEVELOPMENT + +[Redis] +ENABLED = True +TTL_SECONDS = 3600 diff --git a/server/src/utils/redis.py b/server/src/utils/redis.py new file mode 100644 index 000000000..e7f81d317 --- /dev/null +++ b/server/src/utils/redis.py @@ -0,0 +1,34 @@ +import redis +import pickle +from datetime import timedelta + + +class RedisCache(object): + def config(self, config): + self.enabled = config['ENABLED'] == 'True' + if self.enabled: + self.ttl = int(config['TTL_SECONDS']) + self.r = redis.Redis(host='redis') + + def get(self, key): + if not self.enabled: + return None + + value = self.r.get(key) + if value is None: + return None + else: + return pickle.loads(value) + + def set(self, key, value): + if not self.enabled: + return None + + value = pickle.dumps(value) + try: + self.r.setex(key, timedelta(seconds=self.ttl), value) + except Exception as e: + print(e) + + +cache = RedisCache() diff --git a/server/src/utils/sanic.py b/server/src/utils/sanic.py index 3bc6526e0..ddf936926 100644 --- a/server/src/utils/sanic.py +++ b/server/src/utils/sanic.py @@ -11,10 +11,13 @@ async def request(req): req.ctx.start = time.perf_counter() async def response(req, res): - duration = time.perf_counter() - req.ctx.start - res.headers['x-performance'] = json.dumps({ - 'executionTime': round(duration, 4) - }) + try: + duration = time.perf_counter() - req.ctx.start + res.headers['x-performance'] = json.dumps({ + 'executionTime': round(duration, 4) + }) + except Exception: + pass sanic_app.register_middleware(request, attach_to='request') sanic_app.register_middleware(response, attach_to='response') From 3f7b8280b5dabe8710ae5529d4195acb4271e63c Mon Sep 17 00:00:00 2001 From: Jake Mensch Date: Sun, 3 May 2020 11:28:35 -0700 Subject: [PATCH 2/3] /heatmap endpoint --- server/src/app.py | 23 +++++++++++++-- server/src/services/heatmapService.py | 36 ++++++++++++++++++++++++ server/src/services/pinClusterService.py | 4 +-- 3 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 server/src/services/heatmapService.py diff --git a/server/src/app.py b/server/src/app.py index 6bc6bd386..b45df0500 100644 --- a/server/src/app.py +++ b/server/src/app.py @@ -12,6 +12,7 @@ from services.frequencyService import FrequencyService from services.pinService import PinService from services.pinClusterService import PinClusterService +from services.heatmapService import HeatmapService from services.requestCountsService import RequestCountsService from services.requestDetailService import RequestDetailService from services.sqlIngest import DataHandler @@ -231,8 +232,26 @@ async def pinClusters(request): worker = PinClusterService(app.config['Settings']) postArgs = request.json + filters = { + 'startDate': postArgs.get('startDate', None), + 'endDate': postArgs.get('endDate', None), + 'requestTypes': postArgs.get('requestTypes', []), + 'ncList': postArgs.get('ncList', []) + } zoom = int(postArgs.get('zoom', 0)) bounds = postArgs.get('bounds', {}) + options = postArgs.get('options', {}) + + clusters = await worker.get_pin_clusters(filters, zoom, bounds, options) + return json(clusters) + + +@app.route('/heatmap', methods=["POST"]) +@compress.compress() +async def heatmap(request): + worker = HeatmapService(app.config['Settings']) + + postArgs = request.json filters = { 'startDate': postArgs.get('startDate', None), 'endDate': postArgs.get('endDate', None), @@ -240,8 +259,8 @@ async def pinClusters(request): 'ncList': postArgs.get('ncList', []) } - clusters = await worker.get_pin_clusters(filters, zoom, bounds) - return json(clusters) + heatmap = await worker.get_heatmap(filters) + return json(heatmap) @app.route('/requestcounts', methods=["POST"]) diff --git a/server/src/services/heatmapService.py b/server/src/services/heatmapService.py new file mode 100644 index 000000000..ad4a42c69 --- /dev/null +++ b/server/src/services/heatmapService.py @@ -0,0 +1,36 @@ +import pandas as pd +import hashlib +import json +from utils.redis import cache +from .dataService import DataService + + +class HeatmapService(object): + def __init__(self, config=None): + self.config = config + + def pins_key(self, filters): + filters_json = json.dumps(filters, sort_keys=True).encode('utf-8') + hashed_json = hashlib.md5(filters_json).hexdigest() + return 'filters:{}:pins'.format(hashed_json) + + async def get_heatmap(self, filters): + key = self.pins_key(filters) + pins = cache.get(key) + + fields = ['latitude', 'longitude'] + if pins is None: + dataAccess = DataService(self.config) + + filters = dataAccess.standardFilters( + filters['startDate'], + filters['endDate'], + filters['requestTypes'], + filters['ncList']) + + pins = dataAccess.query(fields, filters) + pins = pd.DataFrame(pins, columns=fields) + else: + pins = pins[fields] + + return pins.to_numpy() diff --git a/server/src/services/pinClusterService.py b/server/src/services/pinClusterService.py index cd2c3a30c..220e3e5c1 100644 --- a/server/src/services/pinClusterService.py +++ b/server/src/services/pinClusterService.py @@ -76,6 +76,6 @@ def pin_clusters(self, pins, zoom, bounds, options={}): return clusters - async def get_pin_clusters(self, filters, zoom, bounds): + async def get_pin_clusters(self, filters, zoom, bounds, options): pins = self.get_pins(filters) - return self.pin_clusters(pins, zoom, bounds) + return self.pin_clusters(pins, zoom, bounds, options) From d06be403ac21e7beba1f3d89ee1b63a0aeb02e3a Mon Sep 17 00:00:00 2001 From: Jake Mensch Date: Sun, 3 May 2020 21:00:13 -0700 Subject: [PATCH 3/3] added REDIS_URL to settings --- Orchestration/docker-compose-example.yml | 1 + server/src/settings.example.cfg | 2 +- server/src/utils/redis.py | 10 +++++++--- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Orchestration/docker-compose-example.yml b/Orchestration/docker-compose-example.yml index f74ab7761..cc7ad023c 100644 --- a/Orchestration/docker-compose-example.yml +++ b/Orchestration/docker-compose-example.yml @@ -12,6 +12,7 @@ services: PROJECT_URL: REDACTED GITHUB_TOKEN: REDACTED TOKEN: REDACTED + REDIS_URL: redis://redis:6379 ports: - 5000:5000 diff --git a/server/src/settings.example.cfg b/server/src/settings.example.cfg index 64a09cf53..36c5745cd 100644 --- a/server/src/settings.example.cfg +++ b/server/src/settings.example.cfg @@ -39,5 +39,5 @@ PROJECT_URL = REDACTED GITHUB_SHA = DEVELOPMENT [Redis] -ENABLED = True +REDIS_URL = None TTL_SECONDS = 3600 diff --git a/server/src/utils/redis.py b/server/src/utils/redis.py index e7f81d317..d9e441cdb 100644 --- a/server/src/utils/redis.py +++ b/server/src/utils/redis.py @@ -4,11 +4,15 @@ class RedisCache(object): + def __init__(self): + self.enabled = False + def config(self, config): - self.enabled = config['ENABLED'] == 'True' - if self.enabled: + redis_url = config.get('REDIS_URL') + if redis_url != 'None': + self.enabled = True self.ttl = int(config['TTL_SECONDS']) - self.r = redis.Redis(host='redis') + self.r = redis.from_url(redis_url) def get(self, key): if not self.enabled: