Migrate to Maxmind GeoLite2 (#30)

Rewrite GeoJS backend to use GeoLite2 DBs. Shift over to Openresty!
jloh · Oct 17, 2018 · 6915b7c · 6915b7c
1 parent 538a490
commit 6915b7c
Show file tree

Hide file tree

Showing 17 changed files with 957 additions and 198 deletions.
diff --git a/.circleci/Geoip.conf b/.circleci/Geoip.conf
@@ -15,7 +15,7 @@ EditionIDs GeoLite2-City GeoLite2-ASN
 # The remaining settings are OPTIONAL.
 
 # The directory to store the database files. Defaults to /usr/local/share/GeoIP
-DatabaseDirectory /root/project/download-cache/maxmind2
+#DatabaseDirectory /root/project/download-cache/maxmind2
 
 # The server to use. Defaults to "updates.maxmind.com".
 # Host updates.maxmind.com

diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -2,52 +2,54 @@ version: 2
 jobs:
   build:
     docker:
-      - image: jloh/geojs-tests:beta-0.0.4
+      - image: jloh/geojs-tests:beta-0.0.12
     steps:
       - checkout
-      - restore_cache:
-          keys:
-            - maxmind-geolite2-v1
-      - run:
-          name: 'Download GeoLite1 Maxmind DBs'
-          command: |
-            mkdir -p download-cache/maxmind
-            test -s download-cache/maxmind/GeoLiteCityv6.dat || curl -s http://geolite.maxmind.com/download/geoip/database/GeoLiteCityv6-beta/GeoLiteCityv6.dat.gz | gzip -dc > download-cache/maxmind/GeoLiteCityv6.dat
-            test -s download-cache/maxmind/GeoIPv6.dat       || curl -s http://geolite.maxmind.com/download/geoip/database/GeoIPv6.dat.gz                          | gzip -dc > download-cache/maxmind/GeoIPv6.dat
-            test -s download-cache/maxmind/GeoIPASNumv6.dat  || curl -s http://download.maxmind.com/download/geoip/database/asnum/GeoIPASNumv6.dat.gz              | gzip -dc > download-cache/maxmind/GeoIPASNumv6.dat
+      # We have to do it this way since wget'ing it lands us weird dir names
       - run:
           name: 'Download GoeLite2 Maxmind DBs'
           command: |
             mkdir -p download-cache/maxmind2
             cp .circleci/Geoip.conf /etc/GeoIP.conf
             geoipupdate -v
-      - save_cache:
-          key: maxmind-geolite2-v1
-          paths:
-            - "download-cache/maxmind2"
-      # Required repos
-      # TODO: Work out a way to cache these?
-      - run:
-          name: 'Checkout dependencies'
+      - run:
+          name: 'Install OPM dependencies'
+          command: |
+            opm install \
+            pintsized/lua-resty-http=0.12 \
+            openresty/lua-resty-dns=0.21 \
+            openresty/lua-resty-upload=0.10 \
+            bungle/lua-resty-reqargs=1.4 \
+            xiaooloong/lua-resty-iconv=0.2.0 \
+            anjia0532/lua-resty-maxminddb=0.06
+      - run:
+          name: 'Add in Maxmind ASN'
           command: |
-            git clone -q --branch v0.10 https://github.com/pintsized/lua-resty-http.git repos/lua-resty-http
-            git clone -q --branch v0.18 https://github.com/openresty/lua-resty-dns.git repos/lua-resty-dns
-            git clone -q --branch v0.10 https://github.com/openresty/lua-resty-upload.git repos/lua-resty-upload
-            git clone -q --branch v1.4  https://github.com/bungle/lua-resty-reqargs.git repos/lua-resty-reqargs
-            git clone -q                https://github.com/xiaooloong/lua-resty-iconv.git repos/lua-resty-iconv
+            mkdir -p lib/resty
+            curl -s -o lib/resty/maxminddb_asn.lua $MAXMIND_LUA_URL
+      - run:
+          name: 'Openresty version'
+          command: openresty -V
       - run:
-          name: 'Nginx version'
-          command: nginx -V
+          name: 'Link openresty for tests'
+          command: ln -s /usr/bin/openresty /usr/bin/nginx
+      - run:
+          name: 'Directory for test results'
+          command: mkdir -p test-results/prove
       - run:
           name: 'Tests'
-          command: prove -r t -a test_results.tgz
+          command: prove -r t -a test_results.tgz --formatter TAP::Formatter::JUnit > test-results/prove/nosetests.xml
       - run:
           name: 'Test Coverage'
           command: |
             if [[ "${TEST_COVERAGE}x" == '1x' ]]; then luacov; fi
             if [[ "${TEST_COVERAGE}x" == '1x' ]]; then luacov-coveralls; fi
+      - store_test_results:
+          path: test-results
       - store_artifacts:
           path: test_results.tgz
+      - store_artifacts:
+          path: test-results/
 
 notify:
   webhooks:

diff --git a/.circleci/images/primary/Dockerfile b/.circleci/images/primary/Dockerfile
@@ -1,7 +1,8 @@
-FROM ubuntu:xenial
+FROM openresty/openresty:1.13.6.2-xenial
 
 RUN apt-get update && apt-get install -y software-properties-common
 RUN apt-add-repository ppa:maxmind/ppa
-RUN apt-get update && apt-get install -y lua-cjson nginx-extras cpanminus libluajit-5.1-dev libgd-dev git luarocks geoipupdate
-RUN cpanm -v --notest Test::Nginx TAP::Harness::Archive
-RUN luarocks install luacov-coveralls
+RUN apt-get update && apt-get install -y cpanminus libgd-dev git luarocks geoipupdate libmaxminddb0 libmaxminddb-dev
+RUN cpanm -v --notest Test::Nginx TAP::Harness::Archive TAP::Formatter::JUnit
+# luacov is broken currently :()
+#RUN luarocks install luacov-coveralls --tree=/usr/local/openresty/luajit
diff --git a/conf/v1/dns.conf b/conf/v1/dns.conf
@@ -16,13 +16,34 @@ location = /v1/dns/ptr {
         ngx.say(ptr)
     }
 }
+
+location ~ "/v1/dns/ptr/(?<ip>((([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))|(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])))$" {
+    default_type text/plain;
+    content_by_lua_block {
+        -- Function to get our PTR!
+        local getptr   = require("geojs.utils").get_ptr
+        local ngx_vars = ngx.var
+
+        -- Vars
+        local record
+
+        -- Get the IP we want
+        record = ngx_vars.ip
+
+        local ptr = getptr(record)
+        ngx.say(ptr)
+
+    }
+}
+
 location = /v1/dns/ptr.json {
     default_type application/json;
     content_by_lua_block {
         -- Function to get our PTR!
-        local getptr       = require("geojs.utils").get_ptr
-        local cjson        = require("cjson")
-        local json_encode  = cjson.encode
+        local getptr      = require("geojs.utils").get_ptr
+        local cjson       = require("cjson")
+        local cjson       = require("cjson")
+        local json_encode = cjson.encode
 
         -- Vars!
         local record
@@ -41,6 +62,32 @@ location = /v1/dns/ptr.json {
         )
     }
 }
+
+location ~ "/v1/dns/ptr/(?<ip>((([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))|(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])))\.json$" {
+    default_type application/json;
+    content_by_lua_block {
+        -- Function to get our PTR!
+        local getptr      = require("geojs.utils").get_ptr
+        local ngx_vars    = ngx.var
+        local cjson       = require("cjson")
+        local json_encode = cjson.encode
+
+        -- Vars!
+        local record
+
+        -- Get the IP we want
+        record = ngx_vars.ip
+
+        local ptr = getptr(record)
+
+        ngx.say(
+            json_encode({
+                ["ptr"] = ptr
+            })
+        )
+    }
+}
+
 location = /v1/dns/ptr.js {
     default_type application/javascript;
     content_by_lua_block {
@@ -75,3 +122,36 @@ location = /v1/dns/ptr.js {
         )
     }
 }
+
+location ~ "/v1/dns/ptr/(?<ip>((([0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,7}:|([0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|([0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|([0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|([0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:((:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))|(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])))\.js$" {
+    default_type application/javascript;
+    content_by_lua_block {
+        -- Function to get our PTR!
+        local getptr       = require("geojs.utils").get_ptr
+        local get_callback = require("geojs.utils").generate_callback
+        local ngx_vars     = ngx.var
+        local args         = ngx.req.get_uri_args()
+        local cjson        = require("cjson")
+        local json_encode  = cjson.encode
+
+        -- Vars!
+        local record
+        local callback
+
+        -- Get the IP we want
+        record = ngx_vars.ip
+
+        -- Override our callback if it exists
+        callback = get_callback('ptr', args)
+
+        local ptr = getptr(record)
+        ngx.say(
+            callback,
+            '(',
+            cjson.encode({
+                ["ptr"] = ptr
+            }),
+            ')'
+        )
+    }
+}
diff --git a/conf/v1/hooks.conf b/conf/v1/hooks.conf
@@ -46,14 +46,13 @@ location = /v1/hooks/hipchat {
         local get, post, files = require "resty.reqargs"()
 
         local split        = require("geojs.utils").split
-        local upstreamreq  = require("geojs.utils").upstream_req
+        local geo_lookup   = require("geojs.utils").geo_lookup
         local getptr       = require("geojs.utils").get_ptr
         local trim         = require("geojs.utils").trim
         local validate_ip  = require("geojs.utils").validate_ip
         local cjson        = require("cjson")
         local json_encode  = cjson.encode
         local json_decode  = cjson.decode
-        local reqpath      = '/v1/ip/geo.json'
 
         -- Get the message from HipChat
         local message = post["item"]["message"]["message"]
@@ -77,17 +76,16 @@ location = /v1/hooks/hipchat {
         end
 
         -- Get our data with the requested IP
-        local req_resp = upstreamreq(reqpath, req_ip)
+        local req_data = geo_lookup(req_ip)
         local req_ptr  = getptr(req_ip)
-        local req_data = json_decode(req_resp)
 
         -- Just in case we can't find out country
         if req_data['country'] == nil then
             req_data['country'] = "unknown country"
         end
 
         -- Define some card vars
-        local card_title      = string.format("<strong>%s</strong> is a %s IP belonging to %s", req_ip, req_data['country'], req_data['organization'])
+        local card_title      = string.format("<strong>%s</strong> is a %s IP belonging to %s", req_ip, req_data['country'], req_data['organization_name'])
         local txt_message     = string.format("Results for <b>%s</b><br><br>", req_ip)
         local card_simp_title = string.format("GeoIP results for %s", req_ip)
         local card_body  = ""
@@ -113,8 +111,8 @@ location = /v1/hooks/hipchat {
             card_body   = card_body .. '\n<strong>City:</strong> ' .. req_data['city']
             txt_message = txt_message .. '<br>City: ' .. req_data['city']
         end
-        if req_data['organization'] then
-            txt_message = txt_message .. '<br>Organization: ' .. req_data['organization']
+        if req_data['organization_name'] then
+            txt_message = txt_message .. '<br>Organization: ' .. req_data['organization_name']
         end
         txt_message = txt_message .. '<br><br>Powered by <a href="https://geojs.io" title="GeoJS">GeoJS</a>'
 
@@ -156,7 +154,7 @@ location = /v1/hooks/slack {
         -- Setup local vars
         local cjson        = require "cjson"
         local split        = require("geojs.utils").split
-        local upstream_req = require("geojs.utils").upstream_req
+        local geo_lookup   = require("geojs.utils").geo_lookup
         local getptr       = require("geojs.utils").get_ptr
         local trim         = require("geojs.utils").trim
         local validate_ip  = require("geojs.utils").validate_ip
@@ -229,10 +227,8 @@ location = /v1/hooks/slack {
             local response_url = post['response_url']
 
             -- Get Geo data about this IP
-            local req_resp = upstream_req(reqpath, req_ip)
+            local req_data = geo_lookup(req_ip)
             local req_ptr  = getptr(req_ip)
-            -- Turn our response into data
-            local req_data = json_decode(req_resp)
 
             local resp = ''
             -- Build out response
@@ -249,7 +245,7 @@ location = /v1/hooks/slack {
                 resp = resp .. '\nCity: ' .. req_data['city']
             end
             if req_data['organization'] then
-                resp = resp .. '\nOrganization: ' .. req_data['organization']
+                resp = resp .. '\nOrganization: ' .. req_data['organization_name']
             end
             local title = "IP Information for " .. req_ip
             message = {
@@ -360,7 +356,7 @@ location = /v1/hooks/twistapp {
         local ngx_var      = ngx.var
         local cjson        = require "cjson"
         local split        = require("geojs.utils").split
-        local upstream_req = require("geojs.utils").upstream_req
+        local geo_lookup   = require("geojs.utils").geo_lookup
         local getptr       = require("geojs.utils").get_ptr
         local trim         = require("geojs.utils").trim
         local validate_ip  = require("geojs.utils").validate_ip
@@ -403,10 +399,8 @@ location = /v1/hooks/twistapp {
             end
 
             -- Get Geo data about this IP
-            local req_resp = upstream_req(reqpath, req_ip)
+            local req_data = geo_lookup(req_ip)
             local req_ptr  = getptr(req_ip)
-            -- Turn our response into data
-            local req_data = cjson.decode(req_resp)
 
             -- Build out response
             local resp = '### IP information for **' .. req_ip .. '**'
@@ -423,7 +417,7 @@ location = /v1/hooks/twistapp {
                 resp = resp .. '\nCity: ' .. req_data['city']
             end
             if req_data['organization'] then
-                resp = resp .. '\nOrganization: ' .. req_data['organization']
+                resp = resp .. '\nOrganization: ' .. req_data['organization_name']
             end
             resp = resp .. '\nPowered by [GeoJS](https://geojs.io)'
             message = {