Merge pull request #6 from mwouts/0.1.2

0.1.2

HISTORY.md

@@ -1,6 +1,13 @@
 Release History
 ===============
 
+0.1.2 (2019-04-13)
+------------------
+
+**BugFixes**
+
+- Fix _Unable to download the data in full_ error on `get_indicators`
+
 0.1.1 (2019-04-09)
 ------------------
 

tests/test_indicator.py

@@ -2,6 +2,7 @@
 import numbers
 from world_bank_data import get_indicators, get_series
 from .tools import assert_numeric_or_string
+from pandas.testing import assert_frame_equal
 
 
 def test_indicators_one():
@@ -21,6 +22,12 @@ def test_indicators():
     assert_numeric_or_string(idx)
 
 
+def test_indicators_per_page():
+    idx = get_indicators().sort_index()
+    idx2 = get_indicators(per_page=5000).sort_index()
+    assert_frame_equal(idx, idx2)
+
+
 def test_indicators_topic():
     idx = get_indicators(topic=5)
     assert len(idx.index) < 100

world_bank_data/request.py

@@ -46,6 +46,7 @@ def wb_get(*args, **kwargs):
     params = copy(kwargs)
     language = params.pop('language') if 'language' in params else 'en'
     params.setdefault('format', 'json')
+    params.setdefault('per_page', 20000)
 
     # collapse the list of countries to a single str
     if len(args) > 1:
@@ -74,14 +75,14 @@ def wb_get(*args, **kwargs):
     # Redo the request and get the full information when the first response is incomplete
     if params['format'] == 'json' and isinstance(data, list):
         page_information, data = data
-        if int(page_information['pages']) > 1:
-            params['per_page'] = page_information['total']
-            response = get(url=url, params=params)
-            response.raise_for_status()
-            page_information, data = response.json()
-
-            if int(page_information['pages']) > 1:
-                raise WBRequestError('Unable to download the data in full')
+        if 'page' not in params:
+            current_page = 1
+            while current_page < int(page_information['pages']):
+                params['page'] = current_page = int(page_information['page']) + 1
+                response = get(url=url, params=params)
+                response.raise_for_status()
+                page_information, new_data = response.json()
+                data.extend(new_data)
 
     if not data:
         raise RuntimeError("The request returned no data:\nurl={url}\nparams={params}"

world_bank_data/version.py

@@ -1,3 +1,3 @@
 """version number"""
 
-__version__ = '0.1.1'
+__version__ = '0.1.2'