diff --git a/HISTORY.md b/HISTORY.md index ed44ca5..2a09c2b 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,13 @@ Release History =============== +0.1.2 (2019-04-13) +------------------ + +**BugFixes** + +- Fix _Unable to download the data in full_ error on `get_indicators` + 0.1.1 (2019-04-09) ------------------ diff --git a/tests/test_indicator.py b/tests/test_indicator.py index 565b0a4..efbf596 100644 --- a/tests/test_indicator.py +++ b/tests/test_indicator.py @@ -2,6 +2,7 @@ import numbers from world_bank_data import get_indicators, get_series from .tools import assert_numeric_or_string +from pandas.testing import assert_frame_equal def test_indicators_one(): @@ -21,6 +22,12 @@ def test_indicators(): assert_numeric_or_string(idx) +def test_indicators_per_page(): + idx = get_indicators().sort_index() + idx2 = get_indicators(per_page=5000).sort_index() + assert_frame_equal(idx, idx2) + + def test_indicators_topic(): idx = get_indicators(topic=5) assert len(idx.index) < 100 diff --git a/world_bank_data/request.py b/world_bank_data/request.py index 589b6b9..ed0011c 100644 --- a/world_bank_data/request.py +++ b/world_bank_data/request.py @@ -46,6 +46,7 @@ def wb_get(*args, **kwargs): params = copy(kwargs) language = params.pop('language') if 'language' in params else 'en' params.setdefault('format', 'json') + params.setdefault('per_page', 20000) # collapse the list of countries to a single str if len(args) > 1: @@ -74,14 +75,14 @@ def wb_get(*args, **kwargs): # Redo the request and get the full information when the first response is incomplete if params['format'] == 'json' and isinstance(data, list): page_information, data = data - if int(page_information['pages']) > 1: - params['per_page'] = page_information['total'] - response = get(url=url, params=params) - response.raise_for_status() - page_information, data = response.json() - - if int(page_information['pages']) > 1: - raise WBRequestError('Unable to download the data in full') + if 'page' not in params: + current_page = 1 + while current_page < int(page_information['pages']): + params['page'] = current_page = int(page_information['page']) + 1 + response = get(url=url, params=params) + response.raise_for_status() + page_information, new_data = response.json() + data.extend(new_data) if not data: raise RuntimeError("The request returned no data:\nurl={url}\nparams={params}" diff --git a/world_bank_data/version.py b/world_bank_data/version.py index 7076bb0..841b103 100644 --- a/world_bank_data/version.py +++ b/world_bank_data/version.py @@ -1,3 +1,3 @@ """version number""" -__version__ = '0.1.1' +__version__ = '0.1.2'