diff --git a/tests/test_indicator.py b/tests/test_indicator.py index e1a7f03..81a4b82 100644 --- a/tests/test_indicator.py +++ b/tests/test_indicator.py @@ -1,5 +1,6 @@ import pytest import numbers +import random from world_bank_data import get_indicators, get_series from .tools import assert_numeric_or_string from pandas.testing import assert_frame_equal @@ -95,3 +96,24 @@ def test_indicator_monthly(): idx = get_series('DPANUSSPB', country=['CHN', 'BRA'], date='2012M01:2012M08') assert len(idx.index) > 200 * 12 assert_numeric_or_string(idx) + + +def random_indicators(): + random.seed(2019) + all_indicators = get_indicators() + return random.sample(all_indicators.index.tolist(), 12) + + +@pytest.mark.parametrize('indicator', random_indicators()) +def test_random_indicators(indicator): + try: + idx = get_series(indicator, mrv=1) + assert_numeric_or_string(idx) + except ValueError as err: + assert 'The indicator was not found' in str(err) + + +def test_json_error(): + indicator = 'NV.IND.MANF.KD.87' + with pytest.raises(ValueError, match='The indicator was not found. It may have been deleted or archived.'): + get_series(indicator, mrv=1) diff --git a/world_bank_data/request.py b/world_bank_data/request.py index 9eb6a8b..c4425c2 100644 --- a/world_bank_data/request.py +++ b/world_bank_data/request.py @@ -1,4 +1,7 @@ +# -*- coding: utf-8 -*- + """Request the world bank API""" +import re import json from copy import copy from requests import get, HTTPError @@ -65,7 +68,11 @@ def wb_get(*args, **kwargs): response = get(url=url, params=params, proxies=proxies) response.raise_for_status() - data = response.json() + try: + data = response.json() + except ValueError: # simplejson.errors.JSONDecodeError derives from ValueError + raise ValueError("{msg}\nurl={url}\nparams={params}".format(msg=_extract_message(response.text), + url=url, params=params)) if isinstance(data, list) and data and 'message' in data[0]: try: msg = data[0]['message'][0]['value'] @@ -93,6 +100,17 @@ def wb_get(*args, **kwargs): return data +def _extract_message(msg): + """' + + The indicator was not found. It may have been deleted or archived. +'""" + if 'wb:message' not in msg: + return msg + return re.sub(re.compile('.*]*>', re.DOTALL), '', + re.sub(re.compile('.*', re.DOTALL), '', msg)) + + def _robust_key(*args, **kwargs): if 'proxies' in kwargs: kwargs['proxies'] = json.dumps(kwargs['proxies'])