Test random indicators and intercept JSONDecodeError

tests/test_indicator.py

@@ -1,5 +1,6 @@
 import pytest
 import numbers
+import random
 from world_bank_data import get_indicators, get_series
 from .tools import assert_numeric_or_string
 from pandas.testing import assert_frame_equal
@@ -95,3 +96,24 @@ def test_indicator_monthly():
     idx = get_series('DPANUSSPB', country=['CHN', 'BRA'], date='2012M01:2012M08')
     assert len(idx.index) > 200 * 12
     assert_numeric_or_string(idx)
+
+
+def random_indicators():
+    random.seed(2019)
+    all_indicators = get_indicators()
+    return random.sample(all_indicators.index.tolist(), 12)
+
+
+@pytest.mark.parametrize('indicator', random_indicators())
+def test_random_indicators(indicator):
+    try:
+        idx = get_series(indicator, mrv=1)
+        assert_numeric_or_string(idx)
+    except ValueError as err:
+        assert 'The indicator was not found' in str(err)
+
+
+def test_json_error():
+    indicator = 'NV.IND.MANF.KD.87'
+    with pytest.raises(ValueError, match='The indicator was not found. It may have been deleted or archived.'):
+        get_series(indicator, mrv=1)

world_bank_data/request.py

@@ -1,4 +1,7 @@
+# -*- coding: utf-8 -*-
+
 """Request the world bank API"""
+import re
 import json
 from copy import copy
 from requests import get, HTTPError
@@ -65,7 +68,11 @@ def wb_get(*args, **kwargs):
 
     response = get(url=url, params=params, proxies=proxies)
     response.raise_for_status()
-    data = response.json()
+    try:
+        data = response.json()
+    except ValueError:  # simplejson.errors.JSONDecodeError derives from ValueError
+        raise ValueError("{msg}\nurl={url}\nparams={params}".format(msg=_extract_message(response.text),
+                                                                    url=url, params=params))
     if isinstance(data, list) and data and 'message' in data[0]:
         try:
             msg = data[0]['message'][0]['value']
@@ -93,6 +100,17 @@ def wb_get(*args, **kwargs):
     return data
 
 
+def _extract_message(msg):
+    """'<?xml version="1.0" encoding="utf-8"?>
+<wb:error xmlns:wb="http://www.worldbank.org">
+  <wb:message id="175" key="Invalid format">The indicator was not found. It may have been deleted or archived.</wb:message>
+</wb:error>'"""
+    if 'wb:message' not in msg:
+        return msg
+    return re.sub(re.compile('.*<wb:message[^>]*>', re.DOTALL), '',
+                  re.sub(re.compile('</wb:message>.*', re.DOTALL), '', msg))
+
+
 def _robust_key(*args, **kwargs):
     if 'proxies' in kwargs:
         kwargs['proxies'] = json.dumps(kwargs['proxies'])