Add all station support for IEM Upper Air

Based on akrherz/iem#117, this commit
implements the new all-station upper air request from Iowa State.
Modifications were made to the parsing backend to allow for the returned
JSON with multiple files.

siphon/simplewebservice/iastate.py

@@ -26,7 +26,7 @@ def __init__(self):
 
     @classmethod
     def request_data(cls, time, site_id, **kwargs):
-        """Retrieve upper air observations from Iowa State's upper air archive.
+        """Retrieve upper air observations from Iowa State's archive for a single station.
 
         Parameters
         ----------
@@ -46,10 +46,35 @@ def request_data(cls, time, site_id, **kwargs):
 
         """
         endpoint = cls()
-        df = endpoint._get_data(time, site_id, **kwargs)
+        df = endpoint._get_data(time, site_id, None, **kwargs)
         return df
 
-    def _get_data(self, time, site_id):
+    @classmethod
+    def request_all_data(cls, time, pressure=None, **kwargs):
+        """Retrieve upper air observations from Iowa State's archive for all stations.
+
+        Parameters
+        ----------
+        time : datetime
+            The date and time of the desired observation.
+
+        pressure : `pint.Quantity`, optional
+            The mandatory pressure level at which to request data. If none is given, all the
+            available data in the profiles is returned.
+
+        kwargs
+            Arbitrary keyword arguments to use to initialize source
+
+        Returns
+        -------
+            :class:`pandas.DataFrame` containing the data
+
+        """
+        endpoint = cls()
+        df = endpoint._get_data(time, None, pressure, **kwargs)
+        return df
+
+    def _get_data(self, time, site_id, pressure):
         """Download data from Iowa State's upper air archive.
 
         Parameters
@@ -58,17 +83,24 @@ def _get_data(self, time, site_id):
             Date and time for which data should be downloaded
         site_id : str
             Site id for which data should be downloaded
+        pressure : `pint.Quantity`
+            Mandatory pressure level at which to request data.
 
         Returns
         -------
             :class:`pandas.DataFrame` containing the data
 
         """
-        json_data = self._get_data_raw(time, site_id)
+        json_data = self._get_data_raw(time, site_id, pressure)
         data = {}
-        for pt in json_data['profiles'][0]['profile']:
-            for field in ('drct', 'dwpc', 'hght', 'pres', 'sknt', 'tmpc'):
-                data.setdefault(field, []).append(np.nan if pt[field] is None else pt[field])
+        for profile in json_data['profiles']:
+            for pt in profile['profile']:
+                for field in ('drct', 'dwpc', 'hght', 'pres', 'sknt', 'tmpc'):
+                    data.setdefault(field, []).append(np.nan if pt[field] is None
+                                                      else pt[field])
+                for field in ('station', 'valid'):
+                    data.setdefault(field, []).append(np.nan if profile[field] is None
+                                                      else profile[field])
 
         # Make sure that the first entry has a valid temperature and dewpoint
         idx = np.argmax(~(np.isnan(data['tmpc']) | np.isnan(data['dwpc'])))
@@ -81,6 +113,9 @@ def _get_data(self, time, site_id):
         df['dewpoint'] = ma.masked_invalid(data['dwpc'][idx:])
         df['direction'] = ma.masked_invalid(data['drct'][idx:])
         df['speed'] = ma.masked_invalid(data['sknt'][idx:])
+        df['station'] = data['station'][idx:]
+        df['time'] = [datetime.strptime(time, '%Y-%m-%dT%H:%M:%SZ')
+                      for time in data['valid'][idx:]]
 
         # Calculate the u and v winds
         df['u_wind'], df['v_wind'] = get_wind_components(df['speed'],
@@ -90,10 +125,6 @@ def _get_data(self, time, site_id):
         df = df.dropna(subset=('temperature', 'dewpoint', 'direction', 'speed',
                                'u_wind', 'v_wind'), how='all').reset_index(drop=True)
 
-        df['station'] = json_data['profiles'][0]['station']
-        df['time'] = datetime.strptime(json_data['profiles'][0]['valid'],
-                                       '%Y-%m-%dT%H:%M:%SZ')
-
         # Add unit dictionary
         df.units = {'pressure': 'hPa',
                     'height': 'meter',
@@ -107,7 +138,7 @@ def _get_data(self, time, site_id):
                     'time': None}
         return df
 
-    def _get_data_raw(self, time, site_id):
+    def _get_data_raw(self, time, site_id, pressure):
         r"""Download data from the Iowa State's upper air archive.
 
         Parameters
@@ -116,14 +147,21 @@ def _get_data_raw(self, time, site_id):
             Date and time for which data should be downloaded
         site_id : str
             Site id for which data should be downloaded
+        pressure : `pint.Quantity`
+            Mandatory pressure level at which to request data.
 
         Returns
         -------
         list of json data
 
         """
-        path = ('raob.py?ts={time:%Y%m%d%H}00&station={stid}').format(time=time, stid=site_id)
-        resp = self.get_path(path)
+        query = {'ts': time.strftime('%Y%m%d%H00')}
+        if site_id is not None:
+            query['station'] = site_id
+        if pressure is not None:
+            query['pressure'] = pressure.to('hPa').magnitude
+
+        resp = self.get_path('raob.py', query)
         json_data = json.loads(resp.text)
 
         # See if the return is valid, but has no data