Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Sounding metadata #206

Merged
merged 4 commits into from
Mar 30, 2018
Merged

Sounding metadata #206

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ddb8fc2
Add sounding metadata to returned dataframe for Wyoming simple web se…
jrleeman Mar 29, 2018
6259e64
Add sounding metadata to returned dataframe for Iowa simple web service.
jrleeman Mar 29, 2018
c13f790
Add to tests for Wyoming sounding metadata.
jrleeman Mar 29, 2018
c0c5274
Add to tests for Iowa sounding metadata.
jrleeman Mar 29, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 11 additions & 6 deletions siphon/simplewebservice/iastate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
# SPDX-License-Identifier: BSD-3-Clause
"""Read upper air data from the IA State archives."""

from datetime import datetime
import json
import warnings

Expand Down Expand Up @@ -64,9 +65,8 @@ def _get_data(self, time, site_id):

"""
json_data = self._get_data_raw(time, site_id)

data = {}
for pt in json_data:
for pt in json_data['profiles'][0]['profile']:
for field in ('drct', 'dwpc', 'hght', 'pres', 'sknt', 'tmpc'):
data.setdefault(field, []).append(np.nan if pt[field] is None else pt[field])

Expand All @@ -90,6 +90,10 @@ def _get_data(self, time, site_id):
df = df.dropna(subset=('temperature', 'dewpoint', 'direction', 'speed',
'u_wind', 'v_wind'), how='all').reset_index(drop=True)

df['station'] = json_data['profiles'][0]['station']
df['time'] = datetime.strptime(json_data['profiles'][0]['valid'],
'%Y-%m-%dT%H:%M:%SZ')

# Add unit dictionary
df.units = {'pressure': 'hPa',
'height': 'meter',
Expand All @@ -98,8 +102,9 @@ def _get_data(self, time, site_id):
'direction': 'degrees',
'speed': 'knot',
'u_wind': 'knot',
'v_wind': 'knot'}

'v_wind': 'knot',
'station': None,
'time': None}
return df

def _get_data_raw(self, time, site_id):
Expand All @@ -119,10 +124,10 @@ def _get_data_raw(self, time, site_id):
"""
path = ('raob.py?ts={time:%Y%m%d%H}00&station={stid}').format(time=time, stid=site_id)
resp = self.get_path(path)
json_data = json.loads(resp.text)['profiles'][0]['profile']
json_data = json.loads(resp.text)

# See if the return is valid, but has no data
if not json_data:
if not json_data['profiles'][0]['profile']:
raise ValueError('No data available for {time:%Y-%m-%d %HZ} '
'for station {stid}.'.format(time=time, stid=site_id))
return json_data
35 changes: 30 additions & 5 deletions siphon/simplewebservice/wyoming.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
# SPDX-License-Identifier: BSD-3-Clause
"""Read upper air data from the Wyoming archives."""

from datetime import datetime
from io import StringIO
import warnings

Expand Down Expand Up @@ -69,15 +70,34 @@ def _get_data(self, time, site_id, region='naconf'):

"""
raw_data = self._get_data_raw(time, site_id, region)
soup = BeautifulSoup(raw_data, 'html.parser')
tabular_data = StringIO(soup.find_all('pre')[0].contents[0])
col_names = ['pressure', 'height', 'temperature', 'dewpoint', 'direction', 'speed']
df = pd.read_fwf(raw_data, skiprows=5, usecols=[0, 1, 2, 3, 6, 7], names=col_names)
df = pd.read_fwf(tabular_data, skiprows=5, usecols=[0, 1, 2, 3, 6, 7], names=col_names)
df['u_wind'], df['v_wind'] = get_wind_components(df['speed'],
np.deg2rad(df['direction']))

# Drop any rows with all NaN values for T, Td, winds
df = df.dropna(subset=('temperature', 'dewpoint', 'direction', 'speed',
'u_wind', 'v_wind'), how='all').reset_index(drop=True)

# Parse metadata
meta_data = soup.find_all('pre')[1].contents[0]
lines = meta_data.splitlines()
station = lines[1].split(':')[1].strip()
station_number = int(lines[2].split(':')[1].strip())
sounding_time = datetime.strptime(lines[3].split(':')[1].strip(), '%y%m%d/%H%M')
latitude = float(lines[4].split(':')[1].strip())
longitude = float(lines[5].split(':')[1].strip())
elevation = float(lines[6].split(':')[1].strip())

df['station'] = station
df['station_number'] = station_number
df['time'] = sounding_time
df['latitude'] = latitude
df['longitude'] = longitude
df['elevation'] = elevation

# Add unit dictionary
df.units = {'pressure': 'hPa',
'height': 'meter',
Expand All @@ -86,7 +106,13 @@ def _get_data(self, time, site_id, region='naconf'):
'direction': 'degrees',
'speed': 'knot',
'u_wind': 'knot',
'v_wind': 'knot'}
'v_wind': 'knot',
'station': None,
'station_number': None,
'time': None,
'latitude': 'degrees',
'longitude': 'degrees',
'elevation': 'meter'}
return df

def _get_data_raw(self, time, site_id, region='naconf'):
Expand All @@ -103,7 +129,7 @@ def _get_data_raw(self, time, site_id, region='naconf'):

Returns
-------
a file-like object from which to read the data
text of the server response

"""
path = ('?region={region}&TYPE=TEXT%3ALIST'
Expand All @@ -118,5 +144,4 @@ def _get_data_raw(self, time, site_id, region='naconf'):
'for station {stid}.'.format(time=time, region=region,
stid=site_id))

soup = BeautifulSoup(resp.text, 'html.parser')
return StringIO(soup.find_all('pre')[0].contents[0])
return resp.text
19 changes: 19 additions & 0 deletions siphon/tests/test_iastate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,9 @@ def test_iastate():
"""Test that we are properly parsing data from the Iowa State archive."""
df = IAStateUpperAir.request_data(datetime(1999, 5, 4, 0), 'OUN')

assert(df['time'][0] == datetime(1999, 5, 4, 0))
assert(df['station'][0] == 'KOUN')

assert_almost_equal(df['pressure'][6], 872.7, 2)
assert_almost_equal(df['height'][6], 1172.0, 2)
assert_almost_equal(df['temperature'][6], 18.2, 2)
Expand All @@ -36,13 +39,18 @@ def test_iastate():
assert(df.units['v_wind'] == 'knot')
assert(df.units['speed'] == 'knot')
assert(df.units['direction'] == 'degrees')
assert(df.units['station'] is None)
assert(df.units['time'] is None)


@recorder.use_cassette('iastate_high_alt_sounding')
def test_high_alt_iastate():
"""Test Iowa State data that starts at pressure less than 925 hPa."""
df = IAStateUpperAir.request_data(datetime(2010, 12, 9, 12), 'BOI')

assert(df['time'][0] == datetime(2010, 12, 9, 12))
assert(df['station'][0] == 'KBOI')

assert_almost_equal(df['pressure'][0], 919.0, 2)
assert_almost_equal(df['height'][0], 871.0, 2)
assert_almost_equal(df['temperature'][0], -0.1, 2)
Expand All @@ -51,3 +59,14 @@ def test_high_alt_iastate():
assert_almost_equal(df['v_wind'][0], 1.500, 2)
assert_almost_equal(df['speed'][0], 3.0, 1)
assert_almost_equal(df['direction'][0], 240.0, 1)

assert(df.units['pressure'] == 'hPa')
assert(df.units['height'] == 'meter')
assert(df.units['temperature'] == 'degC')
assert(df.units['dewpoint'] == 'degC')
assert(df.units['u_wind'] == 'knot')
assert(df.units['v_wind'] == 'knot')
assert(df.units['speed'] == 'knot')
assert(df.units['direction'] == 'degrees')
assert(df.units['station'] is None)
assert(df.units['time'] is None)
35 changes: 35 additions & 0 deletions siphon/tests/test_wyoming.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,13 @@ def test_wyoming():
"""Test that we are properly parsing data from the Wyoming archive."""
df = WyomingUpperAir.request_data(datetime(1999, 5, 4, 0), 'OUN')

assert(df['time'][0] == datetime(1999, 5, 4, 0))
assert(df['station'][0] == 'OUN')
assert(df['station_number'][0] == 72357)
assert(df['latitude'][0] == 35.18)
assert(df['longitude'][0] == -97.44)
assert(df['elevation'][0] == 345.0)

assert_almost_equal(df['pressure'][5], 867.9, 2)
assert_almost_equal(df['height'][5], 1219., 2)
assert_almost_equal(df['temperature'][5], 17.4, 2)
Expand All @@ -36,13 +43,26 @@ def test_wyoming():
assert(df.units['v_wind'] == 'knot')
assert(df.units['speed'] == 'knot')
assert(df.units['direction'] == 'degrees')
assert(df.units['latitude'] == 'degrees')
assert(df.units['longitude'] == 'degrees')
assert(df.units['elevation'] == 'meter')
assert(df.units['station'] is None)
assert(df.units['station_number'] is None)
assert(df.units['time'] is None)


@recorder.use_cassette('wyoming_high_alt_sounding')
def test_high_alt_wyoming():
"""Test Wyoming data that starts at pressure less than 925 hPa."""
df = WyomingUpperAir.request_data(datetime(2010, 12, 9, 12), 'BOI')

assert(df['time'][0] == datetime(2010, 12, 9, 12))
assert(df['station'][0] == 'BOI')
assert(df['station_number'][0] == 72681)
assert(df['latitude'][0] == 43.56)
assert(df['longitude'][0] == -116.21)
assert(df['elevation'][0] == 874.0)

assert_almost_equal(df['pressure'][2], 890.0, 2)
assert_almost_equal(df['height'][2], 1133., 2)
assert_almost_equal(df['temperature'][2], 5.4, 2)
Expand All @@ -51,3 +71,18 @@ def test_high_alt_wyoming():
assert_almost_equal(df['v_wind'][2], 5.99, 2)
assert_almost_equal(df['speed'][2], 6.0, 1)
assert_almost_equal(df['direction'][2], 176.0, 1)

assert(df.units['pressure'] == 'hPa')
assert(df.units['height'] == 'meter')
assert(df.units['temperature'] == 'degC')
assert(df.units['dewpoint'] == 'degC')
assert(df.units['u_wind'] == 'knot')
assert(df.units['v_wind'] == 'knot')
assert(df.units['speed'] == 'knot')
assert(df.units['direction'] == 'degrees')
assert(df.units['latitude'] == 'degrees')
assert(df.units['longitude'] == 'degrees')
assert(df.units['elevation'] == 'meter')
assert(df.units['station'] is None)
assert(df.units['station_number'] is None)
assert(df.units['time'] is None)