Skip to content

Commit

Permalink
Merge pull request #258 from Ouranosinc/fix-222
Browse files Browse the repository at this point in the history 
Update Raven version [WIP]
  • Loading branch information
@richardarsenault
richardarsenault authored Apr 30, 2020
2 parents dd8d5dd + 3065436 commit eeca117
Show file tree
Hide file tree
Showing 19 changed files with 123 additions and 160 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ src/
testdata.json
tests/*/*/processor_0
tests/*/*/OstOutput*
tests/*/*/*.nc

# IPython
.ipynb_checkpoints
Expand Down
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ PYTHON_VERSION = 3.6

# Choose Anaconda installer depending on your OS
ANACONDA_URL = https://repo.continuum.io/miniconda
RAVEN_URL = http://www.civil.uwaterloo.ca/jmai/raven/raven-rev245.zip
RAVEN_URL = http://www.civil.uwaterloo.ca/jmai/raven/raven-rev254.zip
RAVEN_SRC = $(CURDIR)/src/RAVEN
OSTRICH_URL = http://www.civil.uwaterloo.ca/jmai/raven/Ostrich_2017-12-19_plus_progressJSON.zip
OSTRICH_SRC = $(CURDIR)/src/OSTRICH
Expand Down
19 changes: 8 additions & 11 deletions raven/models/emulators.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,19 +51,18 @@ class MOHYSE(Raven):
identifier = 'mohyse'
templates = tuple((Path(__file__).parent / 'raven-mohyse').glob("*.rv?"))

params = namedtuple('MOHYSEParams', ', '.join(['par_x{:02}'.format(i) for i in range(1, 9)]))
hrus = namedtuple('MOHYSEHRU', ('par_x09', 'par_x10'))
params = namedtuple('MOHYSEParams', ', '.join(['par_x{:02}'.format(i) for i in range(1, 11)]))

def __init__(self, *args, **kwds):
super().__init__(*args, **kwds)
self.rvp = RV(params=MOHYSE.params(*((None,) * 8)))
self.rvh = RV(name=None, area=None, elevation=None, latitude=None, longitude=None, hrus=MOHYSE.hrus(None, None))
self.rvp = RV(params=MOHYSE.params(*((None,) * 10)))
self.rvh = RV(name=None, area=None, elevation=None, latitude=None, longitude=None)
self.rvt = RVT(**{k: nc() for k in std_vars})
self.rvi = RVI(evaporation="PET_MOHYSE", rain_snow_fraction="RAINSNOW_DATA")
self.rvd = RV(par_rezi_x10=None)

def derived_parameters(self):
self.rvd['par_rezi_x10'] = 1.0 / self.rvh.hrus.par_x10
self.rvd['par_rezi_x10'] = 1.0 / self.rvp.params.par_x10


class MOHYSE_OST(Ostrich, MOHYSE):
Expand All @@ -75,10 +74,8 @@ def __init__(self, *args, **kwds):
self.rvi.suppress_output = True
self.txt = Ost(algorithm='DSS',
max_iterations=50,
lowerBounds=MOHYSE.params(None, None, None, None, None, None, None, None),
upperBounds=MOHYSE.params(None, None, None, None, None, None, None, None),
hruslowerBounds=MOHYSE.hrus(None, None),
hrusupperBounds=MOHYSE.hrus(None, None)
lowerBounds=MOHYSE.params(None, None, None, None, None, None, None, None, None, None),
upperBounds=MOHYSE.params(None, None, None, None, None, None, None, None, None, None),
)

def derived_parameters(self):
Expand Down Expand Up @@ -110,8 +107,8 @@ def derived_parameters(self):
self.rvd['PHREATIC_hlf'] = self.rvp.params.PHREATIC * 0.5
self.rvd['TOPSOIL_m'] = self.rvp.params.TOPSOIL / 1000.
self.rvd['PHREATIC_m'] = self.rvp.params.PHREATIC / 1000.
self.rvd['SUM_MELT_FACTOR'] = self.rvp.params.MIN_MELT_FACTOR + self.rvp.params.MAX_MELT_FACTOR
self.rvd['SUM_SNOW_SWI'] = self.rvp.params.SNOW_SWI_MIN + self.rvp.params.SNOW_SWI_MAX
self.rvd['SUM_MELT_FACTOR'] = self.rvp.params.MAX_MELT_FACTOR # self.rvp.params.MIN_MELT_FACTOR +
self.rvd['SUM_SNOW_SWI'] = self.rvp.params.SNOW_SWI_MAX # self.rvp.params.SNOW_SWI_MIN +


class HMETS_OST(Ostrich, HMETS):
Expand Down
4 changes: 2 additions & 2 deletions raven/models/ostrich-mohyse/ostIn.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,8 @@ BeginParams
par_x06 random {lowerBounds.par_x06} {upperBounds.par_x06} none none none
par_x07 random {lowerBounds.par_x07} {upperBounds.par_x07} none none none
par_x08 random {lowerBounds.par_x08} {upperBounds.par_x08} none none none
par_x09 random {hruslowerBounds.par_x09} {hrusupperBounds.par_x09} none none none
par_x10 random {hruslowerBounds.par_x10} {hrusupperBounds.par_x10} none none none
par_x09 random {lowerBounds.par_x09} {upperBounds.par_x09} none none none
par_x10 random {lowerBounds.par_x10} {upperBounds.par_x10} none none none
EndParams

BeginTiedParams
Expand Down
2 changes: 1 addition & 1 deletion raven/models/raven-mohyse/raven-mohyse.rvh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,6 @@
# 1.0 / MOHYSE_PARA_10, MOHYSE_PARA_9
:Parameters, GAMMA_SCALE, GAMMA_SHAPE,
:Units, 1/d, -
1, {par_rezi_x10}, {hrus.par_x09}
1, {par_rezi_x10}, {params.par_x09}
:EndSubBasinProperties

4 changes: 2 additions & 2 deletions raven/models/rv.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -290,7 +290,7 @@ def linear_transform(self):
slope, intercept = lt or (1, 0)
sf = 1 if self.scale_factor is None else self.scale_factor
offset = 0 if self.add_offset is None else self.add_offset
return ":LinearTransform {:g} {:g}".format(slope * sf, offset * slope + intercept)
return ":LinearTransform {:.15f} {:.15f}".format(slope * sf, offset * slope + intercept)

@linear_transform.setter
def linear_transform(self, value):
Expand Down Expand Up @@ -378,7 +378,7 @@ def __init__(self, **kwargs):
self.latitude = None
self.longitude = None
self.run_index = 0
self.raven_version = '2.9 rev#177'
self.raven_version = '2.9 rev#254'

self._run_name = 'run'
self._start_date = None
Expand Down
39 changes: 14 additions & 25 deletions raven/processes/wps_ostrich_mohyse.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,29 +22,32 @@
par_x05=0.4038,
par_x06=0.0621,
par_x07=0.0273,
par_x08=0.0453)
par_x08=0.0453,
par_x09=0.9039,
par_x10=5.6167)

Lparams_defaults = MOHYSE_OST.params(par_x01=0.01,
par_x02=0.01,
par_x03=0.01,
par_x04=-5.00,
par_x05=0.01,
par_x06=0.01,
par_x07=0.01,
par_x08=0.01)
par_x08=0.01,
par_x09=0.01,
par_x10=0.01
)
Uparams_defaults = MOHYSE_OST.params(par_x01=20.0,
par_x02=1.0,
par_x03=20.0,
par_x04=5.0,
par_x05=0.5,
par_x06=1.0,
par_x07=1.0,
par_x08=1.0)

hrus_defaults = MOHYSE_OST.hrus(par_x09=0.9039, par_x10=5.6167)
Uhrus_defaults = MOHYSE_OST.hrus(par_x09=15.0,
par_x10=15.0)
Lhrus_defaults = MOHYSE_OST.hrus(par_x09=0.01,
par_x10=0.01)
par_x08=1.0,
par_x09=15.0,
par_x10=15.0
)

upperBounds = LiteralInput('upperBounds', 'Comma separated list of model parameters Upper Bounds',
abstract='UParameters: ' + ', '.join(Uparams_defaults._fields),
Expand All @@ -58,18 +61,6 @@
default=', '.join(str(p) for p in list(Lparams_defaults)),
min_occurs=0)

hrusupperBounds = LiteralInput('hrusupperBounds', 'Comma separated list of hru parameters Upper Bounds',
abstract='Uhrus: ' + ', '.join(Uhrus_defaults._fields),
data_type='string',
default=', '.join(str(p) for p in list(Uhrus_defaults)),
min_occurs=0)

hruslowerBounds = LiteralInput('hruslowerBounds', 'Comma separated list of hru parameters Lower Bounds',
abstract='Lhrus: ' + ', '.join(Lhrus_defaults._fields),
data_type='string',
default=', '.join(str(p) for p in list(Lhrus_defaults)),
min_occurs=0)


class OstrichMOHYSEProcess(OstrichProcess):
"""
Expand All @@ -86,10 +77,8 @@ class OstrichMOHYSEProcess(OstrichProcess):
version = ''
model_cls = MOHYSE_OST
tuple_inputs = {'lowerBounds': MOHYSE_OST.params,
'upperBounds': MOHYSE_OST.params,
'hrusupperBounds': MOHYSE_OST.hrus,
'hruslowerBounds': MOHYSE_OST.hrus}
inputs = [wio.ts, wio.nc_spec, lowerBounds, upperBounds, hrusupperBounds, hruslowerBounds, wio.algorithm,
'upperBounds': MOHYSE_OST.params}
inputs = [wio.ts, wio.nc_spec, lowerBounds, upperBounds, wio.algorithm,
wio.max_iterations, wio.start_date, wio.end_date,
wio.duration, wio.run_name, wio.name, wio.area, wio.latitude, wio.longitude, wio.elevation,
wio.random_seed, wio.suppress_output, wio.rain_snow_fraction, wio.evaporation]
Expand Down
17 changes: 5 additions & 12 deletions raven/processes/wps_raven_mohyse.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@
par_x05=0.4038,
par_x06=0.0621,
par_x07=0.0273,
par_x08=0.0453)

hrus_defaults = MOHYSE.hrus(par_x09=0.9039, par_x10=5.6167)
par_x08=0.0453,
par_x09=0.9039,
par_x10=5.6167)

params = LiteralInput('params', 'Comma separated list of model parameters',
abstract='Parameters: ' + ', '.join(params_defaults._fields),
Expand All @@ -23,21 +23,14 @@
min_occurs=0,
max_occurs=config.max_parallel_processes)

hrus = LiteralInput('hrus', 'Comma separated list of HRU parameters',
abstract='Parameters: ' + ', '.join(hrus_defaults._fields),
data_type='string',
default=', '.join(str(p) for p in list(hrus_defaults)),
min_occurs=0,
max_occurs=config.max_parallel_processes)


class RavenMOHYSEProcess(RavenProcess):
identifier = 'raven-mohyse'
abstract = 'MOHYSE hydrological model'
title = 'TODO'
version = ''
model_cls = MOHYSE
tuple_inputs = {'params': MOHYSE.params, 'hrus': MOHYSE.hrus}
tuple_inputs = {'params': MOHYSE.params}

inputs = [wio.ts, wio.nc_spec, params, hrus, wio.start_date, wio.end_date, wio.nc_index, wio.duration, wio.run_name,
inputs = [wio.ts, wio.nc_spec, params, wio.start_date, wio.end_date, wio.nc_index, wio.duration, wio.run_name,
wio.name, wio.area, wio.latitude, wio.longitude, wio.elevation, wio.rain_snow_fraction, wio.evaporation]
1 change: 0 additions & 1 deletion raven/processes/wps_raven_multi_model.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@
data_type='string',
min_occurs=0)

# This won't work until we merge the hrus with the parameters
mohyse = LiteralInput('mohyse', 'Comma separated list of MOHYSE parameters',
abstract='Parameters: ' + ', '.join(MOHYSE.params._fields),
data_type='string',
Expand Down
35 changes: 9 additions & 26 deletions tests/test_ERA5.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
from raven.models import HMETS
import json

import matplotlib.pyplot as plt

params = (9.5019, 0.2774, 6.3942, 0.6884, 1.2875, 5.4134, 2.3641, 0.0973, 0.0464, 0.1998, 0.0222, -1.0919,
2.6851, 0.3740, 1.0000, 0.4739, 0.0114, 0.0243, 0.0069, 310.7211, 916.1947)
Expand All @@ -32,7 +33,7 @@ def test_simple(self, era5_hr):
longitude=-123.3659,
rain_snow_fraction="RAINSNOW_DINGMAN",
tas={'linear_transform': (1.0, -273.15), 'time_shift': -.25},
pr={'linear_transform': (.001, 0.0), 'time_shift': -.25}
pr={'linear_transform': (24000.0, 0.0), 'time_shift': -.25}
)


Expand Down Expand Up @@ -68,35 +69,17 @@ def test_simple(self, era5_hr):
latitude=54.4848,
longitude=-123.3659,
rain_snow_fraction="RAINSNOW_DINGMAN",
pr=json.dumps({'pr': {'linear_transform': (24000.0,0.0), 'time_shift': -.25}}),
tas=json.dumps({'tas': {'linear_transform': (1.0, -273.15), 'time_shift': -.25}}),
pr=json.dumps({'pr': {'linear_transform': (.001, 0.0), 'time_shift': -.25}}),
)

resp = client.get(
service='WPS', request='Execute', version='1.0.0', identifier='raven-hmets',
datainputs=datainputs)

assert_response_success(resp)
# out = get_output(resp.xml)

# There is no diagnostic because we didn't provide observed streamflow.
# Not clear what would/should happen if we pass the TESTDATA netCDF, as it contains
# other variables.
"""
assert 'diagnostics' in out
tmp_file, _ = urlretrieve(out['diagnostics'])
tmp_content = open(tmp_file).readlines()
# checking correctness of NSE (full period 1954-2011 would be NSE=0.636015 as template in Wiki)
assert 'DIAG_NASH_SUTCLIFFE' in tmp_content[0]
idx_diag = tmp_content[0].split(',').index("DIAG_NASH_SUTCLIFFE")
diag = np.float(tmp_content[1].split(',')[idx_diag])
np.testing.assert_almost_equal(diag, -7.03141, 4, err_msg='NSE is not matching expected value')
# checking correctness of RMSE (full period 1954-2011 would be RMSE=28.3759 as template in wiki)
assert 'DIAG_RMSE' in tmp_content[0]
idx_diag = tmp_content[0].split(',').index("DIAG_RMSE")
diag = np.float(tmp_content[1].split(',')[idx_diag])
np.testing.assert_almost_equal(diag, 101.745, 4, err_msg='RMSE is not matching expected value')
"""
# out = get_output(resp.xml)
# tmp,_= urlretrieve(out['hydrograph'])
# q=xr.open_dataset(tmp)
# plt.plot(q['q_sim'])
# plt.show()
21 changes: 12 additions & 9 deletions tests/test_base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from pathlib import Path
import pytest


has_singularity = raven.raven_simg.exists()


Expand Down Expand Up @@ -76,9 +77,9 @@ def test_gr4j_with_no_tags(self):
# Algorithm: DDS # shorter sim-period and lower budget
# :StartDate 1954-01-01 00:00:00 # First tested that example below matches
# :Duration 208 #
np.testing.assert_almost_equal(opt_para, [2.423961, 3.758972, 204.3856, 5.866946, 16.60408, 0.3728098], 4,
np.testing.assert_almost_equal(opt_para, [2.424726, 3.758972, 204.3856, 5.866946, 16.60408, 0.3728098], 3,
err_msg='calibrated parameter set is not matching expected value')
np.testing.assert_almost_equal(opt_func, -0.486033, 4,
np.testing.assert_almost_equal(opt_func, -0.50717, 4,
err_msg='calibrated NSE is not matching expected value')

# # Random number seed: 123 #
Expand Down Expand Up @@ -152,14 +153,16 @@ def test_hmets_with_no_tags(self):
# Algorithm: DDS # shorter sim-period and lower budget
# :StartDate 1954-01-01 00:00:00 # First tested that example below matches
# :Duration 208 #
np.testing.assert_almost_equal(opt_para, [1.806003e+01, 3.510955e+00, 1.195340e+01, 1.413509e+00,
1.662893e+01, 1.794244e+01, -2.226484e-01, 1.391220e-01,
5.429963e-02, 2.361525e-01, 2.706042e-02, -4.562373e+00,
6.481391e-01, 5.493992e-01, 2.509283e+00, 4.213560e-01,
1.784870e-02, 7.768531e-02, 4.568809e-03, 1.147092e-01,
4.028124e-01], 4,

expected_value=[1.777842e+01, 3.317211e+00, 5.727342e+00, 1.419491e+00,
1.382141e+01, 1.637954e+01, 7.166296e-01, 1.389346e-01,
2.620464e-02, 2.245525e-01, 2.839426e-02, -2.003810e+00,
9.479623e-01, 4.803857e-01, 2.524914e+00, 4.117232e-01,
1.950058e-02, 4.494123e-02, 1.405815e-03, 2.815803e-02,
1.007823e+00]
np.testing.assert_almost_equal(opt_para, expected_value, 4,
err_msg='calibrated parameter set is not matching expected value')
np.testing.assert_almost_equal(opt_func, 2.2878, 4,
np.testing.assert_almost_equal(opt_func, 1.43474, 4,
err_msg='calibrated NSE is not matching expected value')

# # Random number seed: 123 #
Expand Down
Loading

0 comments on commit eeca117

Please sign in to comment.