From 3f59e433badd162c2520bf1e3a9904b31756f5f9 Mon Sep 17 00:00:00 2001 From: Arthi Ramachandran <4260771+arthir@users.noreply.github.com> Date: Thu, 16 Apr 2020 21:52:08 -0500 Subject: [PATCH 1/7] DOC/TST Added documentation, moved configs, added (some) tests --- README.md | 4 +- age_model/.DS_Store | Bin 10244 -> 8196 bytes .../extendedcobey.yaml | 8 +- .../extendedmodel_cobey_with_age_test.yaml | 152 ++++++++++++++++++ .../sample_experiment.yaml | 3 +- runScenarios.py | 53 +++++- sample_age4grp_experiment.yaml | 95 ----------- tests/README.md | 2 + tests/test_runScenarios.py | 68 ++++++++ 9 files changed, 275 insertions(+), 110 deletions(-) rename extendedcobey.yaml => experiment_configs/extendedcobey.yaml (94%) create mode 100644 experiment_configs/extendedmodel_cobey_with_age_test.yaml rename sample_experiment.yaml => experiment_configs/sample_experiment.yaml (77%) delete mode 100644 sample_age4grp_experiment.yaml create mode 100644 tests/README.md create mode 100644 tests/test_runScenarios.py diff --git a/README.md b/README.md index 0e911b1f..2a58e2e9 100644 --- a/README.md +++ b/README.md @@ -100,9 +100,9 @@ Note that the user-supplied configuration file is used to provide ### Usage examples: - Using the default emodl template: `python runScenarios.py - --running_location Local --region IL --experiment_config sample_experiment.yaml` + --running_location Local --region IL --experiment_config ./experiment_configs/sample_experiment.yaml` - Using a different emodl template: `python runScenarios.py - --running_location Local --region IL --experiment_config sample_experiment.yaml --emodl_template simplemodel_testing.emodl` + --running_location Local --region IL --experiment_config ./experiment_configs/sample_experiment.yaml --emodl_template simplemodel_testing.emodl` ### 4.3. Postprocessing and visualizing results - latest postprocessing file that calculates incidences for extended SEIR model [extended_model_postprocessing.py](https://github.com/numalariamodeling/covid-chicago/blob/master/extended_model_postprocessing.py) diff --git a/age_model/.DS_Store b/age_model/.DS_Store index 58cee0426ac95c5316f5f4ab5a49dd578fac8343..e6ac345a8f19d170b0ad1b5753dd5ddd86952007 100644 GIT binary patch delta 114 zcmZn(XmOBWU|?W$DortDU;r^WfEYvza8E20o2aMA&jXSN@)-<+lk;;6HWp4}pV%P0 znVo}$gOO)*iQsQ`L1v&5Aduh&60RVv8wo@eUjR55mD0A5QJ AMgRZ+ delta 421 zcmZp1XbF&DU|?W$DortDU{C-uIe-{M3-C-#6q~50C@KQvFar4u40#Mm4Dk%f3>gfm z8xt3^Z)WFU;Yb8Y3o+<1WHJ;mR3a+@D$E9o_". + e.g. the contact matrix + - sampling: Any of the functions available in np.random can be used to randomly samply values for the parameter. + Arguments are passed to the sampling function as kwargs (which are specified in the yaml). + - DateToTimestep: This is a custom function that is supported to compute the amount of time + from an intervention date. e.g. socialDistance_time + - subtract: This subtracts one column in the dataframe (x2) from another (x1). + e.g. SpeciesS (given N and initialAs) + Returns + ------- + df: pd.DataFrame + dataframe with the additional column(s) added + """ if isinstance(parameter_function, int): df[parameter] = parameter_function elif 'matrix' in parameter_function: @@ -41,9 +67,9 @@ def add_config_parameter_column(df, parameter, parameter_function): function_name = parameter_function['custom_function'] function_kwargs = parameter_function['function_kwargs'] if function_name == 'DateToTimestep': - function_kwargs['startdate'] = first_day - df[parameter] = [globals()[function_name](**function_kwargs) for i in range(len(df))] - # Note that the custom_function needs to be imported + startdate_col = function_kwargs['startdate_col'] + df[parameter] = [DateToTimestep(function_kwargs['dates'], df[startdate_col][i]) + for i in range(len(df))] elif function_name == 'subtract': df[parameter] = df[function_kwargs['x1']] - df[function_kwargs['x2']] else: @@ -52,6 +78,8 @@ def add_config_parameter_column(df, parameter, parameter_function): def add_fixed_parameters_region_specific(df, config, region): + """ For each of the region-specific parameters, iteratively add them to the parameters dataframe + """ for parameter_group, parameter_group_values in config['fixed_parameters_region_specific'].items(): if parameter_group in ('populations', 'startdate'): continue @@ -61,6 +89,9 @@ def add_fixed_parameters_region_specific(df, config, region): def add_computed_parameters(df): + """ Parameters that are computed from other parameters are computed and added to the parameters + dataframe. + """ df['fraction_dead'] = df.apply(lambda x: x['cfr'] / x['fraction_severe'], axis=1) df['fraction_hospitalized'] = df.apply(lambda x: 1 - x['fraction_critical'] - x['fraction_dead'], axis=1) return df @@ -115,6 +146,10 @@ def replaceParameters(df, Ki_i, sample_nr, emodl_template, scen_num): raise ValueError("Not all placeholders have been replaced in the template emodl file. " f"Remaining placeholders: {remaining_placeholders}") fin.close() + remaining_placeholders = re.findall(r'@\w+@', data) + if remaining_placeholders: + raise ValueError("Not all placeholders have been replaced in the template emodl file. " + f"Remaining placeholders: {remaining_placeholders}") fin = open(os.path.join(temp_dir, f"simulation_{scen_num}.emodl"), "wt") fin.write(data) fin.close() @@ -163,7 +198,7 @@ def generateScenarios(simulation_population, Kivalues, duration, monitoring_samp def get_experiment_config(experiment_config_file): - config = yaml.load(open(DEFAULT_CONFIG), Loader=yaml.FullLoader) + config = yaml.load(open(BASE_CONFIG), Loader=yaml.FullLoader) yaml_file = open(experiment_config_file) expt_config = yaml.load(yaml_file, Loader=yaml.FullLoader) for param_type, updated_params in expt_config.items(): @@ -288,9 +323,11 @@ def parse_args(): sub_samples=experiment_setup_parameters['number_of_samples'], duration=experiment_setup_parameters['duration'], monitoring_samples=experiment_setup_parameters['monitoring_samples'], - modelname=args.emodl_template, first_day=first_day, Location=Location, + modelname=args.emodl_template, + first_day=first_day, Location=Location, experiment_config=experiment_config) + exit() generateSubmissionFile( nscen, exp_name, trajectories_dir, temp_dir, temp_exp_dir, exe_dir=exe_dir, docker_image=docker_image) diff --git a/sample_age4grp_experiment.yaml b/sample_age4grp_experiment.yaml deleted file mode 100644 index 7363b192..00000000 --- a/sample_age4grp_experiment.yaml +++ /dev/null @@ -1,95 +0,0 @@ -experiment_setup_parameters: - 'number_of_samples': 2 - 'number_of_runs': 1 -fixed_parameters_region_specific: - N_age_4grp: - 'NMH_catchment': - 'N_age0to19': 924766 - 'N_age20to39': 1135240 - 'N_age40to59': 941609 - 'N_age60to100': 713926 - #'Chicago': , - #'IL': , - 'EMS_1': - 'N_age0to19': 170342 - 'N_age20to39': 184052 - 'N_age40to59': 203274 - 'N_age60to100': 182064 - 'EMS_2': - 'N_age0to19': 246836 - 'N_age20to39': 291953 - 'N_age40to59': 311148 - 'N_age60to100': 279459 - 'EMS_3': - 'N_age0to19': 128558 - 'N_age20to39': 147943 - 'N_age40to59': 181366 - 'N_age60to100': 163605 - 'EMS_4': - 'N_age0to19': 156230 - 'N_age20to39': 178549 - 'N_age40to59': 199940 - 'N_age60to100': 166815 - 'EMS_5': - 'N_age0to19': 89601 - 'N_age20to39': 103718 - 'N_age40to59': 113788 - 'N_age60to100': 112211 - 'EMS_6': - 'N_age0to19': 170529 - 'N_age20to39': 215422 - 'N_age40to59': 212023 - 'N_age60to100': 194309 - 'EMS_7': - 'N_age0to19': 459863 - 'N_age20to39': 455969 - 'N_age40to59': 532097 - 'N_age60to100': 377955 - 'EMS_8': - 'N_age0to19': 406488 - 'N_age20to39': 439749 - 'N_age40to59': 492527 - 'N_age60to100': 344784 - 'EMS_9': - 'N_age0to19': 482757 - 'N_age20to39': 506403 - 'N_age40to59': 591629 - 'N_age60to100': 399718 - 'EMS_10': - 'N_age0to19': 252497 - 'N_age20to39': 252957 - 'N_age40to59': 313926 - 'N_age60to100': 238921 - 'EMS_11': - 'N_age0to19': 656456 - 'N_age20to39': 889748 - 'N_age40to59': 697571 - 'N_age60to100': 500905 -fixed_parameters_global: - 'initialAs_age0to19': 3 - 'initialAs_age20to39': 3 - 'initialAs_age40to59': 3 - 'initialAs_age60to100': 3 - 'speciesS_age0to19': - custom_function: subtract - function_kwargs: {'x1': N_age0to19, 'x2': initialAs_age0to19} - 'speciesS_age20to39': - custom_function: subtract - function_kwargs: {'x1': N_age20to39, 'x2': initialAs_age20to39} - 'speciesS_age40to59': - custom_function: subtract - function_kwargs: {'x1': N_age40to59, 'x2': initialAs_age40to59} - 'speciesS_age60to100': - custom_function: subtract - function_kwargs: {'x1': N_age60to100, 'x2': initialAs_age60to100} - C: - matrix: - - [0.425, 0.109, 0.068, 0.155] - - [0.109, 0.989, 0.150, 0.122] - - [0.068, 0.150, 0.483, 0.208] - - [0.155, 0.122, 0.208, 0.369] -sampled_parameters: - 'incubation_pd': - np.random: uniform - function_kwargs: {'low': 1, 'high': 2} -fitted_parameters: diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 00000000..7a538db1 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,2 @@ +To run unit tests, run the following on the command line: +```python -m pytest tests``` diff --git a/tests/test_runScenarios.py b/tests/test_runScenarios.py new file mode 100644 index 00000000..3832c296 --- /dev/null +++ b/tests/test_runScenarios.py @@ -0,0 +1,68 @@ +from datetime import datetime + +import pandas as pd +import pytest + +from runScenarios import add_config_parameter_column + + +@pytest.fixture +def original_df(): + return pd.DataFrame({'sample_number': [1, 2, 3, 4, 5]}) + + +def test_add_config_parameter_column__int(original_df): + new_df = add_config_parameter_column(original_df, "new_column", 10) + assert new_df.shape == (5, 2) + assert "new_column" in new_df.columns + assert set(new_df["new_column"]) == set([10]) + + +def test_add_config_parameter_column__matrix(original_df): + f = {'matrix': [[9, 8], [7, 6]]} + new_df = add_config_parameter_column(original_df, "new_column", f) + assert new_df.shape == (5, 5) + assert all(new_cols in new_df.columns + for new_cols in ["new_column1_1", "new_column1_2", "new_column2_1", "new_column2_2"]) + assert set(new_df["new_column1_1"]) == set([9]) + assert set(new_df["new_column1_2"]) == set([8]) + assert set(new_df["new_column2_1"]) == set([7]) + assert set(new_df["new_column2_2"]) == set([6]) + + +def test_add_config_parameter_column__random_uniform(original_df): + f = {'np.random': 'uniform', 'function_kwargs': {'low': 5, 'high': 6}} + new_df = add_config_parameter_column(original_df, "new_column", f) + assert new_df.shape == (5, 2) + assert "new_column" in new_df.columns + assert all((new_df["new_column"] >= 5) & (new_df["new_column"] <= 6)) + + +def test_add_config_parameter_column__datetotimestep(): + df = pd.DataFrame({'sample_number': [1, 2, 3, 4, 5], + 'startdate': [datetime(2020, 2, 20)]*5}) + f = {'custom_function': 'DateToTimestep', + 'function_kwargs': {'dates': datetime(2020, 3, 1), 'startdate_col': 'startdate'}} + new_df = add_config_parameter_column(df, "new_column", f) + assert new_df.shape == (5, 3) + assert "new_column" in new_df.columns + assert set(new_df["new_column"]) == set([10]) + + +def test_add_config_parameter_column__subtract(): + df = pd.DataFrame({'sample_number': [1, 2, 3, 4, 5], + 'col1': [2, 4, 6, 8, 10], + 'col2': [1, 3, 5, 7, 9]}) + f = {'custom_function': 'subtract', + 'function_kwargs': {'x1': 'col1', 'x2': 'col2'}} + new_df = add_config_parameter_column(df, "new_column", f) + assert new_df.shape == (5, 4) + assert "new_column" in new_df.columns + assert set(new_df["new_column"]) == set([1]) + + +def test_add_config_parameter_column__error(): + f = {'weird_function': {}} + with pytest.raises(ValueError) as e: + add_config_parameter_column(pd.DataFrame, "new_column", f) + assert "Unknown type of parameter" in str(e.value) From a5a37331a21f629d836886daffffe7abeb8bf8f9 Mon Sep 17 00:00:00 2001 From: Arthi Ramachandran <4260771+arthir@users.noreply.github.com> Date: Thu, 16 Apr 2020 21:52:28 -0500 Subject: [PATCH 2/7] Adding dev-requirements --- dev-requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 dev-requirements.txt diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 00000000..bb06fc3b --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1 @@ +pytest>=5 From 90c10d7b27998591cb8f19b7d9f5b87a6b7f0b01 Mon Sep 17 00:00:00 2001 From: Arthi Ramachandran <4260771+arthir@users.noreply.github.com> Date: Thu, 16 Apr 2020 22:08:20 -0500 Subject: [PATCH 3/7] datetotimestep is computing based on another column, instead of a constant --- runScenarios.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/runScenarios.py b/runScenarios.py index 0dbb479b..75c9c079 100644 --- a/runScenarios.py +++ b/runScenarios.py @@ -43,7 +43,7 @@ def add_config_parameter_column(df, parameter, parameter_function, first_day=Non e.g. the contact matrix - sampling: Any of the functions available in np.random can be used to randomly samply values for the parameter. Arguments are passed to the sampling function as kwargs (which are specified in the yaml). - - DateToTimestep: This is a custom function that is supported to compute the amount of time + - DateToTimestep: This is a custom function that is supported to compute the amount of time from an intervention date. e.g. socialDistance_time - subtract: This subtracts one column in the dataframe (x2) from another (x1). e.g. SpeciesS (given N and initialAs) @@ -90,23 +90,23 @@ def add_fixed_parameters_region_specific(df, config, region): def add_computed_parameters(df): """ Parameters that are computed from other parameters are computed and added to the parameters - dataframe. + dataframe. """ df['fraction_dead'] = df.apply(lambda x: x['cfr'] / x['fraction_severe'], axis=1) df['fraction_hospitalized'] = df.apply(lambda x: 1 - x['fraction_critical'] - x['fraction_dead'], axis=1) return df -def generateParameterSamples(samples, pop, first_day, config): +def generateParameterSamples(samples, pop, config): """ Given a yaml configuration file (e.g. ./extendedcobey.yaml), generate a dataframe of the parameters for a simulation run using the specified functions/sampling mechansims. - Supported functions are in the FUNCTIONS variable. """ df = pd.DataFrame() df['sample_num'] = range(samples) df['speciesS'] = pop df['initialAs'] = config['experiment_setup_parameters']['initialAs'] + df['startdate'] = experiment_config['fixed_parameters_region_specific']['startdate'][region] for parameter, parameter_function in config['sampled_parameters'].items(): df = add_config_parameter_column(df, parameter, parameter_function) @@ -159,7 +159,7 @@ def generateScenarios(simulation_population, Kivalues, duration, monitoring_samp nruns, sub_samples, modelname, first_day, Location, experiment_config): lst = [] scen_num = 0 - dfparam = generateParameterSamples(samples=sub_samples, pop=simulation_population, first_day=first_day, + dfparam = generateParameterSamples(samples=sub_samples, pop=simulation_population, config=experiment_config) for sample in range(sub_samples): From 3a0fe07f8d0d3a40b540a0056f3ed6a41a217d3d Mon Sep 17 00:00:00 2001 From: Arthi Ramachandran <4260771+arthir@users.noreply.github.com> Date: Mon, 20 Apr 2020 14:53:00 -0500 Subject: [PATCH 4/7] cleaning up --- experiment_configs/extendedcobey.yaml | 2 +- .../extendedmodel_cobey_with_age_test.yaml | 152 ------------------ 2 files changed, 1 insertion(+), 153 deletions(-) delete mode 100644 experiment_configs/extendedmodel_cobey_with_age_test.yaml diff --git a/experiment_configs/extendedcobey.yaml b/experiment_configs/extendedcobey.yaml index 1337e68b..f2ced5d0 100644 --- a/experiment_configs/extendedcobey.yaml +++ b/experiment_configs/extendedcobey.yaml @@ -35,7 +35,7 @@ fixed_parameters_region_specific: 'EMS_8': 2020-02-28 'EMS_9': 2020-02-28 'EMS_10': 2020-02-28 - 'EMS_11': 2020-02-28 + 'EMS_11': 2020-02-28 fixed_parameters_global: sampled_parameters: 'incubation_pd': diff --git a/experiment_configs/extendedmodel_cobey_with_age_test.yaml b/experiment_configs/extendedmodel_cobey_with_age_test.yaml deleted file mode 100644 index 5bbef30d..00000000 --- a/experiment_configs/extendedmodel_cobey_with_age_test.yaml +++ /dev/null @@ -1,152 +0,0 @@ -experiment_setup_parameters: - 'number_of_samples': 2 - 'number_of_runs': 1 - 'duration': 365 - 'monitoring_samples': 365 # needs to be smaller than duration - 'random_seed': 19 -fixed_parameters: - populations: - 'IL': 12830632 - 'NMH_catchment': 315000 - 'Chicago': 2700000 - 'EMS_1': 736370 - 'EMS_2': 1124941 - 'EMS_3': 619366 - 'EMS_4': 698886 - 'EMS_5': 417674 - 'EMS_6': 788985 - 'EMS_7': 1814891 - 'EMS_8': 1673408 - 'EMS_9': 1970275 - 'EMS_10': 1052839 - 'EMS_11': 2716921 - startdate: - 'NMH_catchment': 2020-02-28 - 'Chicago': 2020-02-20 - 'IL': 2020-02-28 - 'EMS_1': 2020-02-28 - 'EMS_2': 2020-02-28 - 'EMS_3': 2020-02-28 - 'EMS_4': 2020-02-28 - 'EMS_5': 2020-02-28 - 'EMS_6': 2020-02-28 - 'EMS_7': 2020-02-28 - 'EMS_8': 2020-02-28 - 'EMS_9': 2020-02-28 - 'EMS_10': 2020-02-28 - 'EMS_11': 2020-02-28 -sampled_parameters: - 'incubation_pd': - np.random: uniform - function_kwargs: {'low': 4.2, 'high': 6.63} - 'time_to_symptoms': - np.random: uniform - function_kwargs: {'low': 1, 'high':5} - 'time_to_hospitalization': - np.random: uniform - function_kwargs: {'low':2, 'high':10} - 'time_to_critical': - np.random: uniform - function_kwargs: {'low':4, 'high':9} - 'time_to_death': - np.random: uniform - function_kwargs: {'low':3, 'high':11} - 'recovery_rate_asymp': - np.random: uniform - function_kwargs: {'low':6, 'high':16} - 'recovery_rate_mild': - np.random: uniform - function_kwargs: {'low':19.4, 'high':21.3} - 'recovery_rate_hosp': - np.random: uniform - function_kwargs: {'low':19.5, 'high':21.1} - 'recovery_rate_crit': - np.random: uniform - function_kwargs: {'low':25.3, 'high':31.6} - 'fraction_symptomatic': - np.random: uniform - function_kwargs: {'low':0.5, 'high':0.8} - 'fraction_severe': - np.random: uniform - function_kwargs: {'low':0.2, 'high':0.5} - 'fraction_critical': - np.random: uniform - function_kwargs: {'low':0.2, 'high':0.5} - 'cfr': - np.random: uniform - function_kwargs: {'low':0.002675, 'high':0.007775} - 'reduced_inf_of_det_cases': - np.random: uniform - function_kwargs: {'low':0.5, 'high':0.9} - 'd_Sym': - np.random: uniform - function_kwargs: {'low':0.2, 'high':0.3} - 'd_Sys': - np.random: uniform - function_kwargs: {'low':0.7, 'high':0.9} - 'd_As': - np.random: uniform - function_kwargs: {'low':0, 'high':0} - 'social_multiplier_1': - np.random: uniform - function_kwargs: {'low':0.9, 'high':1} - 'social_multiplier_2': - np.random: uniform - function_kwargs: {'low':0.6, 'high':0.9} - 'social_multiplier_3': - np.random: uniform - function_kwargs: {'low':0.05, 'high':0.3} - 'socialDistance_time1': - custom_function: DateToTimestep - function_kwargs: {'dates': 2020-03-12} - 'socialDistance_time2': - custom_function: DateToTimestep - function_kwargs: {'dates': 2020-03-17} - 'socialDistance_time3': - custom_function: DateToTimestep - function_kwargs: {'dates': 2020-03-21} -fitted_parameters: - # Note IL, NMH, Chicago and EMS_3 were fitted, the other EMS area Kis need updating/checking - Kis: - 'NMH_catchment': - np: linspace - function_kwargs: {'start': 1.5e-6, 'stop': 2e-6, 'num': 3} - 'Chicago': - np: linspace - function_kwargs: {'start': 2e-7, 'stop': 3e-7, 'num': 3} - 'IL': - np: linspace - function_kwargs: {'start': 3.5e-8, 'stop': 5.3e-8, 'num': 3} - 'EMS_1': - np: linspace - function_kwargs: {'start': 5e-7, 'stop': 9e-7, 'num': 3} - 'EMS_2': - np: linspace - function_kwargs: {'start': 5e-7, 'stop': 9e-7, 'num': 3} - 'EMS_3': - np: linspace - function_kwargs: {'start': 5e-7, 'stop': 9e-7, 'num': 3} - 'EMS_4': - np: linspace - function_kwargs: {'start': 5e-7, 'stop': 9e-7, 'num': 3} - 'EMS_5': - np: linspace - function_kwargs: {'start': 5e-7, 'stop': 9e-7, 'num': 3} - 'EMS_6': - np: linspace - function_kwargs: {'start': 5e-7, 'stop': 9e-7, 'num': 3} - 'EMS_7': - np: linspace - function_kwargs: {'start': 5e-7, 'stop': 9e-7, 'num': 3} - 'EMS_8': - np: linspace - function_kwargs: {'start': 5e-7, 'stop': 9e-7, 'num': 3} - 'EMS_9': - np: linspace - function_kwargs: {'start': 5e-7, 'stop': 9e-7, 'num': 3} - 'EMS_10': - np: linspace - function_kwargs: {'start': 5e-7, 'stop': 9e-7, 'num': 3} - 'EMS_11': - np: linspace - function_kwargs: {'start': 5e-7, 'stop': 9e-7, 'num': 3} From c42583c1b274a3cf080ef2afbb1725d667c5b182 Mon Sep 17 00:00:00 2001 From: Arthi Ramachandran <4260771+arthir@users.noreply.github.com> Date: Mon, 20 Apr 2020 15:03:26 -0500 Subject: [PATCH 5/7] Fixing merge --- experiment_configs/extendedcobey.yaml | 4 +- .../sample_age4grp_experiment.yaml | 103 +++++------------- 2 files changed, 28 insertions(+), 79 deletions(-) diff --git a/experiment_configs/extendedcobey.yaml b/experiment_configs/extendedcobey.yaml index f2ced5d0..d1b99a42 100644 --- a/experiment_configs/extendedcobey.yaml +++ b/experiment_configs/extendedcobey.yaml @@ -109,10 +109,10 @@ sampled_parameters: function_kwargs: {'dates': 2020-03-17, 'startdate_col': 'startdate'} 'socialDistance_time3': custom_function: DateToTimestep - function_kwargs: {'dates': 2020-03-21} + function_kwargs: {'dates': 2020-03-21, 'startdate_col': 'startdate'} 'socialDistanceSTOP_time': custom_function: DateToTimestep - function_kwargs: {'dates': 2020-04-24} + function_kwargs: {'dates': 2020-04-24, 'startdate_col': 'startdate'} fitted_parameters: # Note IL, NMH, Chicago and EMS_3 were fitted, the other EMS area Kis need updating/checking Kis: diff --git a/experiment_configs/sample_age4grp_experiment.yaml b/experiment_configs/sample_age4grp_experiment.yaml index 7363b192..ddf810ed 100644 --- a/experiment_configs/sample_age4grp_experiment.yaml +++ b/experiment_configs/sample_age4grp_experiment.yaml @@ -1,87 +1,36 @@ experiment_setup_parameters: 'number_of_samples': 2 'number_of_runs': 1 + age_bins: + - age0to19 + - age20to39 + - age40to59 + - age60to100 fixed_parameters_region_specific: - N_age_4grp: - 'NMH_catchment': - 'N_age0to19': 924766 - 'N_age20to39': 1135240 - 'N_age40to59': 941609 - 'N_age60to100': 713926 - #'Chicago': , + N: + expand_by_age: True + 'NMH_catchment': [78401,96245,79829,60526] + #'Chicago': , #'IL': , - 'EMS_1': - 'N_age0to19': 170342 - 'N_age20to39': 184052 - 'N_age40to59': 203274 - 'N_age60to100': 182064 - 'EMS_2': - 'N_age0to19': 246836 - 'N_age20to39': 291953 - 'N_age40to59': 311148 - 'N_age60to100': 279459 - 'EMS_3': - 'N_age0to19': 128558 - 'N_age20to39': 147943 - 'N_age40to59': 181366 - 'N_age60to100': 163605 - 'EMS_4': - 'N_age0to19': 156230 - 'N_age20to39': 178549 - 'N_age40to59': 199940 - 'N_age60to100': 166815 - 'EMS_5': - 'N_age0to19': 89601 - 'N_age20to39': 103718 - 'N_age40to59': 113788 - 'N_age60to100': 112211 - 'EMS_6': - 'N_age0to19': 170529 - 'N_age20to39': 215422 - 'N_age40to59': 212023 - 'N_age60to100': 194309 - 'EMS_7': - 'N_age0to19': 459863 - 'N_age20to39': 455969 - 'N_age40to59': 532097 - 'N_age60to100': 377955 - 'EMS_8': - 'N_age0to19': 406488 - 'N_age20to39': 439749 - 'N_age40to59': 492527 - 'N_age60to100': 344784 - 'EMS_9': - 'N_age0to19': 482757 - 'N_age20to39': 506403 - 'N_age40to59': 591629 - 'N_age60to100': 399718 - 'EMS_10': - 'N_age0to19': 252497 - 'N_age20to39': 252957 - 'N_age40to59': 313926 - 'N_age60to100': 238921 - 'EMS_11': - 'N_age0to19': 656456 - 'N_age20to39': 889748 - 'N_age40to59': 697571 - 'N_age60to100': 500905 + 'EMS_1': [170342, 184052, 203274, 182064] + 'EMS_2': [246836, 291953, 311148, 279459] + 'EMS_3': [128558, 147943, 181366, 163605] + 'EMS_4': [156230, 178549, 199940, 166815] + 'EMS_5': [89601, 103718, 113788, 112211] + 'EMS_6': [170529, 215422, 212023, 194309] + 'EMS_7': [459863, 455969, 532097, 377955] + 'EMS_8': [406488, 439749, 492527, 344784] + 'EMS_9': [482757, 506403, 591629, 399718] + 'EMS_10': [252497, 252957, 313926, 238921] + 'EMS_11': [656456, 889748, 697571, 500905] fixed_parameters_global: - 'initialAs_age0to19': 3 - 'initialAs_age20to39': 3 - 'initialAs_age40to59': 3 - 'initialAs_age60to100': 3 - 'speciesS_age0to19': + initialAs: + expand_by_age: True + list: [3, 3, 3, 3] + speciesS: + expand_by_age: True custom_function: subtract - function_kwargs: {'x1': N_age0to19, 'x2': initialAs_age0to19} - 'speciesS_age20to39': - custom_function: subtract - function_kwargs: {'x1': N_age20to39, 'x2': initialAs_age20to39} - 'speciesS_age40to59': - custom_function: subtract - function_kwargs: {'x1': N_age40to59, 'x2': initialAs_age40to59} - 'speciesS_age60to100': - custom_function: subtract - function_kwargs: {'x1': N_age60to100, 'x2': initialAs_age60to100} + function_kwargs: {'x1': N, 'x2': initialAs} C: matrix: - [0.425, 0.109, 0.068, 0.155] From 319abfe5dc15a3889ca36e884f95e1f6240d8295 Mon Sep 17 00:00:00 2001 From: Arthi Ramachandran <4260771+arthir@users.noreply.github.com> Date: Mon, 20 Apr 2020 15:12:35 -0500 Subject: [PATCH 6/7] TST making fixes based on tests --- runScenarios.py | 8 ++++---- tests/test_runScenarios.py | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/runScenarios.py b/runScenarios.py index 660cc14f..8b8651e6 100644 --- a/runScenarios.py +++ b/runScenarios.py @@ -36,8 +36,8 @@ def _parse_config_parameter(df, parameter, parameter_function): function_kwargs = parameter_function['function_kwargs'] if function_name == 'DateToTimestep': startdate_col = function_kwargs['startdate_col'] - df[parameter] = [DateToTimestep(function_kwargs['dates'], df[startdate_col][i]) - for i in range(len(df))] + return [DateToTimestep(function_kwargs['dates'], df[startdate_col][i]) + for i in range(len(df))] elif function_name == 'subtract': return df[function_kwargs['x1']] - df[function_kwargs['x2']] else: @@ -78,7 +78,7 @@ def add_config_parameter_column(df, parameter, parameter_function, age_bins=None df: pd.DataFrame dataframe with the additional column(s) added """ - if parameter_function.get('expand_by_age'): + if isinstance(parameter_function, dict) and parameter_function.get('expand_by_age'): if not age_bins: raise ValueError("Ages bins must be specified if using an age expansion") if 'list' in parameter_function: @@ -99,7 +99,7 @@ def add_config_parameter_column(df, parameter, parameter_function, age_bins=None else: raise ValueError(f"Unknown type of parameter {parameter} for expand_by_age") else: - if 'matrix' in parameter_function: + if isinstance(parameter_function, dict) and 'matrix' in parameter_function: m = parameter_function['matrix'] for i, row in enumerate(m): for j, item in enumerate(row): diff --git a/tests/test_runScenarios.py b/tests/test_runScenarios.py index 3832c296..d3d6691c 100644 --- a/tests/test_runScenarios.py +++ b/tests/test_runScenarios.py @@ -46,6 +46,7 @@ def test_add_config_parameter_column__datetotimestep(): new_df = add_config_parameter_column(df, "new_column", f) assert new_df.shape == (5, 3) assert "new_column" in new_df.columns + print(new_df) assert set(new_df["new_column"]) == set([10]) From 863472909fb763887d1ab28746435a2842ac1fa5 Mon Sep 17 00:00:00 2001 From: Arthi Ramachandran <4260771+arthir@users.noreply.github.com> Date: Mon, 20 Apr 2020 17:17:17 -0500 Subject: [PATCH 7/7] TST cleaning up test cases --- tests/test_runScenarios.py | 43 ++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/tests/test_runScenarios.py b/tests/test_runScenarios.py index d3d6691c..fe42eb27 100644 --- a/tests/test_runScenarios.py +++ b/tests/test_runScenarios.py @@ -13,21 +13,25 @@ def original_df(): def test_add_config_parameter_column__int(original_df): new_df = add_config_parameter_column(original_df, "new_column", 10) - assert new_df.shape == (5, 2) - assert "new_column" in new_df.columns - assert set(new_df["new_column"]) == set([10]) + + correct_df = pd.DataFrame({ + 'sample_number': [1, 2, 3, 4, 5], + 'new_column': [10]*5}) + pd.testing.assert_frame_equal(new_df, correct_df) def test_add_config_parameter_column__matrix(original_df): f = {'matrix': [[9, 8], [7, 6]]} new_df = add_config_parameter_column(original_df, "new_column", f) assert new_df.shape == (5, 5) - assert all(new_cols in new_df.columns - for new_cols in ["new_column1_1", "new_column1_2", "new_column2_1", "new_column2_2"]) - assert set(new_df["new_column1_1"]) == set([9]) - assert set(new_df["new_column1_2"]) == set([8]) - assert set(new_df["new_column2_1"]) == set([7]) - assert set(new_df["new_column2_2"]) == set([6]) + correct_df = pd.DataFrame({ + 'sample_number': [1, 2, 3, 4, 5], + 'new_column1_1': [9]*5, + 'new_column1_2': [8]*5, + 'new_column2_1': [7]*5, + 'new_column2_2': [6]*5, + }) + pd.testing.assert_frame_equal(new_df, correct_df) def test_add_config_parameter_column__random_uniform(original_df): @@ -44,10 +48,11 @@ def test_add_config_parameter_column__datetotimestep(): f = {'custom_function': 'DateToTimestep', 'function_kwargs': {'dates': datetime(2020, 3, 1), 'startdate_col': 'startdate'}} new_df = add_config_parameter_column(df, "new_column", f) - assert new_df.shape == (5, 3) - assert "new_column" in new_df.columns - print(new_df) - assert set(new_df["new_column"]) == set([10]) + correct_df = pd.DataFrame({ + 'sample_number': [1, 2, 3, 4, 5], + 'startdate': [datetime(2020, 2, 20)]*5, + 'new_column': [10]*5}) + pd.testing.assert_frame_equal(new_df, correct_df) def test_add_config_parameter_column__subtract(): @@ -57,13 +62,15 @@ def test_add_config_parameter_column__subtract(): f = {'custom_function': 'subtract', 'function_kwargs': {'x1': 'col1', 'x2': 'col2'}} new_df = add_config_parameter_column(df, "new_column", f) - assert new_df.shape == (5, 4) - assert "new_column" in new_df.columns - assert set(new_df["new_column"]) == set([1]) + correct_df = pd.DataFrame({ + 'sample_number': [1, 2, 3, 4, 5], + 'col1': [2, 4, 6, 8, 10], + 'col2': [1, 3, 5, 7, 9], + 'new_column': [1]*5}) + pd.testing.assert_frame_equal(new_df, correct_df) def test_add_config_parameter_column__error(): f = {'weird_function': {}} - with pytest.raises(ValueError) as e: + with pytest.raises(ValueError, match="Unknown type of parameter"): add_config_parameter_column(pd.DataFrame, "new_column", f) - assert "Unknown type of parameter" in str(e.value)