Skip to content

Commit

Permalink
modified examples
Browse files Browse the repository at this point in the history
  • Loading branch information
schwemro committed Aug 8, 2023
1 parent f0fb9b4 commit 735e6f9
Show file tree
Hide file tree
Showing 21 changed files with 132 additions and 74 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,9 @@ def nanmeanweighted(y, w, axis=None):

base_path = Path(__file__).parent
# directory of results
base_path_results = base_path / "output"
if not os.path.exists(base_path_results):
os.mkdir(base_path_results)
base_path_output = base_path / "output"
if not os.path.exists(base_path_output):
os.mkdir(base_path_output)
# directory of figures
base_path_figs = base_path / "figures"
if not os.path.exists(base_path_figs):
Expand Down Expand Up @@ -189,11 +189,11 @@ def nanmeanweighted(y, w, axis=None):
glm_location_formulas = {
"interaction_theta": "y ~ theta_pwp * theta_ufc * theta_ac + ks",
"no_interaction": "y ~ theta_pwp + theta_ufc + theta_ac + ks",
"theta": "y ~ theta_pwp + theta_ufc + theta_ac",
"theta_pwp": "y ~ theta_pwp",
"theta_ufc": "y ~ theta_ufc",
"theta_ac": "y ~ theta_ac",
"ks": "y ~ ks",
# "theta": "y ~ theta_pwp + theta_ufc + theta_ac",
# "theta_pwp": "y ~ theta_pwp",
# "theta_ufc": "y ~ theta_ufc",
# "theta_ac": "y ~ theta_ac",
# "ks": "y ~ ks",
}

vars_sim = ["transp", "q_ss", "theta", "rt50_s", "tt50_q_ss", "tt50_transp"]
Expand Down Expand Up @@ -295,25 +295,25 @@ def nanmeanweighted(y, w, axis=None):
soil_depth
][future]["AICC"] = smem.aicc(ll_values, nobs, res.params.shape[0])

# # plot the line fit
# fig, ax = plt.subplots(figsize=(3, 3))
# ax.scatter(yhat, y, color='black', s=4)
# line_fit = sm.OLS(y, sm.add_constant(yhat, prepend=True)).fit()
# abline_plot(model_results=line_fit, ax=ax, color='black')
# ax.set_ylabel(f'{_lab[delta]}{_lab_unit1[var_sim]} (RoGeR) [%]')
# ax.set_xlabel(f'{_lab[delta]}{_lab_unit1[var_sim]} (GLM) [%]')
# fig.tight_layout()
# file = base_path_figs / "residuals" / f"{glm_key}_{location}_{delta}_{land_cover_scenario}_{climate_scenario}_{soil_depth}_{future}_line_fit.png"
# fig.savefig(file, dpi=300)

# # plot the residuals
# fig, ax = plt.subplots(figsize=(3, 3))
# ax.scatter(yhat, res.resid_pearson, color='black', s=4)
# ax.set_ylabel(f'{_lab_unit1[var_sim]} (RoGeR) [%]')
# ax.set_xlabel(f'{_lab[delta]}{_lab_unit1[var_sim]} (GLM)')
# fig.tight_layout()
# file = base_path_figs / "residuals" / f"{glm_key}_{location}_{var_sim}_{delta}_{land_cover_scenario}_{climate_scenario}_{soil_depth}_{future}_residuals.png"
# fig.savefig(file, dpi=300)
# plot the line fit
fig, ax = plt.subplots(figsize=(3, 3))
ax.scatter(yhat, y, color='black', s=4)
line_fit = sm.OLS(y, sm.add_constant(yhat, prepend=True)).fit()
abline_plot(model_results=line_fit, ax=ax, color='black')
ax.set_ylabel(f'{_lab[delta]}{_lab_unit1[var_sim]} (RoGeR) [%]')
ax.set_xlabel(f'{_lab[delta]}{_lab_unit1[var_sim]} (GLM) [%]')
fig.tight_layout()
file = base_path_figs / "residuals" / "GLM" / f"{glm_key}_{location}_{delta}_{land_cover_scenario}_{climate_scenario}_{soil_depth}_{future}_line_fit.png"
fig.savefig(file, dpi=300)

# plot the residuals
fig, ax = plt.subplots(figsize=(3, 3))
ax.scatter(yhat, res.resid_pearson, color='black', s=4)
ax.set_ylabel(f'{_lab_unit1[var_sim]} (RoGeR) [%]')
ax.set_xlabel(f'{_lab[delta]}{_lab_unit1[var_sim]} (GLM)')
fig.tight_layout()
file = base_path_figs / "residuals" / "GLM" / f"{glm_key}_{location}_{var_sim}_{delta}_{land_cover_scenario}_{climate_scenario}_{soil_depth}_{future}_residuals.png"
fig.savefig(file, dpi=300)

# fig, ax = plt.subplots(figsize=(3, 3))
# ax.scatter(x.loc[:, 'theta_ufc'].values, y, color='black', s=4)
Expand Down Expand Up @@ -347,16 +347,16 @@ def nanmeanweighted(y, w, axis=None):
# file = base_path_figs / "residuals" / f"{glm_key}_{location}_{var_sim}_{delta}_{land_cover_scenario}_{climate_scenario}_{soil_depth}_{future}_ks.png"
# fig.savefig(file, dpi=300)

# # plot the distribution of residuals
# fig, ax = plt.subplots(figsize=(3, 3))
# resid = res.resid_deviance.copy()
# resid_std = stats.zscore(resid)
# ax.hist(resid_std, bins=25, color='black')
# ax.set_xlabel(f'{_lab[delta]}{_lab_unit1[var_sim]} (GLM)')
# fig.tight_layout()
# file = base_path_figs / "residuals" / f"{glm_key}_{location}_{var_sim}_{delta}_{land_cover_scenario}_{climate_scenario}_{soil_depth}_{future}_residuals_hist.png"
# fig.savefig(file, dpi=300)
# plt.close('all')
# plot the distribution of residuals
fig, ax = plt.subplots(figsize=(3, 3))
resid = res.resid_deviance.copy()
resid_std = stats.zscore(resid)
ax.hist(resid_std, bins=25, color='black')
ax.set_xlabel(f'{_lab[delta]}{_lab_unit1[var_sim]} (GLM)')
fig.tight_layout()
file = base_path_figs / "residuals" / "GLM" / f"{glm_key}_{location}_{var_sim}_{delta}_{land_cover_scenario}_{climate_scenario}_{soil_depth}_{future}_residuals_hist.png"
fig.savefig(file, dpi=300)
plt.close('all')

# Store data (serialize)
with open(glm_location_file, "wb") as handle:
Expand All @@ -371,11 +371,11 @@ def nanmeanweighted(y, w, axis=None):
mlm_soil_formulas = {
"interaction_theta": "~ theta_pwp * theta_ufc * theta_ac + ks",
"no_interaction": "~ theta_pwp + theta_ufc + theta_ac + ks",
"theta": "~ theta_pwp + theta_ufc + theta_ac",
"theta_pwp": "~ theta_pwp",
"theta_ufc": "~ theta_ufc",
"theta_ac": "~ theta_ac",
"ks": "~ ks",
# "theta": "~ theta_pwp + theta_ufc + theta_ac",
# "theta_pwp": "~ theta_pwp",
# "theta_ufc": "~ theta_ufc",
# "theta_ac": "~ theta_ac",
# "ks": "~ ks",
}

vars_sim = ["transp", "q_ss", "theta", "rt50_s", "tt50_q_ss", "tt50_transp"]
Expand Down Expand Up @@ -469,6 +469,38 @@ def nanmeanweighted(y, w, axis=None):
y, yhat
)

# plot the line fit
fig, ax = plt.subplots(figsize=(3, 3))
ax.scatter(yhat, y, color='black', s=4)
line_fit = sm.OLS(y, sm.add_constant(yhat, prepend=True)).fit()
abline_plot(model_results=line_fit, ax=ax, color='black')
ax.set_ylabel(f'{_lab[delta]}{_lab_unit1[var_sim]} (RoGeR) [%]')
ax.set_xlabel(f'{_lab[delta]}{_lab_unit1[var_sim]} (MLM) [%]')
fig.tight_layout()
file = base_path_figs / "residuals" / "MLM" / f"{mlm_key}_{var_sim}_{delta}_{land_cover_scenario}_{soil_depth}_line_fit_for_soil.png"
fig.savefig(file, dpi=300)

# plot the residuals
resid = yhat - y
fig, ax = plt.subplots(figsize=(3, 3))
ax.scatter(yhat, resid, color='black', s=4)
ax.set_ylabel(f'{_lab_unit1[var_sim]} (RoGeR) [%]')
ax.set_xlabel(f'{_lab[delta]}{_lab_unit1[var_sim]} (MLM)')
fig.tight_layout()
file = base_path_figs / "residuals" / "MLM" / f"{mlm_key}_{var_sim}_{delta}_{land_cover_scenario}_{soil_depth}_residuals_for_soil.png"
fig.savefig(file, dpi=300)

# plot the distribution of residuals
fig, ax = plt.subplots(figsize=(3, 3))
resid_std = stats.zscore(resid)
ax.hist(resid_std, bins=25, color='black')
ax.set_xlabel(f'{_lab[delta]}{_lab_unit1[var_sim]} (MLM)')
fig.tight_layout()
file = base_path_figs / "residuals" / "MLM" / f"{mlm_key}_{var_sim}_{delta}_{land_cover_scenario}_{soil_depth}_residuals_hist_for_soil.png"
fig.savefig(file, dpi=300)
plt.close('all')


# Store the data (serialize)
with open(mlm_soil_file, "wb") as handle:
pickle.dump(dict_mlm_soil, handle, protocol=pickle.HIGHEST_PROTOCOL)
Expand All @@ -482,10 +514,10 @@ def nanmeanweighted(y, w, axis=None):
"avg_ipr": "~ dPREC_avg + dPREC_ipr + dTA_avg + dTA_ipr",
"avg": "~ dPREC_avg + dTA_avg",
"ipr": "~ dPREC_ipr + dTA_ipr",
"dPREC_avg": "~ dPREC_avg",
"dPREC": "~ dPREC_avg + dPREC_ipr",
"dTA_avg": "~ dTA_avg",
"dTA": "~ dTA_avg + dTA_ipr",
# "dPREC_avg": "~ dPREC_avg",
# "dPREC": "~ dPREC_avg + dPREC_ipr",
# "dTA_avg": "~ dTA_avg",
# "dTA": "~ dTA_avg + dTA_ipr",
}

vars_sim = ["transp", "q_ss", "theta", "rt50_s", "tt50_q_ss", "tt50_transp"]
Expand Down Expand Up @@ -608,6 +640,36 @@ def nanmeanweighted(y, w, axis=None):
dict_mlm_climate[mlm_key][var_sim][delta][land_cover_scenario][soil_depth]["R2"] = r2_score(
y, yhat
)
# plot the line fit
fig, ax = plt.subplots(figsize=(3, 3))
ax.scatter(yhat, y, color='black', s=4)
line_fit = sm.OLS(y, sm.add_constant(yhat, prepend=True)).fit()
abline_plot(model_results=line_fit, ax=ax, color='black')
ax.set_ylabel(f'{_lab[delta]}{_lab_unit1[var_sim]} (RoGeR) [%]')
ax.set_xlabel(f'{_lab[delta]}{_lab_unit1[var_sim]} (MLM) [%]')
fig.tight_layout()
file = base_path_figs / "residuals" / "MLM" / f"{mlm_key}_{var_sim}_{delta}_{land_cover_scenario}_{soil_depth}_line_fit_for_climate.png"
fig.savefig(file, dpi=300)

# plot the residuals
resid = yhat - y
fig, ax = plt.subplots(figsize=(3, 3))
ax.scatter(yhat, resid, color='black', s=4)
ax.set_ylabel(f'{_lab_unit1[var_sim]} (RoGeR) [%]')
ax.set_xlabel(f'{_lab[delta]}{_lab_unit1[var_sim]} (MLM)')
fig.tight_layout()
file = base_path_figs / "residuals" / "MLM" / f"{mlm_key}_{var_sim}_{delta}_{land_cover_scenario}_{soil_depth}_residuals_for_climate.png"
fig.savefig(file, dpi=300)

# plot the distribution of residuals
fig, ax = plt.subplots(figsize=(3, 3))
resid_std = stats.zscore(resid)
ax.hist(resid_std, bins=25, color='black')
ax.set_xlabel(f'{_lab[delta]}{_lab_unit1[var_sim]} (MLM)')
fig.tight_layout()
file = base_path_figs / "residuals" / "MLM" / f"{mlm_key}_{var_sim}_{delta}_{land_cover_scenario}_{soil_depth}_residuals_hist_for_climate.png"
fig.savefig(file, dpi=300)
plt.close('all')

# Store the data (serialize)
with open(mlm_climate_file, "wb") as handle:
Expand All @@ -622,10 +684,10 @@ def nanmeanweighted(y, w, axis=None):
"avg_ipr": "~ dPREC_avg + dPREC_ipr + dTA_avg + dTA_ipr",
"avg": "~ dPREC_avg + dTA_avg",
"ipr": "~ dPREC_ipr + dTA_ipr",
"dPREC_avg": "~ dPREC_avg",
"dPREC": "~ dPREC_avg + dPREC_ipr",
"dTA_avg": "~ dTA_avg",
"dTA": "~ dTA_avg + dTA_ipr",
# "dPREC_avg": "~ dPREC_avg",
# "dPREC": "~ dPREC_avg + dPREC_ipr",
# "dTA_avg": "~ dTA_avg",
# "dTA": "~ dTA_avg + dTA_ipr",
}

vars_sim = ["transp", "q_ss", "theta", "rt50_s", "tt50_q_ss", "tt50_transp"]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

eval "$(conda shell.bash hook)"
conda activate roger
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_crop_monte_carlo/svat
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_crop_monte_carlo

python svat_crop.py -b numpy -d cpu --lys-experiment lys1 -td "${TMPDIR}"
# Move output from local SSD to global workspace
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

eval "$(conda shell.bash hook)"
conda activate roger
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_crop_monte_carlo/svat
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_crop_monte_carlo

python svat_crop.py -b numpy -d cpu --lys-experiment lys2_bromide -td "${TMPDIR}"
# Move output from local SSD to global workspace
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

eval "$(conda shell.bash hook)"
conda activate roger
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_crop_monte_carlo/svat
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_crop_monte_carlo

python svat_crop.py -b numpy -d cpu --lys-experiment lys2 -td "${TMPDIR}"
# Move output from local SSD to global workspace
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

eval "$(conda shell.bash hook)"
conda activate roger
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_crop_monte_carlo/svat
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_crop_monte_carlo

python svat_crop.py -b numpy -d cpu --lys-experiment lys3 -td "${TMPDIR}"
# Move output from local SSD to global workspace
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

eval "$(conda shell.bash hook)"
conda activate roger
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_crop_monte_carlo/svat
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_crop_monte_carlo

python svat_crop.py -b numpy -d cpu --lys-experiment lys4 -td "${TMPDIR}"
# Move output from local SSD to global workspace
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

eval "$(conda shell.bash hook)"
conda activate roger
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_crop_monte_carlo/svat
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_crop_monte_carlo

python svat_crop.py -b numpy -d cpu --lys-experiment lys8_bromide -td "${TMPDIR}"
# Move output from local SSD to global workspace
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

eval "$(conda shell.bash hook)"
conda activate roger
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_crop_monte_carlo/svat
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_crop_monte_carlo

python svat_crop.py -b numpy -d cpu --lys-experiment lys8 -td "${TMPDIR}"
# Move output from local SSD to global workspace
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

eval "$(conda shell.bash hook)"
conda activate roger
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_crop_monte_carlo/svat
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_crop_monte_carlo

python svat_crop.py -b numpy -d cpu --lys-experiment lys9_bromide -td "${TMPDIR}"
# Move output from local SSD to global workspace
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

eval "$(conda shell.bash hook)"
conda activate roger
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_crop_monte_carlo/svat
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_crop_monte_carlo

python svat_crop.py -b numpy -d cpu --lys-experiment lys9 -td "${TMPDIR}"
# Move output from local SSD to global workspace
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ def main():
lines.append(" \n")
lines.append('eval "$(conda shell.bash hook)"\n')
lines.append("conda activate roger\n")
lines.append(f"cd {base_path_bwuc}/svat\n")
lines.append(f"cd {base_path_bwuc}\n")
lines.append(" \n")
lines.append(
'python svat_crop.py -b numpy -d cpu --lys-experiment %s -td "${TMPDIR}"\n'
Expand All @@ -44,8 +44,6 @@ def main():
file.writelines(lines)
file.close()
subprocess.Popen(f"chmod +x {file_path}", shell=True)

subprocess.Popen("chmod +x submit_slurm_jobs.sh", shell=True)
return


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

eval "$(conda shell.bash hook)"
conda activate roger
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_monte_carlo/svat
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_monte_carlo

python svat.py -b numpy -d cpu --lys-experiment lys1 -td "${TMPDIR}"
# Move output from local SSD to global workspace
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

eval "$(conda shell.bash hook)"
conda activate roger
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_monte_carlo/svat
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_monte_carlo

python svat.py -b numpy -d cpu --lys-experiment lys2_bromide -td "${TMPDIR}"
# Move output from local SSD to global workspace
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

eval "$(conda shell.bash hook)"
conda activate roger
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_monte_carlo/svat
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_monte_carlo

python svat.py -b numpy -d cpu --lys-experiment lys2 -td "${TMPDIR}"
# Move output from local SSD to global workspace
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

eval "$(conda shell.bash hook)"
conda activate roger
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_monte_carlo/svat
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_monte_carlo

python svat.py -b numpy -d cpu --lys-experiment lys3 -td "${TMPDIR}"
# Move output from local SSD to global workspace
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

eval "$(conda shell.bash hook)"
conda activate roger
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_monte_carlo/svat
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_monte_carlo

python svat.py -b numpy -d cpu --lys-experiment lys4 -td "${TMPDIR}"
# Move output from local SSD to global workspace
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

eval "$(conda shell.bash hook)"
conda activate roger
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_monte_carlo/svat
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_monte_carlo

python svat.py -b numpy -d cpu --lys-experiment lys8_bromide -td "${TMPDIR}"
# Move output from local SSD to global workspace
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@

eval "$(conda shell.bash hook)"
conda activate roger
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_monte_carlo/svat
cd /home/fr/fr_fr/fr_rs1092/roger/examples/plot_scale/reckenholz/svat_monte_carlo

python svat.py -b numpy -d cpu --lys-experiment lys8 -td "${TMPDIR}"
# Move output from local SSD to global workspace
Expand Down
Loading

0 comments on commit 735e6f9

Please sign in to comment.