initial framework

hydroevaluate/dataloader/data.py

@@ -14,7 +14,7 @@ def _load_data(self):
         norm_x, norm_c = self._normalize()
         self.x, self.c = self._kill_nan(norm_x, norm_c)
 
-    def _read_xc():
+    def _read_xc(self):
         pass
 
     def _kill_nan():

hydroevaluate/dataloader/gfs.py

@@ -1,3 +1,8 @@
+import pandas as pd
+
+from hydroevaluate.utils.heutils import convert_baseDatetime_iso
+
+
 def process_gfsTp(time, stcd, tolerance=0.05):
     # 处理 gfsTp
     gfstp_df = read_forcing_dataframe("gfs_tp", stcd, time)

hydroevaluate/dataloader/gpm.py

@@ -1,3 +1,6 @@
+import pandas as pd
+
+
 def process_gpmData(time, stcd, tolerance=0.05):
     # 处理 gpmData
     gpm_df = read_forcing_dataframe("gpm_tp", stcd, time)

hydroevaluate/dataloader/smap.py

@@ -1,58 +1,4 @@
-def process_gfsSoilData(time, stcd):
-    # 处理 gfsSoilData
-    gfsSoil_df = read_forcing_dataframe("gfs_soil", stcd, time)
-
-    gfsSoil_df["forecastdatetime"] = pd.to_datetime(gfsSoil_df["forecastdatetime"])
-
-    # 计算 intersection_area 和 soilw 的乘积
-    gfsSoil_df["intersection_area_soilw"] = (
-        gfsSoil_df["intersection_area"] * gfsSoil_df["soilw"]
-    )
-
-    # 计算每个 forecastdatetime 下的 intersection_area 和 intersection_area_soilw 的和
-    grouped = (
-        gfsSoil_df.groupby("forecastdatetime")
-        .agg({"intersection_area_soilw": "sum", "intersection_area": "sum"})
-        .reset_index()
-    )
-
-    # 计算新的 soilw_cal_from_origin 列
-    grouped["gfs_soil"] = (
-        grouped["intersection_area_soilw"] / grouped["intersection_area"]
-    )
-
-    # 确保时间序列为一小时间隔，插值缺失数据
-    full_time_range = pd.date_range(
-        start=grouped["forecastdatetime"].min(),
-        end=grouped["forecastdatetime"].max(),
-        freq="H",
-    )
-    grouped = (
-        grouped.set_index("forecastdatetime")
-        .reindex(full_time_range)
-        .interpolate(method="time")
-        .reset_index()
-    )
-    grouped = grouped.rename(columns={"index": "forecastdatetime"})
-
-    # 用第一个非空值填充开始的缺失值，用最后一个非空值填充结束的缺失值
-    grouped.fillna(method="ffill", inplace=True)
-    grouped.fillna(method="bfill", inplace=True)
-
-    # 修改列名
-    grouped = grouped.rename(columns={"forecastdatetime": "time"})
-
-    # 添加 basin 列
-    grouped["basin"] = stcd
-
-    # 转换为 DataArray
-    result_dataarray = to_dataarray(
-        grouped, dims=["time"], coords={"time": grouped["time"]}, name="gfs_soil"
-    )
-    result_dataarray = result_dataarray.rename("sm_surface")
-    result_dataarray = result_dataarray.expand_dims("basin").assign_coords(basin=[stcd])
-
-    return result_dataarray
+import pandas as pd
 
 
 def process_smapData_wrong(data, stcd, tolerance=0.05):

hydroevaluate/evaluator/eval.py

@@ -0,0 +1,10 @@
+import warnings
+import yaml
+
+warnings.filterwarnings("ignore")
+
+cfg_path_dir = "scripts/conf/"
+
+
+def sim_eval(cfgs):
+    print("All processes are finished!")

hydroevaluate/hydroevaluate.py

@@ -1,7 +1,7 @@
 """
 Author: Wenyu Ouyang
 Date: 2024-05-30 09:11:04
-LastEditTime: 2024-06-02 16:03:02
+LastEditTime: 2024-06-02 16:24:18
 LastEditors: Wenyu Ouyang
 Description: main function for hydroevaluate
 FilePath: \hydroevaluate\hydroevaluate\hydroevaluate.py
@@ -113,102 +113,30 @@ def evaluate(self, obs_xr):
         print(test_log)
         return eval_log, pred_xr, obs_xr
 
-
-def auto_stream():
-    test_config_path = os.path.join(work_dir, "scripts/conf/v002.yml")
-    with open(test_config_path, "r+") as fp:
-        test_conf_yml = yaml.load(fp, Loader)
-    # 配置文件中的weight_dir应与模型保存位置相对应，目前模型路径是直接指定，而非选择最新
-    # test_model_name = test_read_history(user_model_type='model', version='300')
-    eval_log, preds_xr, obss_xr = load_torchmodel(test_config_path)
-    preds_xr_sf_np = preds_xr["streamflow"].to_numpy().T
-    obss_xr_sf_np = obss_xr["streamflow"].to_numpy().T
-    eval_log["Metrics"] = {}
-    eval_log["Config"] = {}
-    eval_log["Basin"] = obss_xr["basin"].to_numpy().tolist()
-    eval_log["Metrics"]["NSE"] = eval_log["NSE of streamflow"].tolist()
-    eval_log.pop("NSE of streamflow")
-    eval_log["Metrics"]["MAE"] = eval_log["Bias of streamflow"].tolist()
-    eval_log.pop("Bias of streamflow")
-    eval_log["Metrics"]["KGE"] = eval_log["KGE of streamflow"].tolist()
-    eval_log.pop("KGE of streamflow")
-    eval_log["Metrics"]["RMSE"] = eval_log["RMSE of streamflow"].tolist()
-    eval_log.pop("RMSE of streamflow")
-    eval_log["Metrics"]["Bias of peak height(mm/h)"] = {}
-    eval_log["Metrics"]["Bias of peak appearance(h)"] = {}
-    eval_log["Reports"] = {
-        "Total streamflow(mm/h)": {},
-        "Peak rainfall(mm)": {},
-        "Peak streamflow(mm/h)": {},
-        "Streamflow peak appearance": {},
-    }
-    for i in range(preds_xr_sf_np.shape[0]):
-        basin = obss_xr["basin"].to_numpy()[i]
-        pred_peaks_index = signal.argrelmax(preds_xr_sf_np[i])
-        pred_peaks_time = (preds_xr["time_now"].to_numpy())[pred_peaks_index]
-        obs_peaks_index = signal.argrelmax(obss_xr_sf_np[i])
-        obss_peaks_time = (obss_xr["time_now"].to_numpy())[obs_peaks_index]
-        eval_log["Metrics"]["Bias of peak height(mm/h)"][basin] = np.mean(
-            [
-                abs(obss_xr_sf_np[i] - preds_xr_sf_np[i])
-                for i in range(len(obs_peaks_index))
-            ]
-        ).tolist()
-
-        eval_log["Metrics"]["Bias of peak appearance(h)"][basin] = (
-            np.mean(
+    def send_report(self, eval_log):
+        private_yml = self.cfg
+        # https://zhuanlan.zhihu.com/p/631317974
+        send_address = private_yml["email"]["send_address"]
+        password = private_yml["email"]["authenticate_code"]
+        server = smtplib.SMTP_SSL("smtp.qq.com", 465)
+        login_result = server.login(send_address, password)
+        if login_result == (235, b"Authentication successful"):
+            content = yaml.dump(data=eval_log, Dumper=Dumper)
+            # https://service.mail.qq.com/detail/124/995
+            # https://stackoverflow.com/questions/58223773/send-a-list-of-dictionaries-formatted-with-indents-as-a-string-through-email-u
+            msg = MIMEMultipart()
+            msg["From"] = "nickname<" + send_address + ">"
+            msg["To"] = str(
                 [
-                    abs(obss_peaks_time[i] - pred_peaks_time[i])
-                    for i in range(len(obss_peaks_time))
+                    "nickname<" + addr + ">;"
+                    for addr in private_yml["email"]["to_address"]
                 ]
-            ).tolist()
-            / 3.6e12
-        )
-        # 在这里是所有预测值在[0，forecast_length]内的总洪量
-        eval_log["Reports"]["Total streamflow(mm/h)"][basin] = np.sum(
-            preds_xr_sf_np[i][
-                0 : test_conf_yml["model_cfgs"]["model_hyperparam"]["forecast_length"]
-            ]
-        ).tolist()
-        # rainfall对于这个模型是输入先验值，地位“微妙”，找不到合适地点插入, 暂且留空
-        eval_log["Reports"]["Peak rainfall(mm)"][basin] = 200
-        eval_log["Reports"]["Peak streamflow(mm/h)"][basin] = np.max(
-            preds_xr_sf_np[i][
-                0 : test_conf_yml["model_cfgs"]["model_hyperparam"]["forecast_length"]
-            ]
-        ).tolist()
-        eval_log["Reports"]["Streamflow peak appearance"][basin] = (
-            np.datetime_as_string(pred_peaks_time, unit="s").tolist()
-        )
-    eval_log["Config"]["model_name"] = test_conf_yml["model_cfgs"]["model_name"]
-    eval_log["Config"]["model_hyperparam"] = test_conf_yml["model_cfgs"][
-        "model_hyperparam"
-    ]
-    eval_log["Config"]["weight_path"] = test_conf_yml["model_cfgs"]["weight_dir"]
-    eval_log["Config"]["t_range_train"] = test_conf_yml["train_period"]
-    eval_log["Config"]["t_range_test"] = test_conf_yml["test_period"]
-    eval_log["Config"]["dataset"] = test_conf_yml["data_cfgs"]["dataset"]
-    eval_log["Config"]["sampler"] = test_conf_yml["data_cfgs"]["sampler"]
-    eval_log["Config"]["scaler"] = test_conf_yml["data_cfgs"]["scaler"]
-    # https://zhuanlan.zhihu.com/p/631317974
-    send_address = private_yml["email"]["send_address"]
-    password = private_yml["email"]["authenticate_code"]
-    server = smtplib.SMTP_SSL("smtp.qq.com", 465)
-    login_result = server.login(send_address, password)
-    if login_result == (235, b"Authentication successful"):
-        content = yaml.dump(data=eval_log, Dumper=Dumper)
-        # https://service.mail.qq.com/detail/124/995
-        # https://stackoverflow.com/questions/58223773/send-a-list-of-dictionaries-formatted-with-indents-as-a-string-through-email-u
-        msg = MIMEMultipart()
-        msg["From"] = "nickname<" + send_address + ">"
-        msg["To"] = str(
-            ["nickname<" + addr + ">;" for addr in private_yml["email"]["to_address"]]
-        )
-        msg["Subject"] = "model_report"
-        msg.attach(MIMEText(content, "plain"))
-        server.sendmail(
-            send_address, private_yml["email"]["to_address"], msg.as_string()
-        )
-        print("发送成功")
-    else:
-        print("发送失败")
+            )
+            msg["Subject"] = "model_report"
+            msg.attach(MIMEText(content, "plain"))
+            server.sendmail(
+                send_address, private_yml["email"]["to_address"], msg.as_string()
+            )
+            print("发送成功")
+        else:
+            print("发送失败")