Make synthetic tests work with only one mth5

No longer need test1.h5 for test_compare_aurora_vs_archived_emtf.py
Also, using the dataset df that comes from the mth5 to define
dataset before calling config creator in make_processing_configs

[Issue(s): #165, #118, #132]

aurora/test_utils/synthetic/make_processing_configs_new.py

@@ -16,13 +16,13 @@ def get_mth5_path(test_case_id):
     mth5_path = DATA_PATH.joinpath(f"{test_case_id}.h5")
     if (test_case_id == "test1r2") or (test_case_id == "test2r1"):
         mth5_path = DATA_PATH.joinpath("test12rr.h5")
-    return mth5_path
+    return DATA_PATH.joinpath("test12rr.h5")
 
 def create_dataset_definition(test_case_id):
     mth5_path = get_mth5_path(test_case_id)
     return None
 
-def create_test_run_config(test_case_id, matlab_or_fortran=""):
+def create_test_run_config(test_case_id, dd_df, matlab_or_fortran=""):
     """
     Use config creator to generate a processing config file for the synthetic data.  
     
@@ -83,21 +83,11 @@ def create_test_run_config(test_case_id, matlab_or_fortran=""):
         num_samples_overlap = 32
         config_id = f"{local_station_id}"
 
-
-    mth5_obj = initialize_mth5(mth5_path, mode="a")
-    ch_summary = mth5_obj.channel_summary
-    dataset_definition = DatasetDefinition()
-    dataset_definition.from_mth5_channel_summary(ch_summary)
-    dd_df = dataset_definition.df
-    #channel_summary_to_dataset_definition2(ch_summary)
-    dd_df["mth5_path"] = str(mth5_path)
-    mth5_obj.close_mth5()
     cc = ConfigCreator(config_path=CONFIG_PATH)
 
     if test_case_id=="test1":
         p = cc.create_run_processing_object(emtf_band_file=emtf_band_setup_file)
         p.id = config_id
-        dd_df["remote"] = False
         p.stations.from_dataset_dataframe(dd_df)
 
         for decimation in p.decimations:
@@ -110,7 +100,6 @@ def create_test_run_config(test_case_id, matlab_or_fortran=""):
         p.drop_reference_channels()
         cc.to_json(p)
     elif test_case_id=="test2r1":
-        dd_df["remote"] = [True, False]
         p = cc.create_run_processing_object(emtf_band_file=emtf_band_setup_file)
         p.id = config_id
         p.stations.from_dataset_dataframe(dd_df)
@@ -122,7 +111,7 @@ def create_test_run_config(test_case_id, matlab_or_fortran=""):
             decimation.window.overlap = num_samples_overlap
             decimation.regression.max_redescending_iterations = 2
 
-    return p, dd_df
+    return p#, dd_df
 
 
 

tests/synthetic/test_compare_aurora_vs_archived_emtf.py

@@ -241,7 +241,8 @@ def process_synthetic_1_standard(
     return
 
 
-def aurora_vs_emtf(test_case_id, emtf_version, auxilliary_z_file, z_file_base):
+def aurora_vs_emtf(test_case_id, emtf_version, auxilliary_z_file, z_file_base,
+                   dddf=None):
     """
 
     Parameters
@@ -263,8 +264,15 @@ def aurora_vs_emtf(test_case_id, emtf_version, auxilliary_z_file, z_file_base):
     -------
 
     """
-    processing_config, dd_df = create_test_run_config(test_case_id,
-                                               matlab_or_fortran=emtf_version)
+    if test_case_id=="test1":
+        dd_df = dddf[dddf.station_id=="test1"]
+    elif dddf is not None:
+        dd_df = dddf
+
+    processing_config = create_test_run_config(test_case_id, dd_df,
+                                                      matlab_or_fortran=emtf_version)
+
+
     expected_rms_misfit = get_expected_rms_misfit(test_case_id, emtf_version)
 
     process_synthetic_1_standard(
@@ -281,21 +289,22 @@ def aurora_vs_emtf(test_case_id, emtf_version, auxilliary_z_file, z_file_base):
     return
 
 
-def run_test1(emtf_version):
+def run_test1(emtf_version, dddf=None):
     print(f"Test1 vs {emtf_version}")
     test_case_id = "test1"
     auxilliary_z_file = EMTF_OUTPUT_PATH.joinpath("test1.zss")
     z_file_base = f"{test_case_id}_aurora_{emtf_version}.zss"
-    aurora_vs_emtf(test_case_id, emtf_version, auxilliary_z_file, z_file_base)
+    aurora_vs_emtf(test_case_id, emtf_version, auxilliary_z_file, z_file_base,
+                   dddf=dddf)
     return
 
-def run_test2r1():
+def run_test2r1(dddf=None):
     print(f"Test2r1")
     test_case_id = "test2r1"
     emtf_version = "fortran"
     auxilliary_z_file = EMTF_OUTPUT_PATH.joinpath("test2r1.zrr")
     z_file_base = f"{test_case_id}_aurora_{emtf_version}.zrr"
-    aurora_vs_emtf(test_case_id, emtf_version, auxilliary_z_file, z_file_base)
+    aurora_vs_emtf(test_case_id, emtf_version, auxilliary_z_file, z_file_base, dddf=dddf)
     return
 
 def make_mth5s():
@@ -306,10 +315,17 @@ def make_mth5s():
 
 def test():
     mth5_paths = make_mth5s()
-    super_summary = extract_run_summaries_from_mth5s(mth5_paths)
-    run_test1("fortran")
-    run_test1("matlab")
-    run_test2r1()
+
+    super_summary = extract_run_summaries_from_mth5s([mth5_paths[1],])
+
+    dataset_df = super_summary[super_summary.station_id=="test1"]
+    dataset_df["remote"] = False
+    run_test1("fortran", dddf=dataset_df)
+    run_test1("matlab", dddf=dataset_df)
+
+    dataset_df = super_summary.copy(deep=True)
+    dataset_df["remote"] = [True, False]
+    run_test2r1(dddf=dataset_df)
     print("success")