Merge pull request #301 from simpeg/dev

Dev
- patches for Canbera HPC gadi examples (including workarounds for antique MTH5 files)

.github/workflows/tests.yml

@@ -77,7 +77,7 @@ jobs:
     - name: "Upload coverage to Codecov"
       uses: codecov/codecov-action@v1
       with:
-        fail_ci_if_error: true
+        fail_ci_if_error: false
 
     - name: Build Doc
       if: ${{ (github.ref == 'refs/heads/main') && (matrix.python-version == '3.8')}}

aurora/config/__init__.py

@@ -1,5 +1,6 @@
 from aurora.config.emtf_band_setup import (
     BANDS_DEFAULT_FILE,
+    BANDS_TEST_FAST_FILE,
     BANDS_256_26_FILE,
     BANDS_256_29_FILE,
 )

aurora/config/emtf_band_setup/__init__.py

@@ -1,6 +1,6 @@
 from aurora.general_helper_functions import BAND_SETUP_PATH
-import pathlib
 
 BANDS_DEFAULT_FILE = BAND_SETUP_PATH.joinpath("bs_test.cfg")
+BANDS_TEST_FAST_FILE = BAND_SETUP_PATH.joinpath("bs_test_fast.cfg")
 BANDS_256_26_FILE = BAND_SETUP_PATH.joinpath("bs_256_26.cfg")
 BANDS_256_29_FILE = BAND_SETUP_PATH.joinpath("bs_256_29.cfg")

aurora/config/emtf_band_setup/bs_test_fast.cfg

@@ -0,0 +1,9 @@
+8
+1 6 7
+1 5 5
+2 6 6
+2 5 5
+3 6 6
+3 5 5
+4 7 9
+4 5 6

aurora/pipelines/process_mth5.py

@@ -299,13 +299,22 @@ def process_mth5(
     tfk.make_processing_summary()
     tfk.validate()
     # See Note #1
-    tfk.initialize_mth5s(mode="a")
-    tfk.check_if_fc_levels_already_exist()  # populate the "fc" column of dataset_df
-    print(f"fc_levels_already_exist = {tfk.dataset_df['fc']}")
-    print(
-        f"Processing config indicates {len(tfk.config.decimations)} "
-        f"decimation levels "
-    )
+    if config.decimations[0].save_fcs:
+        mth5_mode = "a"
+    else:
+        mth5_mode = "r"
+    tfk.initialize_mth5s(mode=mth5_mode)
+    try:
+        tfk.check_if_fc_levels_already_exist()  # populate the "fc" column of dataset_df
+        print(f"fc_levels_already_exist = {tfk.dataset_df['fc']}")
+        print(
+            f"Processing config indicates {len(tfk.config.decimations)} "
+            f"decimation levels "
+        )
+    except:
+        msg = "WARNING -- Unable to execute check for FC Levels"
+        msg = f"{msg} Possibly FCs not present at all (file from old MTH5 version)?"
+        print(f"{msg}")
 
     tf_dict = {}
 
@@ -333,6 +342,15 @@ def process_mth5(
 
             run_xrds = row["run_dataarray"].to_dataset("channel")
             run_obj = row.mth5_obj.from_reference(row.run_reference)
+
+            # Musgraves workaround for old MT data
+            try:
+                assert row.run_id == run_obj.metadata.id
+            except AssertionError:
+                print("WARNING Run ID in dataset_df does not match run_obj")
+                print("WARNING Forcing run metadata to match dataset_df")
+                run_obj.metadata.id = row.run_id
+
             stft_obj = make_stft_objects(
                 tfk.config, i_dec_level, run_obj, run_xrds, units, row.station_id
             )

aurora/pipelines/run_summary.py

@@ -23,19 +23,14 @@
 import copy
 import pandas as pd
 
+
+from mt_metadata.transfer_functions.processing.aurora.channel_nomenclature import ALLOWED_INPUT_CHANNELS
+from mt_metadata.transfer_functions.processing.aurora.channel_nomenclature import ALLOWED_OUTPUT_CHANNELS
 import mth5
 from mth5.utils.helpers import initialize_mth5
 
 
-INPUT_CHANNELS = [
-    "hx",
-    "hy",
-]
-OUTPUT_CHANNELS = [
-    "ex",
-    "ey",
-    "hz",
-]
+
 RUN_SUMMARY_COLUMNS = [
     "survey",
     "station_id",
@@ -149,8 +144,8 @@ def drop_invalid_rows(self):
 
 def channel_summary_to_run_summary(
     ch_summary,
-    allowed_input_channels=INPUT_CHANNELS,
-    allowed_output_channels=OUTPUT_CHANNELS,
+    allowed_input_channels=ALLOWED_INPUT_CHANNELS,
+    allowed_output_channels=ALLOWED_OUTPUT_CHANNELS,
     sortby=["station_id", "start"],
 ):
     """

aurora/pipelines/transfer_function_kernel.py

@@ -483,7 +483,7 @@ def export_tf_collection(self, tf_collection):
 
         def make_decimation_dict_for_tf(tf_collection, processing_config):
             """
-            Decimation dict is used by mt_metadata's TF class when it is writng z-files.
+            Decimation dict is used by mt_metadata's TF class when it is writing z-files.
             If no z-files will be written this is not needed
 
             sample element of decimation_dict:

aurora/test_utils/synthetic/make_mth5_from_asc.py

@@ -38,13 +38,18 @@ def create_run_ts_from_synthetic_run(run, df, channel_nomenclature="default"):
     Loop over stations and make ChannelTS objects.
     Need to add a tag in the channels
     so that when you call a run it will get all the filters with it.
+
     Parameters
     ----------
-    config: dict
-        one-off data structure used to hold information mth5 needs to initialize
+    run: aurora.test_utils.synthetic.station_config.SyntheticRun
+        One-off data structure with information mth5 needs to initialize
         Specifically sample_rate, filters,
     df : pandas.DataFrame
         time series data in columns labelled from ["ex", "ey", "hx", "hy", "hz"]
+    channel_nomenclature : string
+        Keyword corresponding to channel nomenclature mapping in CHANNEL_MAPS variable
+        from channel_nomenclature.py module in mt_metadata.
+        Supported values are ['default', 'lemi12', 'lemi34', 'phoenix123']
 
     Returns
     -------
@@ -109,19 +114,20 @@ def create_mth5_synthetic_file(
         Where the mth5 will be stored.  This is generated by the station_config,
         but may change in this method based on add_nan_values or channel_nomenclature
     plot: bool
-        If true plots get made
+        Set to false unless you want to look at a plot of the time series
     add_nan_values: bool
+        If true, some np.nan are sprinkled into the time series.  Intended to be used for tests.
     file_version: string
-    channel_nomenclature: string
+        One of ["0.1.0", "0.2.0"], corresponding to the version of mth5 to create
+    channel_nomenclature : string
+        Keyword corresponding to channel nomenclature mapping in CHANNEL_MAPS variable
+        from channel_nomenclature.py module in mt_metadata.
+        Supported values are ['default', 'lemi12', 'lemi34', 'phoenix123']
     force_make_mth5: bool
+        If set to true, the file will be made, even if it already exists.
+        If false, and file already exists, skip the make job.
 
 
-    Returns
-    -------
-
-    plot : bool
-        set to false unless you want to look at a plot of the time series
-
     Returns
     -------
     mth5_path: pathlib.Path
@@ -192,7 +198,7 @@ def create_mth5_synthetic_file(
             raise NotImplementedError
 
     m.close_mth5()
-    # Following lines used to visually confirm start/end times were
+    # Following lines used to visually confirm start/end times were packed
     # m.open_mth5(mth5_path, mode="a")
     # channel_summary_df = m.channel_summary.to_dataframe()
     # print(channel_summary_df[["start", "end"]])

aurora/transfer_function/kernel_dataset.py

@@ -188,7 +188,13 @@ def local_survey_id(self):
 
     @property
     def local_survey_metadata(self):
-        return self.survey_metadata[self.local_survey_id]
+        try:
+            return self.survey_metadata[self.local_survey_id]
+        except KeyError:
+            msg = f"Unexpected key {self.local_survey_id} not found in survey_metadata"
+            msg += f"{msg} WARNING -- Maybe old MTH5 -- trying to use key '0'"
+            print(msg)
+            return self.survey_metadata["0"]
 
     def _add_duration_column(self):
         """ """

aurora/transfer_function/plot/comparison_plots.py

@@ -52,43 +52,47 @@ def compare_two_z_files(
     print(f"scale_factor1: {scale_factor1}")
     fig, axs = plt.subplots(nrows=2, dpi=300, sharex=True)  # figsize=(8, 6.),
     markersize = kwargs.get("markersize", 3)
+    # Make LaTeX symbol strings
+    rho_phi_strings = {}
+    rho_phi_strings["rho"] = {}
+    rho_phi_strings["phi"] = {}
+    for xy_or_yx in ["xy", "yx"]:
+        rho_phi_strings["rho"][xy_or_yx] = f"$\\rho_{{{xy_or_yx}}}$"
+        rho_phi_strings["phi"][xy_or_yx] = f"$\phi_{{{xy_or_yx}}}$"
+
+    markers = {}
+    markers["xy"] = "^"
+    markers["yx"] = "o"
+    file1_colors = {}
+    file2_colors = {}
+    file1_colors["xy"] = "black"
+    file1_colors["yx"] = "black"
+    file2_colors["xy"] = "red"
+    file2_colors["yx"] = "blue"
+
+    rho_or_phi = "rho"
+    for xy_or_yx in ["xy", "yx"]:
+        plot_rho(
+            axs[0],
+            zfile1.periods,
+            zfile1.rho(xy_or_yx) * scale_factor1,
+            label=f"{label1} {rho_phi_strings[rho_or_phi][xy_or_yx]}",
+            markersize=markersize,
+            marker=markers[xy_or_yx],
+            color=file1_colors[xy_or_yx],
+        )
+        plot_rho(
+            axs[0],
+            zfile2.periods,
+            zfile2.rho(xy_or_yx) * scale_factor2,
+            label=f"{label2} {rho_phi_strings[rho_or_phi][xy_or_yx]}",
+            markersize=markersize,
+            marker=markers[xy_or_yx],
+            color=file2_colors[xy_or_yx],
+        )
 
-    plot_rho(
-        axs[0],
-        zfile1.periods,
-        zfile1.rxy * scale_factor1,
-        label=f"{label1} rxy",
-        markersize=markersize,
-        marker="^",
-        color="red",
-    )
-    plot_rho(
-        axs[0],
-        zfile2.periods,
-        zfile2.rxy * scale_factor2,
-        label=f"{label2} rxy",
-        markersize=markersize,
-        marker="^",
-        color="black",
-    )
-    plot_rho(
-        axs[0],
-        zfile1.periods,
-        zfile1.ryx * scale_factor1,
-        label=f"{label1} ryx",
-        markersize=markersize,
-        color="blue",
-    )
-    plot_rho(
-        axs[0],
-        zfile2.periods,
-        zfile2.ryx,
-        label=f"{label2} ryx",
-        markersize=markersize,
-        color="black",
-    )
     axs[0].legend(prop={"size": 6})
-    # axs[0].set_ylabel("$\rho_a$")
+    # axs[0].set_ylabel("$\\rho_a$")
     axs[0].set_ylabel("Apparent Resistivity $\Omega$-m")
     rho_ylims = kwargs.get("rho_ylims", [1, 1e3])
     if use_ylims:
@@ -97,44 +101,35 @@ def compare_two_z_files(
     if use_xlims:
         axs[0].set_xlim(xlims[0], xlims[1])
 
-    plot_phi(
-        axs[1],
-        zfile1.periods,
-        zfile1.pxy,
-        label=f"{label1} pxy",
-        markersize=markersize,
-        marker="^",
-        color="red",
-    )
-    plot_phi(
-        axs[1],
-        zfile2.periods,
-        zfile2.pxy,
-        label=f"{label2} pxy",
-        markersize=markersize,
-        marker="^",
-        color="black",
-    )
-    plot_phi(
-        axs[1],
-        zfile1.periods,
-        zfile1.pyx,
-        label=f"{label1} pyx",
-        markersize=markersize,
-        color="blue",
-    )
-    plot_phi(
-        axs[1],
-        zfile2.periods,
-        zfile2.pyx,
-        label=f"{label1} pyx",
-        markersize=markersize,
-        color="black",
-    )
+    rho_or_phi = "phi"
+    for xy_or_yx in ["xy", "yx"]:
+        plot_phi(
+            axs[1],
+            zfile1.periods,
+            zfile1.phi(xy_or_yx) * scale_factor1,
+            label=f"{label1} {rho_phi_strings[rho_or_phi][xy_or_yx]}",
+            markersize=markersize,
+            marker=markers[xy_or_yx],
+            color=file1_colors[xy_or_yx],
+        )
+        plot_phi(
+            axs[1],
+            zfile2.periods,
+            zfile2.phi(xy_or_yx) * scale_factor2,
+            label=f"{label2} {rho_phi_strings[rho_or_phi][xy_or_yx]}",
+            markersize=markersize,
+            marker=markers[xy_or_yx],
+            color=file2_colors[xy_or_yx],
+        )
+
+    axs[1].legend(prop={"size": 6})
     axs[1].set_xlabel("Period (s)")
     axs[1].set_ylabel("Phase (degrees)")
     phi_ylims = kwargs.get("phi_ylims", [0, 90])
     axs[1].set_ylim(phi_ylims[0], phi_ylims[1])
+
+    axs[0].grid(which = 'both', axis = 'both',)
+    axs[1].grid(which='both', axis='both', )
     if out_file:
         # if out_file[-3:] != ".png":
         #     out_file+=".png"