Fix issue 105

.flake8

@@ -1,5 +1,5 @@
 [flake8]
-ignore = E203, E266, E501, W503, F403, F401
+ignore = E203, E266, E501, W503, F403, F401, W605
 max-line-length = 79
 max-complexity = 18
-select = B,C,E,F,W,T4,B9
+select = B,C,E,F,W,T4,B9

.pre-commit-config.yaml

@@ -1,10 +1,10 @@
 repos:
 -   repo: https://github.com/ambv/black
-    rev: stable
+    rev: 22.6.0
     hooks:
     - id: black
       language_version: python3.6
 -   repo: https://gitlab.com/pycqa/flake8
-    rev: 3.7.9
+    rev: 3.9.2
     hooks:
     - id: flake8

mth5/io/lemi424.py

@@ -2,7 +2,7 @@
 """
 Created on Tue May 11 15:31:31 2021
 
-:copyright: 
+:copyright:
     Jared Peacock (jpeacock@usgs.gov)
 
 :license: MIT
@@ -25,7 +25,7 @@ class LEMI424:
 
     """
 
-    def __init__(self, fn=None):
+    def __init__(self, fn=[]):
         self.logger = logging.getLogger(f"{__name__}.{self.__class__.__name__}")
         self.fn = fn
         self._has_data = False
@@ -61,16 +61,44 @@ def __init__(self, fn=None):
         if self.fn:
             self.read()
 
+    @property
+    def num_source_files(self):
+        return len(self.fn)
+
     @property
     def fn(self):
         return self._fn
 
+    @property
+    def validate_fn(self):
+        """
+        Need to check that the filenames are sequential
+        """
+        return True
+
     @fn.setter
-    def fn(self, value):
-        if value is not None:
-            value = Path(value)
-            if not value.exists():
-                raise IOError(f"Could not find {value}")
+    def fn(self, value, sort=True):
+        """
+
+        Parameters
+        ----------
+        value:string or pathlib.Path, or list of these
+
+        """
+        if isinstance(value, list):
+            value = [Path(x) for x in value]
+            exists = [x.exists() for x in value]
+            for i_fn, cond in enumerate(exists):
+                if not cond:
+                    raise IOError(f"Could not find {value[i_fn]}")
+        elif value is not None:
+            value = [
+                Path(value),
+            ]
+            if not value[0].exists():
+                raise IOError(f"Could not find {value[0]}")
+        if sort:
+            value.sort()
         self._fn = value
 
     @property
@@ -80,16 +108,16 @@ def start(self):
                 [
                     "-".join(
                         [
-                            f"{self._df.year.min()}",
-                            f"{self._df.month.min():02d}",
-                            f"{self._df.day.min():02d}",
+                            f"{self._df.iloc[0].year}",
+                            f"{self._df.iloc[0].month:02d}",
+                            f"{self._df.iloc[0].day:02d}",
                         ]
                     ),
                     ":".join(
                         [
-                            f"{self._df.hour.min():02d}",
-                            f"{self._df.minute.min():02d}",
-                            f"{self._df.second.min():02d}",
+                            f"{self._df.iloc[0].hour:02d}",
+                            f"{self._df.iloc[0].minute:02d}",
+                            f"{self._df.iloc[0].second:02d}",
                         ]
                     ),
                 ]
@@ -102,16 +130,16 @@ def end(self):
                 [
                     "-".join(
                         [
-                            f"{self._df.year.max()}",
-                            f"{self._df.month.max():02d}",
-                            f"{self._df.day.max():02d}",
+                            f"{self._df.iloc[-1].year}",
+                            f"{self._df.iloc[-1].month:02d}",
+                            f"{self._df.iloc[-1].day:02d}",
                         ]
                     ),
                     ":".join(
                         [
-                            f"{self._df.hour.max():02d}",
-                            f"{self._df.minute.max():02d}",
-                            f"{self._df.second.max():02d}",
+                            f"{self._df.iloc[-1].hour:02d}",
+                            f"{self._df.iloc[-1].minute:02d}",
+                            f"{self._df.iloc[-1].second:02d}",
                         ]
                     ),
                 ]
@@ -160,7 +188,7 @@ def run_metadata(self):
             r.time_period.start = self.start
             r.time_period.end = self.end
 
-    def read(self, fn=None):
+    def read(self, fn=[]):
         """
         Read a LEMI424 file using pandas
 
@@ -170,16 +198,24 @@ def read(self, fn=None):
         :rtype: TYPE
 
         """
-        if fn is not None:
+        if fn:
             self.fn = fn
 
-        if not self.fn.exists():
+        exists = [x.exists() for x in self.fn]
+        if all(x for x in exists):
+            pass
+        else:
             msg = "Could not find file %s"
-            self.logger.error(msg, self.fn)
-            raise IOError(msg % self.fn)
+            for i_fn, cond in enumerate(exists):
+                if not cond:
+                    self.logger.error(msg, self.fn[i_fn])
+                    raise IOError(msg % self.fn[i_fn])
 
-        self._df = pd.read_csv(self.fn, delimiter="\s+", names=self.column_names)
+        dfs = self.num_source_files * [None]
+        for i, fn in enumerate(self.fn):
+            dfs[i] = pd.read_csv(fn, delimiter="\s+", names=self.column_names)
 
+        self._df = pd.concat(dfs)
         self._has_data = True
 
     def to_run_ts(self, fn=None, e_channels=["e1", "e2"]):

mth5/utils/helpers.py

@@ -42,7 +42,7 @@ def initialize_mth5(h5_path, mode="a", file_version="0.1.0"):
     return mth5_obj
 
 
-def read_back_data(mth5_path, station_id, run_id):
+def read_back_data(mth5_path, station_id, run_id, survey=None):
     """
 
     Parameters
@@ -63,7 +63,8 @@ def read_back_data(mth5_path, station_id, run_id):
     processing_config["local_station_id"] = station_id
     config = processing_config
     m = initialize_mth5(config["mth5_path"], mode="r")
-    local_run_obj = m.get_run(config["local_station_id"], run_id)
+    local_run_obj = m.get_run(config["local_station_id"], run_id, survey=survey)
     local_run_ts = local_run_obj.to_runts()
-    print("success")
+    data_array = local_run_ts.dataset.to_array()
+    print(f"data shape = {data_array.shape}")
     return local_run_obj, local_run_ts

tests/version_1/test_make_mth5.py

@@ -68,17 +68,17 @@ def test_df_input_inventory(self):
         with self.subTest(name="stations"):
             self.assertListEqual(
                 sorted(self.stations),
-                sorted([ss.code for ss in inv.networks[0].stations]),
+                sorted(list(set([ss.code for ss in inv.networks[0].stations]))),
             )
         with self.subTest(name="channels_CAS04"):
             self.assertListEqual(
                 sorted(self.channels),
-                sorted([ss.code for ss in inv.networks[0].stations[0].channels]),
+                sorted(list(set([ss.code for ss in inv.networks[0].stations[0].channels]))),
             )
         with self.subTest(name="channels_NVR08"):
             self.assertListEqual(
                 sorted(self.channels),
-                sorted([ss.code for ss in inv.networks[0].stations[1].channels]),
+                sorted(list(set([ss.code for ss in inv.networks[0].stations[1].channels]))),
             )
 
     def test_csv_input_inventory(self):
@@ -91,12 +91,12 @@ def test_csv_input_inventory(self):
         with self.subTest(name="channels_CAS04"):
             self.assertListEqual(
                 sorted(self.channels),
-                sorted([ss.code for ss in inv.networks[0].stations[0].channels]),
+                sorted(list(set([ss.code for ss in inv.networks[0].stations[0].channels]))),
             )
         with self.subTest(name="channels_NVR08"):
             self.assertListEqual(
                 sorted(self.channels),
-                sorted([ss.code for ss in inv.networks[0].stations[1].channels]),
+                sorted(list(set([ss.code for ss in inv.networks[0].stations[1].channels]))),
             )
 
     def test_fail_csv_inventory(self):

tests/version_2/test_make_mth5.py

@@ -70,12 +70,12 @@ def test_df_input_inventory(self):
         with self.subTest(name="channels_CAS04"):
             self.assertListEqual(
                 sorted(self.channels),
-                sorted([ss.code for ss in inv.networks[0].stations[0].channels]),
+                sorted(list(set([ss.code for ss in inv.networks[0].stations[0].channels]))),
             )
         with self.subTest(name="channels_NVR08"):
             self.assertListEqual(
                 sorted(self.channels),
-                sorted([ss.code for ss in inv.networks[0].stations[1].channels]),
+                sorted(list(set([ss.code for ss in inv.networks[0].stations[1].channels]))),
             )
 
     def test_csv_input_inventory(self):
@@ -85,15 +85,16 @@ def test_csv_input_inventory(self):
                 sorted(self.stations),
                 sorted([ss.code for ss in inv.networks[0].stations]),
             )
+
         with self.subTest(name="channels_CAS04"):
             self.assertListEqual(
                 sorted(self.channels),
-                sorted([ss.code for ss in inv.networks[0].stations[0].channels]),
+                sorted(list(set([ss.code for ss in inv.networks[0].stations[0].channels]))),
             )
         with self.subTest(name="channels_NVR08"):
             self.assertListEqual(
                 sorted(self.channels),
-                sorted([ss.code for ss in inv.networks[0].stations[1].channels]),
+                sorted(list(set([ss.code for ss in inv.networks[0].stations[1].channels]))),
             )
 
     def test_fail_csv_inventory(self):