update the file integrity checker (#148)

* update the file integrity checker to load metadata from the raw file and some other refactoring

* fixed code style

oceanstream/L0_unprocessed_data/raw_handler.py

@@ -24,10 +24,12 @@
 # Import necessary libraries
 import os
 import re
+import xml.etree.ElementTree as ET
 from datetime import datetime, timedelta
 from typing import Dict, List, Union
 
 import echopype as ep
+from echopype.convert.utils.ek_raw_io import RawSimradFile
 
 SUPPORTED_SONAR_MODELS = ["EK60", "ES70", "EK80", "EA640", "AZFP", "AD2CP"]
 TIME_BETWEEN_FILES = 30  # time in minutes between two consecutive files
@@ -164,86 +166,81 @@ def file_integrity_checking(
     Returns:
 
     - dict: A dictionary containing the following keys:
-    'file_path': Absolute path to the file.
-    'campaign_id': Identifier for the measuring\
-    campaign extracted from the file name.
-    'date': Date and time when the measurement started,\
-     extracted from the file name.
-    'sonar_model': Type of sonar that produced the file.
-    'file_integrity': Boolean indicating if the file is readable by echopype.
-    'use_swap': Applicable only for raw files.\
+        'file_path': Absolute path to the file.
+        'campaign_id': Identifier for the measuring campaign,
+                      extracted from the file name.
+        'date': Date and time when the measurement started,
+                extracted from the file name. Returns None if the date
+                and time cannot be determined.
+        'file_integrity': Boolean indicating if the file is of a supported type.
+        'use_swap': Applicable only for raw files.\
      A Boolean indicating whether the option was used when reading raw files or not.
 
     Raises:
-
-    - Exception: If the file type is not supported or
-    if there are issues reading the file.
+    - Exception: If the file type is not supported.
 
     Example:
-
     file_integrity_checking("/path/to/JR161-D20230509-T100645.raw")
     {
-    'file_path': '/path/to/JR161-D20230509-T100645.raw',
-    'campaign_id': 'JR161',
-    'date': datetime.datetime(2023, 5, 9, 10, 6, 45),
-    'sonar_model': 'EK60',
-    'file_integrity': True
-    'use_swap': False
+        'file_path': '/path/to/JR161-D20230509-T100645.raw',
+        'campaign_id': 'JR161',
+        'date': datetime.datetime(2023, 5, 9, 10, 6, 45),
+        'file_integrity': True
     }
     """
     return_dict = {}
+
     # get file name from path
-    file_name = os.path.split(file_path)[-1]
-    # eliminate file type
-    file_name = file_name.split(".")[0]
-    campaign_id = file_name.split("-")[0]
-    no_date_from_file_name = False
-    date = datetime.now()
-    try:
-        pattern_date = r"D(\d{4})(\d{2})(\d{2})"
-        pattern_time = r"T(\d{2})(\d{2})(\d{2})"
+    file_path = os.path.abspath(file_path)
+    _, file_name = os.path.split(file_path)
+    _, file_extension = os.path.splitext(file_path)
+    file_extension = file_extension.lower()
+
+    if file_extension not in [".raw", ".nc", ".zarr"]:
+        raise Exception("File type not supported for " + str(file_path))
 
-        matches_date = re.findall(pattern_date, file_name)[0]
-        matches_time = re.findall(pattern_time, file_name)[0]
+    file_integrity = True
+    metadata = None
+    sonar_model = None
+    date = None
+    campaign_id = None
 
-        year, month, day = matches_date
-        hour, minute, second = matches_time
+    if ".raw" == file_extension:
+        metadata = parse_metadata(file_path)
+        if metadata is not None:
+            campaign_id = metadata.get("survey_name", None)
+            date = metadata.get("timestamp", None)
+            sonar_model = detect_sonar_model(file_path, metadata=metadata)
+
+    if not metadata:
+        campaign_id = file_name.split("-")[0]
 
-        datetime_string = f"D{year}{month}{day}-T{hour}{minute}{second}"
-        date = datetime.strptime(datetime_string, "D%Y%m%d-T%H%M%S")
-    except Exception as e:
-        e += "!"
-        no_date_from_file_name = True
-
-    if ".raw" in file_path:
-        for s_m in SUPPORTED_SONAR_MODELS:
-            try:
-                ed = ep.open_raw(file_path, sonar_model=s_m, use_swap=use_swap)  # type: ignore
-                file_integrity = True
-                break
-            except Exception:
-                continue
-        if not file_integrity:
-            raise Exception("File type not supported for " + str(file_path))
-    elif ".nc" in file_path or ".zarr" in file_path:
         try:
-            ed = ep.open_converted(file_path)
-            file_integrity = True
-        except ValueError:
-            raise Exception("File type not supported for " + str(file_path))
-    else:
-        raise Exception("File type not supported for " + str(file_path))
-    if no_date_from_file_name:
-        datetime_string = ed["Top-level"].date_created
-        date = datetime.strptime(datetime_string, "%Y-%m-%dT%H:%M:%SZ")
+            pattern_date = r"D(\d{4})(\d{2})(\d{2})"
+            pattern_time = r"T(\d{2})(\d{2})(\d{2})"
+
+            matches_date = re.findall(pattern_date, file_name)[0]
+            matches_time = re.findall(pattern_time, file_name)[0]
+
+            year, month, day = matches_date
+            hour, minute, second = matches_time
+
+            datetime_string = f"D{year}{month}{day}-T{hour}{minute}{second}"
+            date = datetime.strptime(datetime_string, "D%Y%m%d-T%H%M%S")
+        except Exception as e:
+            e += "!"
+            file_integrity = False
+            date = None
 
     return_dict["file_path"] = file_path
     return_dict["campaign_id"] = campaign_id
     return_dict["date"] = date
-    return_dict["sonar_model"] = ed.sonar_model
     return_dict["file_integrity"] = file_integrity
-    if ".raw" in file_path:
+    return_dict["sonar_model"] = sonar_model
+
+    if ".raw" == file_extension:
         return_dict["use_swap"] = use_swap
+
     return return_dict
 
 
@@ -271,7 +268,7 @@ def read_raw_files(
     """
     ret_list = []
     for f_i in file_dicts:
-        opened_file = _read_file(f_i["file_path"], f_i["sonar_model"], f_i["use_swap"])
+        opened_file = _read_file(file_path=f_i["file_path"])
         ret_list.append(opened_file)
     return ret_list
 
@@ -301,9 +298,7 @@ def read_processed_files(file_paths: List[str]) -> List[ep.echodata.EchoData]:
     return ret_list
 
 
-def _read_file(
-    file_path: str, sonar_model: str = "EK80", use_swap: bool = False
-) -> ep.echodata.EchoData:
+def _read_file(file_path: str, sonar_model: str = None) -> ep.echodata.EchoData:
     """
     Reads an echo sounder file and
     returns the corresponding Dataset.
@@ -334,7 +329,10 @@ def _read_file(
     """
     file_name = os.path.split(file_path)[-1]
     if ".raw" in file_name:
-        ed = ep.open_raw(file_path, sonar_model=sonar_model, use_swap=use_swap)  # type: ignore
+        if sonar_model is None:
+            sonar_model = detect_sonar_model(file_path)
+
+        ed = ep.open_raw(file_path, sonar_model=sonar_model)  # type: ignore
     elif ".nc" in file_name or ".zarr" in file_name:
         ed = ep.open_converted(file_path)  # create an EchoData object
     else:
@@ -373,7 +371,7 @@ def convert_raw_files(
     """
     ret_list = []
     for f_i in file_dicts:
-        opened_file = _read_file(f_i["file_path"], f_i["sonar_model"], f_i["use_swap"])
+        opened_file = _read_file(file_path=f_i["file_path"])
         _write_file(opened_file, save_path, save_file_type)
         file_name = os.path.split(f_i["file_path"])[-1]
         file_type = save_file_type
@@ -506,3 +504,41 @@ def concatenate_files(
         list_of_datasets.append(_read_file(file_info["file_path"]))
     combined_dataset = ep.combine_echodata(list_of_datasets)
     return combined_dataset
+
+
+def parse_metadata(file_path):
+    try:
+        with RawSimradFile(file_path, "r", storage_options={}) as fid:
+            config_datagram = fid.read(1)
+            return config_datagram
+    except Exception as e:
+        print(f"Error parsing metadata from {file_path}. Error: {e}")
+        return None
+
+
+def detect_sonar_model(file_path: str, metadata=None) -> str:
+    if metadata is None:
+        metadata = parse_metadata(file_path)
+
+    if metadata is None:
+        return None
+
+    if "sounder_name" not in metadata:
+        try:
+            xml_string = metadata.get("xml", None)
+            root = ET.fromstring(xml_string)
+            header = root.find("Header")
+            application_name = header.attrib.get("ApplicationName")
+
+            if application_name == "EK80":
+                return "EK80"
+
+        except ET.ParseError:
+            return None
+
+        return None
+
+    if metadata["sounder_name"] == "EK60" or metadata["sounder_name"] == "ER60":
+        return "EK60"
+
+    return metadata["sounder_name"]

oceanstream/L2_calibrated_data/processed_data_io.py

@@ -111,8 +111,14 @@ def write_processed(
         raise TypeError("Expected a xarray Dataset")
 
     path = Path(file_path)
-    if not path.is_dir():
-        raise FileNotFoundError(f"Invalid path provided: {path}")
+    if not path.exists():
+        try:
+            path.mkdir(parents=True, exist_ok=True)
+        except Exception as e:
+            raise Exception(f"An error occurred while creating the directory: {e}")
+    else:
+        if not path.is_dir():
+            raise NotADirectoryError(f"Path exists but is not a directory: {path}")
 
     if not file_name:
         file_name = Path(sv.source_filenames[0].values.item()).stem