Rcal 596 Updates for association processing

CHANGES.rst

@@ -1,5 +1,7 @@
 0.17.0 (unreleased)
 ===================
+- Add check for filetypes for association processing [#241]
+
 - Remove the ``random_utils`` module and make ``maker_utils`` entirely deterministic. [#217]
 
 - Add tests to ensure consistency between file-level schemas in RAD and the corresponding

src/roman_datamodels/datamodels/_utils.py

@@ -7,7 +7,7 @@
 import asdf
 import packaging.version
 
-from roman_datamodels import validate
+from roman_datamodels import validate, filetype
 
 from ._core import MODEL_REGISTRY, DataModel
 
@@ -31,7 +31,7 @@
 def rdm_open(init, memmap=False, target=None, **kwargs):
     """
     Datamodel open/create function.
-        This function opens a Roman datamodel from an asdf file or generates
+        This function opens a Roman datamodel from various files or generates
         the datamodel from an existing one.
 
     Parameters
@@ -57,6 +57,11 @@ def rdm_open(init, memmap=False, target=None, **kwargs):
     """
     with validate.nuke_validation():
         file_to_close = None
+        if isinstance(init, str):
+            input_file_type = filetype.check(init)
+            if input_file_type =='asn':
+                print("Returning an asn string:", init)
+                return init
         if target is not None:
             if not issubclass(target, DataModel):
                 raise ValueError("Target must be a subclass of DataModel")
@@ -73,6 +78,7 @@ def rdm_open(init, memmap=False, target=None, **kwargs):
                     return init
             # Copy the object so it knows not to close here
             return init.copy()
+            # Copy the object so it knows not to close here
         else:
             try:
                 kwargs["copy_arrays"] = not memmap

src/roman_datamodels/filetype.py

@@ -0,0 +1,56 @@
+import io
+import os
+from pathlib import Path
+from typing import Union
+
+
+def check(init: Union[os.PathLike, Path, io.FileIO]) -> str:
+    """
+    Determine the type of a file and return it as a string
+
+    Parameters
+    ----------
+
+    init : str
+        file path or file object
+
+    Returns
+    -------
+    file_type: str
+        a string with the file type ("asdf" or "asn")
+
+    """
+
+    supported = ("asdf", "json")
+
+    if isinstance(init, (str, os.PathLike, Path)):
+        path, ext = os.path.splitext(init)
+        ext = ext.strip(".")
+
+        if not ext:
+            raise ValueError(f"Input file path does not have an extension: {init}")
+
+        if ext not in supported:  # Could be the file is zipped; try splitting again
+            path, ext = os.path.splitext(path)
+            ext = ext.strip(".")
+
+            if ext not in supported:
+                raise ValueError(f"Unrecognized file type for: {init}")
+
+        if ext == "json":  # Assume json input is an association
+            return "asn"
+
+        return ext
+    elif hasattr(init, "read") and hasattr(init, "seek"):
+        magic = init.read(5)
+        init.seek(0, 0)
+
+        if not magic or len(magic) < 5:
+            raise ValueError(f"Cannot get file type of {str(init)}")
+
+        if magic == b"#ASDF":
+            return "asdf"
+
+        return "asn"
+    else:
+        raise ValueError(f"Cannot get file type of {str(init)}")

tests/data/example_schema.json

@@ -0,0 +1,43 @@
+{
+      "date" : {
+          "title" : "[yyyy-mm-ddThh:mm:ss.ssssss] UTC date file created",
+          "type" : "string",
+          "sql_dtype" : "datetime2",
+          "fits_keyword" : "DATE",
+          "description" : "The UTC date and time when the HDU was created, in the form YYYY-MM-DDThh:mm:ss.ssssss, where YYYY shall be the four-digit calendar year number, MM the two-digit month number with January given by 01 and December by 12, and DD the two-digit day of the month. The literal T shall separate the date and time, hh shall be the two-digit hour in the day, mm the two-digit number of minutes after the hour, and ss.ssssss the number of seconds (two digits followed by a fraction accurate to microseconds) after the minute. Default values must not be given to any portion of the date/time string, and leading zeros must not be omitted.",
+          "calculation" : "Operating system time in the format of YYYY-MM-DDThh:mm:ss.ssssss",
+          "default_value" : "",
+          "example" : "2015-01-01T00:00:00.000001",
+          "units" : "",
+          "sw_source" : "calculation",
+          "source" : "Science Data Processing (SDP)",
+          "destination" : ["ScienceCommon.date","GuideStar.date"],
+          "level" : "1a",
+          "si" : "Multiple",
+          "section" : "Basic",
+          "mode" : "All",
+          "fits_hdu" : "PRIMARY",
+          "misc" : ""
+          },
+
+      "origin" : {
+          "title" : "institution responsible for creating FITS file",
+          "type" : "string",
+          "sql_dtype" : "nvarchar(20)",
+          "fits_keyword" : "ORIGIN",
+          "description" : "Identifies the organization or institution responsible for creating the FITS file.",
+          "calculation" : "",
+          "default_value" : "STSCI",
+          "example" : "STSCI",
+          "units" : "",
+          "sw_source" : "",
+          "source" : "Science Data Processing (SDP)",
+          "destination" : ["ScienceCommon.origin","GuideStar.origin"],
+          "level" : "1a",
+          "si" : "Multiple",
+          "section" : "Basic",
+          "mode" : "All",
+          "fits_hdu" : "PRIMARY",
+          "misc" : ""
+          }
+}

tests/data/fake.asdf

@@ -0,0 +1 @@
+not actually an ASDF file

tests/data/fake.json

@@ -0,0 +1 @@
+not actually a JSON file

tests/test_filetype.py

@@ -0,0 +1,41 @@
+from pathlib import Path
+
+import pytest
+
+from roman_datamodels import filetype
+
+DATA_DIRECTORY = Path(__file__).parent / "data"
+
+
+def test_filetype():
+    file_1 = filetype.check(DATA_DIRECTORY / "empty.json")
+    file_2 = filetype.check(DATA_DIRECTORY / "example_schema.json")
+    with  open(DATA_DIRECTORY / "fake.json") as file_h:
+        file_3 = filetype.check(file_h)
+    file_4 = filetype.check(DATA_DIRECTORY / "empty.asdf")
+    file_5 = filetype.check(DATA_DIRECTORY / "pluto.asdf")
+    with  open(DATA_DIRECTORY / "pluto.asdf", "rb") as file_h:
+        file_6 = filetype.check(file_h)
+    file_7 = filetype.check(DATA_DIRECTORY / "fake.asdf")
+    with  open(DATA_DIRECTORY / "fake.json") as file_h:
+        file_8 = filetype.check(file_h)
+    file_9 = filetype.check(str(DATA_DIRECTORY / "pluto.asdf"))
+
+    assert file_1 == "asn"
+    assert file_2 == "asn"
+    assert file_3 == "asn"
+    assert file_4 == "asdf"
+    assert file_5 == "asdf"
+    assert file_6 == "asdf"
+    assert file_7 == "asdf"
+    assert file_8 == "asn"
+    assert file_9 == "asdf"
+
+    with pytest.raises(ValueError):
+        filetype.check(DATA_DIRECTORY / "empty.txt")
+
+    with pytest.raises(ValueError):
+        filetype.check(2)
+
+    with pytest.raises(ValueError):
+        filetype.check("test")