Multi thread compress

CHANGELOG.md

@@ -69,3 +69,9 @@ autoNICER is a program that allows individuals wanting to work with data from th
 - Limited get_caldb_ver() pulls to once per run of autonicer rather than once per OBSID passed in
 - Added error handling for datasets lacking the required metadata
 - Added unix like passthrough to `--inlist` so use cases like `--inlist *` will pass in all datasets for the current working dir
+
+### v1.2.3
+- Fixed error handling messages for missing metadata in cl.evt files
+- Switched the pull of required metadata from hdu 0 (primary header) to hdu 1 (event header) for the cl.evt NICER files
+- Added OBSID corolation for easier identification
+

autonicer/autonicer.py

@@ -19,6 +19,7 @@
 import shutil
 import warnings
 import glob
+import concurrent.futures
 from astropy.utils.exceptions import AstropyWarning
 
 
@@ -278,16 +279,19 @@ def gz_comp(file):
             else:
                 with open(file, "rb") as f_in:
                     with gzip.open(f"{file}.gz", "wb") as f_out:
-                        print(f"{file} -> {file}.gz")
                         shutil.copyfileobj(f_in, f_out)
                 os.remove(file)
+                return f"{file} -> {file}.gz"
 
         print("Compressing ufa.evt files")
         print("----------------------------------------------------------")
         # files and loop to compress the ufa files
         files = glob.glob("*ufa.evt")
-        for i in files:
-            gz_comp(i)
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            comps = [executor.submit(gz_comp, i) for i in files]
+
+            for j in concurrent.futures.as_completed(comps):
+                print(j.result())
 
         # compression of the non-bc mpu7_cl.evt file if barycenter correction is selected
         # if self.bc_sel.lower() == "n":

autonicer/reprocess.py

@@ -9,6 +9,23 @@
 from termcolor import colored
 import sys
 import glob
+import concurrent.futures
+
+
+def extract_gz(file) -> str:
+    with gzip.open(file, "rb") as gz_in:
+        fname = str(file).split(".gz")
+        with open(fname[0], "wb") as orig_out:
+            shutil.copyfileobj(gz_in, orig_out)
+    os.remove(file)
+    return f"{file} -> {fname[0]}"
+
+
+def extract_tar(file) -> str:
+    with tarfile.open(file, "r:gz") as tfile:
+        tfile.extractall()
+    os.remove(file)
+    return f"{file} extracted"
 
 
 class Reprocess:
@@ -103,20 +120,16 @@ def decompress(self):
         print(colored(f"######## Decompressing {self.obsid} ########", "green"))
         os.chdir(f"{self.base_dir}/xti/event_cl/")
         gzs = glob.glob("*.evt.gz")
-        for i in gzs:
-            with gzip.open(i, "rb") as gz_in:
-                fname = str(i).split(".gz")
-                with open(fname[0], "wb") as orig_out:
-                    print(f"{i} -> {fname[0]}")
-                    shutil.copyfileobj(gz_in, orig_out)
-            os.remove(i)
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            exts_gz = [executor.submit(extract_gz, i) for i in gzs]
+            for j in concurrent.futures.as_completed(exts_gz):
+                print(j.result())
+
         tars = glob.glob("*.tar.gz")
-        for i in tars:
-            tfile = tarfile.open(i, "r:gz")
-            print(f"Extracting {i}")
-            tfile.extractall()
-            tfile.close()
-            os.remove(i)
+        with concurrent.futures.ThreadPoolExecutor() as executor:
+            ext_tar = [executor.submit(extract_tar, i) for i in tars]
+            for j in concurrent.futures.as_completed(ext_tar):
+                print(j.result())
         if len(gzs) > 0 or len(tars) > 0:
             self.comp_det = True
         os.chdir(self.base_dir)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "autonicer"
-version = "1.2.3"
+version = "1.2.4"
 description = "A program that retrieves NICER observational data sets and performs a default data reduction process on the NICER observational data"
 authors = ["Tsar Bomba Nick <njkuechel@protonmail.com>"]
 license = "Apache-2.0"

tests/test_autonicer.py

@@ -9,6 +9,7 @@
 import subprocess as sp
 import tarfile
 import glob
+from termcolor import colored
 
 base_dir = os.getcwd()
 os.mkdir("data")
@@ -278,7 +279,7 @@ def test_checkcal_reprocess(capsys):
     out, err = capsys.readouterr()
     passing = f"----------  Passing Reprocess of 3013010102  ----------"
     fail = "DATASETS NOT FOUND"
-    unix = "Migrating to 3013010102"
+    unix = f"Migrating to {colored('3013010102', 'cyan')}"
     unix2 = "test.csv is not a directory! Passing..."
     assert passing in out
     assert fail in out