Add support for full operational mission scanning law (#53)

* added functionality for downloading a file from ftp or sftp, as is needed for operational mission scanning law

* add inverse function

* initial enable of full mission scan law

* fix md5 hash

* add new option to docstring

* support for ecsv if necessary

src/gaiaunlimited/fetch_utils.py

@@ -1,3 +1,4 @@
+import ftplib
 import gzip
 import hashlib
 import io
@@ -20,7 +21,7 @@ class DownloadError(Exception):
 
 def get_datadir():
     """Get gaiasf data directory as Path.
-    
+
     Return type:
         pathlib.PosixPath object
     """
@@ -83,6 +84,87 @@ def download(url, file, desc=None, chunk_size=1024, md5sum=None):
     file.seek(0)
 
 
+def download_ftp(url, dest_file, desc=None, chunk_size=1024, md5sum=None):
+    """Download file from an FTP url.
+
+    Args:
+        url (str): url string
+        dest_file (file object): destination file object to write the content to.
+        desc (str, optional): Description of progressbar. Defaults to None.
+        chunk_size (int, optional): Chunk size to iteratively update progrss and md5sum. Defaults to 1024.
+        md5sum (str, optional): The expected md5sum to check against. Defaults to None.
+
+    Raises:
+        DownloadError: raised when md5sum differs.
+        ValueError: raised when the input url is invalid
+    """
+    url_bits = [x for x in url.split("/") if x]
+    if url_bits[0].startswith("ftp"):
+        FTP_Class = ftplib.FTP
+    elif url_bits[0].startswith("sftp"):
+        FTP_Class = ftplib.FTP_TLS
+    else:
+        raise ValueError("invalid url for FTP download")
+
+    auth_bits = url_bits[1].split("@")
+    if len(auth_bits) == 1:
+        user = ""
+        passwd = ""
+        host = auth_bits[0]
+    else:
+        user, passwd = auth_bits[0].split(":")
+        host = auth_bits[1]
+
+    path = "/".join(url_bits[2:-1])
+    filename = url_bits[-1]
+
+    if desc is None:
+        desc = filename
+
+    if len(path) == 0 or len(filename) == 0:
+        raise ValueError(
+            "failed to parse input url as a valid server, path, and filename"
+        )
+
+    with FTP_Class(host=host, user=user, passwd=passwd) as ftp:
+        ftp.prot_p()
+        ftp.cwd(path)
+        total = ftp.size(filename)
+        sig = hashlib.md5()
+
+        with io.BytesIO() as rawfile:
+            with tqdm(
+                desc=desc,
+                total=total,
+                unit="iB",
+                unit_scale=True,
+                unit_divisor=1024,
+            ) as bar:
+
+                def tqdm_callback(data):
+                    bar.update(len(data))
+                    rawfile.write(data)
+
+                ftp.retrbinary(f"RETR {filename}", tqdm_callback)
+
+            if md5sum:
+                if sig.hexdigest() != md5sum:
+                    raise DownloadError(
+                        "The MD5 sum of the downloaded file is incorrect.\n"
+                        + f"  download: {sig.hexdigest()}\n"
+                        + f"  expected: {md5sum}\n"
+                    )
+
+            rawfile.seek(0)
+            if filename.endswith("gz"):
+                with gzip.open(rawfile) as tmp:
+                    shutil.copyfileobj(tmp, dest_file)
+            else:
+                shutil.copyfileobj(rawfile, dest_file)
+
+    dest_file.seek(0)
+
+
 scanlaw_datafiles = {
     "dr2_cog3": {
         "url": "https://zenodo.org/record/8300616/files/cog_dr2_scanning_law_v2.csv",
@@ -104,6 +186,11 @@ def download(url, file, desc=None, chunk_size=1024, md5sum=None):
         "md5sum": "82d24407396f6008a9d72608839533a8",
         "column_mapping": {"jd_time": "tcb_at_gaia"},
     },
+    "full_operational_mission": {
+        "url": "sftp://anonymous:@ftp.cosmos.esa.int/GAIA_PUBLIC_DATA/GaiaScanningLaw/FullGaiaMissionScanningLaw/commanded_scan_law.csv.gz",
+        "md5sum": "d41d8cd98f00b204e9800998ecf8427e",
+        "column_mapping": {"jd_time": "tcb_at_gaia"},
+    },
 }
 
 
@@ -137,8 +224,11 @@ def download_scanninglaw(name):
             return
         with io.BytesIO() as f:
             desc = "Downloading {name} scanning law file".format(name=name)
-            download(item["url"], f, md5sum=item["md5sum"], desc=desc)
-            df = pd.read_csv(f).rename(columns=item["column_mapping"])
+            if item["url"].startswith("ftp") or item["url"].startswith("sftp"):
+                download_ftp(item["url"], f, md5sum=item["md5sum"], desc=desc)
+            else:
+                download(item["url"], f, md5sum=item["md5sum"], desc=desc)
+            df = pd.read_csv(f, comment="#").rename(columns=item["column_mapping"])
             savedir.mkdir(exist_ok=True)
             df.to_pickle(savepath)
 
@@ -166,7 +256,7 @@ def _get_data(self, filename):
         """Download data files specified in datafiles dict class attribute."""
         savedir = get_datadir()
         if not savedir.exists():
-            print('Creating directory',savedir)
+            print("Creating directory", savedir)
             os.makedirs(savedir)
         fullpath = get_datadir() / filename
         if not fullpath.exists():

src/gaiaunlimited/scanninglaw.py

@@ -2,12 +2,11 @@
 from pathlib import Path
 
 # from time import perf_counter
-
 import astropy.coordinates as coord
 import astropy.units as u
-from astropy.coordinates.funcs import spherical_to_cartesian
 import numpy as np
 import pandas as pd
+from astropy.coordinates.funcs import spherical_to_cartesian
 from scipy import spatial
 
 from gaiaunlimited import fetch_utils
@@ -37,6 +36,19 @@ def obmt2tcbgaia(obmt):
     return (obmt - 1717.6256) / 4 - (2455197.5 - 2457023.5 - 0.25)
 
 
+def tcbgaia2obmt(tcb_jd):
+    """
+    Calculate OnBoard Mission Time (OBMT, revs) from Gaia Barycenter coordinate time (TCB, days).
+
+    Args:
+        tcb: TCB in days.
+
+    Returns:
+        obmt: OBMT in revs.
+    """
+    return 4 * (tcb_jd + (2455197.5 - 2457023.5 - 0.25)) + 1717.6256
+
+
 def make_rotmat(fov1_xyz, fov2_xyz):
     """Make rotational matrix from ICRS to Gaia body frame(ish).
 
@@ -60,10 +72,10 @@ def make_rotmat(fov1_xyz, fov2_xyz):
 def angle2dist3d(sepangle):
     """
     Get equivalent 3d distance of an angle on a unit sphere.
-    
+
     Args:
         sepangle (float): separation in degree
-        
+
     Returns:
         float: distance corresponding to the angle on a unit sphere
     """
@@ -74,10 +86,10 @@ def angle2dist3d(sepangle):
 def cartesian_to_spherical(xyz):
     """
     Convert cartesian XYZ to (longitude,latitude).
-    
+
     Args:
         xyz ((N,3) array): (X,Y,Z) coordinates for each point
-        
+
     Returns:
         (2,N) array: longitude and latitude of each point
     """
@@ -90,6 +102,7 @@ def cartesian_to_spherical(xyz):
 
 # def spherical_to_cartesian()
 
+
 # TODO jit
 def check_gaps(gaps, x):
     """Check if values of array x falls in any gaps.
@@ -108,6 +121,10 @@ def check_gaps(gaps, x):
 
 
 version_mapping = {
+    "full_operational_mission": {
+        "filename": "commanded_scan_law.csv",
+        "column_mapping": {"jd_time": "tcb_at_gaia"},
+    },
     "dr3_nominal": {
         "filename": "CommandedScanLaw_001.csv",
         "column_mapping": {"jd_time": "tcb_at_gaia"},
@@ -148,7 +165,8 @@ class GaiaScanningLaw:
 
     Args:
         version (str, required): Version of the FoV pointing data file to use.
-            One of ["dr3_nominal", "dr2_nominal", "dr2_cog3"]. Defaults to "dr3_nominal".
+            One of ["dr3_nominal", "dr2_nominal", "dr2_cog3",
+            "full_operational_mission"]. Defaults to "dr3_nominal".
         gaplist (str, optional): Name of the gap list. Defaults to "dr3/Astrometry".
             The gaplist should be "<dr?>/<sample_name>". Possible values are:
 
@@ -165,10 +183,10 @@ class GaiaScanningLaw:
         "cog3_2020": [1192.13, 3750.56],
         "dr2_nominal": [1192.13, 3750.56],
         "dr3_nominal": [1192.13, 5230.09],
+        "full_operational_mission": [1078.38, 17052.625],
     }
 
     def __init__(self, version="dr3_nominal", gaplist="dr3/Astrometry"):
-
         if version not in version_mapping:
             raise ValueError("Unsupported version")
         self.version = version