diff --git a/src/gaiaunlimited/fetch_utils.py b/src/gaiaunlimited/fetch_utils.py index 63257da..af5c93e 100644 --- a/src/gaiaunlimited/fetch_utils.py +++ b/src/gaiaunlimited/fetch_utils.py @@ -1,3 +1,4 @@ +import ftplib import gzip import hashlib import io @@ -20,7 +21,7 @@ class DownloadError(Exception): def get_datadir(): """Get gaiasf data directory as Path. - + Return type: pathlib.PosixPath object """ @@ -83,6 +84,87 @@ def download(url, file, desc=None, chunk_size=1024, md5sum=None): file.seek(0) +def download_ftp(url, dest_file, desc=None, chunk_size=1024, md5sum=None): + """Download file from an FTP url. + + Args: + url (str): url string + dest_file (file object): destination file object to write the content to. + desc (str, optional): Description of progressbar. Defaults to None. + chunk_size (int, optional): Chunk size to iteratively update progrss and md5sum. Defaults to 1024. + md5sum (str, optional): The expected md5sum to check against. Defaults to None. + + Raises: + DownloadError: raised when md5sum differs. + ValueError: raised when the input url is invalid + """ + url_bits = [x for x in url.split("/") if x] + if url_bits[0].startswith("ftp"): + FTP_Class = ftplib.FTP + elif url_bits[0].startswith("sftp"): + FTP_Class = ftplib.FTP_TLS + else: + raise ValueError("invalid url for FTP download") + + auth_bits = url_bits[1].split("@") + if len(auth_bits) == 1: + user = "" + passwd = "" + host = auth_bits[0] + else: + user, passwd = auth_bits[0].split(":") + host = auth_bits[1] + + path = "/".join(url_bits[2:-1]) + filename = url_bits[-1] + + if desc is None: + desc = filename + + if len(path) == 0 or len(filename) == 0: + raise ValueError( + "failed to parse input url as a valid server, path, and filename" + ) + + with FTP_Class(host=host, user=user, passwd=passwd) as ftp: + ftp.prot_p() + ftp.cwd(path) + total = ftp.size(filename) + sig = hashlib.md5() + + with io.BytesIO() as rawfile: + with tqdm( + desc=desc, + total=total, + unit="iB", + unit_scale=True, + unit_divisor=1024, + ) as bar: + + def tqdm_callback(data): + bar.update(len(data)) + rawfile.write(data) + + ftp.retrbinary(f"RETR {filename}", tqdm_callback) + + if md5sum: + if sig.hexdigest() != md5sum: + raise DownloadError( + "The MD5 sum of the downloaded file is incorrect.\n" + + f" download: {sig.hexdigest()}\n" + + f" expected: {md5sum}\n" + ) + + rawfile.seek(0) + if filename.endswith("gz"): + with gzip.open(rawfile) as tmp: + shutil.copyfileobj(tmp, dest_file) + else: + shutil.copyfileobj(rawfile, dest_file) + + dest_file.seek(0) + + scanlaw_datafiles = { "dr2_cog3": { "url": "https://zenodo.org/record/8300616/files/cog_dr2_scanning_law_v2.csv", @@ -104,6 +186,11 @@ def download(url, file, desc=None, chunk_size=1024, md5sum=None): "md5sum": "82d24407396f6008a9d72608839533a8", "column_mapping": {"jd_time": "tcb_at_gaia"}, }, + "full_operational_mission": { + "url": "sftp://anonymous:@ftp.cosmos.esa.int/GAIA_PUBLIC_DATA/GaiaScanningLaw/FullGaiaMissionScanningLaw/commanded_scan_law.csv.gz", + "md5sum": "d41d8cd98f00b204e9800998ecf8427e", + "column_mapping": {"jd_time": "tcb_at_gaia"}, + }, } @@ -137,8 +224,11 @@ def download_scanninglaw(name): return with io.BytesIO() as f: desc = "Downloading {name} scanning law file".format(name=name) - download(item["url"], f, md5sum=item["md5sum"], desc=desc) - df = pd.read_csv(f).rename(columns=item["column_mapping"]) + if item["url"].startswith("ftp") or item["url"].startswith("sftp"): + download_ftp(item["url"], f, md5sum=item["md5sum"], desc=desc) + else: + download(item["url"], f, md5sum=item["md5sum"], desc=desc) + df = pd.read_csv(f, comment="#").rename(columns=item["column_mapping"]) savedir.mkdir(exist_ok=True) df.to_pickle(savepath) @@ -166,7 +256,7 @@ def _get_data(self, filename): """Download data files specified in datafiles dict class attribute.""" savedir = get_datadir() if not savedir.exists(): - print('Creating directory',savedir) + print("Creating directory", savedir) os.makedirs(savedir) fullpath = get_datadir() / filename if not fullpath.exists(): diff --git a/src/gaiaunlimited/scanninglaw.py b/src/gaiaunlimited/scanninglaw.py index 896c3e4..db984ab 100644 --- a/src/gaiaunlimited/scanninglaw.py +++ b/src/gaiaunlimited/scanninglaw.py @@ -2,12 +2,11 @@ from pathlib import Path # from time import perf_counter - import astropy.coordinates as coord import astropy.units as u -from astropy.coordinates.funcs import spherical_to_cartesian import numpy as np import pandas as pd +from astropy.coordinates.funcs import spherical_to_cartesian from scipy import spatial from gaiaunlimited import fetch_utils @@ -37,6 +36,19 @@ def obmt2tcbgaia(obmt): return (obmt - 1717.6256) / 4 - (2455197.5 - 2457023.5 - 0.25) +def tcbgaia2obmt(tcb_jd): + """ + Calculate OnBoard Mission Time (OBMT, revs) from Gaia Barycenter coordinate time (TCB, days). + + Args: + tcb: TCB in days. + + Returns: + obmt: OBMT in revs. + """ + return 4 * (tcb_jd + (2455197.5 - 2457023.5 - 0.25)) + 1717.6256 + + def make_rotmat(fov1_xyz, fov2_xyz): """Make rotational matrix from ICRS to Gaia body frame(ish). @@ -60,10 +72,10 @@ def make_rotmat(fov1_xyz, fov2_xyz): def angle2dist3d(sepangle): """ Get equivalent 3d distance of an angle on a unit sphere. - + Args: sepangle (float): separation in degree - + Returns: float: distance corresponding to the angle on a unit sphere """ @@ -74,10 +86,10 @@ def angle2dist3d(sepangle): def cartesian_to_spherical(xyz): """ Convert cartesian XYZ to (longitude,latitude). - + Args: xyz ((N,3) array): (X,Y,Z) coordinates for each point - + Returns: (2,N) array: longitude and latitude of each point """ @@ -90,6 +102,7 @@ def cartesian_to_spherical(xyz): # def spherical_to_cartesian() + # TODO jit def check_gaps(gaps, x): """Check if values of array x falls in any gaps. @@ -108,6 +121,10 @@ def check_gaps(gaps, x): version_mapping = { + "full_operational_mission": { + "filename": "commanded_scan_law.csv", + "column_mapping": {"jd_time": "tcb_at_gaia"}, + }, "dr3_nominal": { "filename": "CommandedScanLaw_001.csv", "column_mapping": {"jd_time": "tcb_at_gaia"}, @@ -148,7 +165,8 @@ class GaiaScanningLaw: Args: version (str, required): Version of the FoV pointing data file to use. - One of ["dr3_nominal", "dr2_nominal", "dr2_cog3"]. Defaults to "dr3_nominal". + One of ["dr3_nominal", "dr2_nominal", "dr2_cog3", + "full_operational_mission"]. Defaults to "dr3_nominal". gaplist (str, optional): Name of the gap list. Defaults to "dr3/Astrometry". The gaplist should be "/". Possible values are: @@ -165,10 +183,10 @@ class GaiaScanningLaw: "cog3_2020": [1192.13, 3750.56], "dr2_nominal": [1192.13, 3750.56], "dr3_nominal": [1192.13, 5230.09], + "full_operational_mission": [1078.38, 17052.625], } def __init__(self, version="dr3_nominal", gaplist="dr3/Astrometry"): - if version not in version_mapping: raise ValueError("Unsupported version") self.version = version