From 0421383586a86d3f3d1cfb7ff0ef0d485ce37ca3 Mon Sep 17 00:00:00 2001 From: Sean-Morrison Date: Fri, 1 Nov 2024 12:12:54 -0500 Subject: [PATCH 01/14] update boss paths for v6_2+ --- python/sdss_access/path/path.py | 228 +++++++++++++++++++++++++++++--- tests/path/test_sdss5.py | 20 ++- 2 files changed, 226 insertions(+), 22 deletions(-) diff --git a/python/sdss_access/path/path.py b/python/sdss_access/path/path.py index 9b951d8..78948cd 100644 --- a/python/sdss_access/path/path.py +++ b/python/sdss_access/path/path.py @@ -335,10 +335,22 @@ def extract(self, name, example): template = re.sub('@healpixgrp[|]', '{healpixgrp}', template) elif re.search('@configgrp[|]', template): template = re.sub('@configgrp[|]', '{configgrp}', template) - elif re.search('@isplate[|]', template): + if re.search('@isplate[|]', template): template = re.sub('@isplate[|]', '{isplate}', template) - elif re.search('@pad_fieldid[|]', template): + if re.search('@pad_fieldid[|]', template): template = re.sub('@pad_fieldid[|]', '{fieldid}', template) + if re.search('@spcoaddfolder[|]',template): + template = re.sub('@spcoaddfolder[|]', '{spcoaddfolder}', template) + if re.search('@sptypefolder[|]',template): + template = re.sub('@sptypefolder[|]', '{sptypefolder}', template) + if re.search('@epochflag[|]', template): + template = re.sub('@epochflag[|]', '{epochflag}', template) + if re.search('@spcoaddobs[|]',template): + template = re.sub('@spcoaddobs[|]', '{obs}', template) + if re.search('@spcoaddgrp[|]',template): + template = re.sub('@spcoaddgrp[|]', '{spcoaddgrp}', template) + if re.search('@fieldgrp[|]',template): + template = re.sub('@fieldgrp[|]', '{fieldgrp}', template) if re.search('@plateid6[|]', template): template = re.sub('@plateid6[|]', '{plateid:0>6}', template) if re.search('@component_default[|]', template): @@ -386,12 +398,28 @@ def extract(self, name, example): drval = re.match('^DR[1-9][0-9]', value).group(0) otherval = value.split(drval)[-1] pdict = {keys[0]: drval, keys[1]: otherval} + elif keys == ['fieldid','isplate']: + # for {fieldid}{isplate} as isplate is calculated automatically + if value.endswith('p') and value[:-1].isdigit(): + value = value[:-1] + pdict = {keys[0]:value} + elif keys == ['run2d','epochflag']: + # for {run2d}{epochflag} as epochflag is calculated automatically + pdict = {keys[0]:value.replace('-epoch','')} + elif keys == ['coadd', 'obs']: + value = value.split('_') + if len(value) == 1: + value.append('') + pdict = {keys[0]: value[0], keys[1]: value[1]} elif keys[0] in ['rc', 'br', 'filter', 'camrow']: # for {camrow}{camcol}, {filter}{camcol}, {br}{id}, etc pdict = {keys[0]: value[0], keys[1]: value[1:]} else: raise ValueError('This case has not yet been accounted for.') path_dict.update(pdict) + elif keys[0] in ['sptypefolder','fieldgrp','spcoaddfolder','spcoaddgrp']: + # supress the keys since they are automatically calculated + continue else: path_dict[keys[0]] = value return path_dict @@ -1529,34 +1557,200 @@ def pad_fieldid(self, filetype, **kwargs): fieldid = str(fieldid) if run2d in ['v6_0_1','v6_0_2', 'v6_0_3', 'v6_0_4']: return str(fieldid) - if fieldid.isnumeric(): + elif fieldid.isnumeric(): return str(fieldid).zfill(6) else: return fieldid - def tilegrp(self, filetype, **kwargs): - ''' Returns LVM tile id group subdirectory + def spcoaddfolder(self, filetype, **kwargs): + ''' Returns the reorganized subfolder structure for the BOSS idlspec2d run2d version Parameters - ---------- + --------- filetype : str - File type parameter. - tileid : int or str - LVM Tile ID number. Will be converted to int internally. + File type parameter + run2d : str + BOSS idlspec2d run2d version + coadd : str + Name of the custom coadd schema + Returns + ------- + sptypefolder : str + ''' + + run2d = kwargs.get('run2d', None) + coaddname = kwargs.get('coadd', None) + + if (not run2d): + return '' + elif (('v5' in run2d) or (str(run2d) in ['26','103','104']) or + ('v6_0' in run2d) or ('v6_1' in run2d)): + return '' + else: + if filetype.lower() in ['spall_coadd','spall-lite_coadd','spallline_coadd']: + return('summary') + elif filetype.lower() in ['speclite_coadd','specfull_coadd', + 'spallfield_coadd','spalllinefield_coadd']: + return(coaddname) + else: + return('fields') + + def spcoaddgrp(self, filetype, **kwargs): + ''' Returns the coadd group (field group analog) subfolder structure for the BOSS idlspec2d run2d version + + Parameters + --------- + filetype : str + File type parameter + run2d : str + BOSS idlspec2d run2d version + coadd : str + Name of the custom coadd schema Returns ------- - tileidgrp : str - Tile ID group directory in the format ``NNNNXX``. + spcoaddgrp : str + ''' + run2d = kwargs.get('run2d', None) + coaddname = kwargs.get('coadd', None) + + if (not run2d): + return '' + elif (('v5' in run2d) or (str(run2d) in ['26','103','104']) or + ('v6_0' in run2d) or ('v6_1' in run2d)): + return '' + else: + return(coaddname) + + def sptypefolder(self, filetype, **kwargs): + ''' Returns the reorganized subfolder structure for the BOSS idlspec2d run2d version + Parameters + --------- + filetype : str + File type parameter + run2d : str + BOSS idlspec2d run2d version + Returns + ------- + sptypefolder : str ''' - tileid = kwargs.get('tileid', None) - if not tileid: - return '0000XX' - elif '*' in str(tileid): - return '{0}XX'.format(tileid) - return '{:0>4d}XX'.format(int(tileid) // 1000) + run2d = kwargs.get('run2d', None) + + if (not run2d): + return '' + elif ('v5' in run2d) or (str(run2d) in ['26','103','104']): + return '' + elif ('v6_0' in run2d) or ('v6_1' in run2d): + if filetype.lower() in ['speclite_epoch','specfull_epoch', + 'spallfield_epoch','spalllinefield_epoch']: + return('epoch/spectra') + else: + return '' + else: + if filetype.lower() in ['fieldlist_epoch','spall_epoch', + 'spall-lite_epoch','spallline_epoch']: + return('summary/epoch') + elif filetype.lower() in ['speclite_epoch','specfull_epoch', + 'spallfield_epoch','spalllinefield_epoch']: + return('spectra/epoch') + elif filetype.lower() in ['conflist','fieldlist','spall', + 'spall-lite','spallline']: + return('summary/daily') + elif filetype.lower() in ['speclite','specfull', + 'spallfield','spalllinefield']: + return('daily') + else: + return('fields') + + def spcoaddobs(self, filetype, **kwargs): + ''' Returns the formatted observatory flag for custom coadds for the BOSS idlspec2d + + Parameters + ---------- + filetype : str + File type parameter + run2d : str + BOSS idlspec2d run2d version + obs : str + Observatory of observations: LCO, APO, ''(for merged) + + Returns + ------- + obs : str + ''' + + obs = kwargs.get('obs', None) + run2d = kwargs.get('run2d', None) + + if not obs: + return '' + + elif obs == '': + return '' + elif (('v6_0' in run2d) or ('v6_1' in run2d)) and obs.lower() == 'apo': + return '' + elif obs == '*': + return obs + else: + return '_{}'.format(obs.lower()) + + def epochflag(self, filetype, **kwargs): + ''' Returns the flag for epoch coadds for the BOSS idlspec2d + + Parameters + ---------- + filetype : str + File type parameter + run2d : str + BOSS idlspec2d run2d version + + Returns + ------- + epochflag : str + ''' + + run2d = kwargs.get('run2d', None) + + if (('v6_0' in run2d) or ('v6_1' in run2d)): + return '' + else: + return '-epoch' + + def fieldgrp(self, filetype, **kwargs): + ''' Returns the fieldid group for the BOSS idlspec2d run2d version + + Parameters + --------- + filetype : str + File type parameter + run2d : str + BOSS idlspec2d run2d version + fieldid : str or int + Field ID number. Will be converted to str internally. + + Returns + ------- + fieldgrp : str + ''' + + fieldid = kwargs.get('fieldid', None) + run2d = kwargs.get('run2d', None) + + if (not fieldid): + return '' + + if ('v5' in run2d) or (str(run2d) in ['26','103','104']): + return '' + elif ('v6_0' in run2d) or ('v6_1' in run2d): + return '' + else: + fieldid = str(fieldid) + if fieldid.isnumeric(): + return '{:0>3d}XXX'.format(int(fieldid) // 1000) + else: + return(fieldid) class AccessError(Exception): pass diff --git a/tests/path/test_sdss5.py b/tests/path/test_sdss5.py index 9b56c87..2a46b5b 100644 --- a/tests/path/test_sdss5.py +++ b/tests/path/test_sdss5.py @@ -59,12 +59,22 @@ def test_apogee_paths(self, path, name, special, keys, exp): 'v6_0_4/1234p/spFrame-b1-00005432.fits.gz'), ('spField', '@pad_fieldid', {'run2d': 'v6_1_1', 'mjd': '59630', 'fieldid': '*'}, 'v6_1_1/*/spField-*-59630.fits'), - ('lvm_frame', '@tilegrp', {'drpver': 'master', 'mjd': 60235, - 'tileid': 1055360, 'kind': 'CFrame', 'expnum': 6817}, - '1055XX/1055360/60235/lvmCFrame-00006817.fits')], + ('spCOADDS','@spcoaddfolder',{'run2d':'v6_2_0','survey':'BHM','coadd':'allepoch','phase':'SDSSV'}, + 'v6_2_0/fields/SDSSV_BHM_COADDS.par'), + ('spField','@sptypefolder',{'run2d':'v6_2_0','mjd': '59630', 'fieldid': '123456'}, + 'v6_2_0/fields/123XXX/123456/spField-123456-59630.fits'), + ('spField','@fieldgrp',{'run2d':'v6_2_0','mjd': '59630', 'fieldid': '123456'}, + 'v6_2_0/fields/123XXX/123456/spField-123456-59630.fits'), + ('spFullsky','@spcoaddobs',{'run2d':'v6_2_0','mjd': '59630','coadd':'allepoch','obs':'apo'}, + 'v6_2_0/fields/allepoch/allepoch_apo/spFullsky-allepoch_apo-59630.fits'), + ('spAll_epoch','@epochflag',{'run2d':'v6_2_0'}, + 'v6_2_0/summary/epoch/spAll-v6_2_0-epsilon-epoch.fits.gz'), + ('spAll_coadd','@spcoaddgrp',{'run2d':'v6_2_0','coadd':'allepoch'}, + 'v6_2_0/summary/test/spAll-v6_2_0-epsilon-allepoch.fits.gz')], ids=['configgrp', 'apgprefix-apo', 'apgprefix-lco', 'apgprefix-ins', - 'isplate-v6_0_4','pad_fieldid-5','pad_fieldid-6', 'frame-pad', 'frame-nopadp', - 'pad_fieldid-*', 'lvm-tileid']) + 'isplate-v6_0_4','pad_fieldid-5','pad_fieldid-6', 'frame-pad', + 'frame-nopadp', 'pad_fieldid-*','spcoaddfolder', 'sptypefolder', + 'fieldgrp','spcoaddobs','epochflag','spcoaddgrp']) def test_special_function(self, path, name, special, keys, exp): assert special in path.templates[name] full = path.full(name, **keys) From f93506bf07590da05e67e1c2919073612f6fd157 Mon Sep 17 00:00:00 2001 From: Sean-Morrison Date: Tue, 5 Nov 2024 15:14:17 -0600 Subject: [PATCH 02/14] Fixed bug in sptypefolder special function --- python/sdss_access/path/path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/sdss_access/path/path.py b/python/sdss_access/path/path.py index 78948cd..fdd90a6 100644 --- a/python/sdss_access/path/path.py +++ b/python/sdss_access/path/path.py @@ -1647,7 +1647,7 @@ def sptypefolder(self, filetype, **kwargs): 'spallfield_epoch','spalllinefield_epoch']: return('epoch/spectra') else: - return '' + return 'epoch' else: if filetype.lower() in ['fieldlist_epoch','spall_epoch', 'spall-lite_epoch','spallline_epoch']: From 4f642d29796d3b1fc07b9a26e2ff7fe44057c3eb Mon Sep 17 00:00:00 2001 From: Sean-Morrison Date: Tue, 5 Nov 2024 15:44:54 -0600 Subject: [PATCH 03/14] Update path.py --- python/sdss_access/path/path.py | 99 ++++++++++++++------------------- 1 file changed, 42 insertions(+), 57 deletions(-) diff --git a/python/sdss_access/path/path.py b/python/sdss_access/path/path.py index fdd90a6..9c23329 100644 --- a/python/sdss_access/path/path.py +++ b/python/sdss_access/path/path.py @@ -1557,10 +1557,9 @@ def pad_fieldid(self, filetype, **kwargs): fieldid = str(fieldid) if run2d in ['v6_0_1','v6_0_2', 'v6_0_3', 'v6_0_4']: return str(fieldid) - elif fieldid.isnumeric(): + if fieldid.isnumeric(): return str(fieldid).zfill(6) - else: - return fieldid + return fieldid def spcoaddfolder(self, filetype, **kwargs): ''' Returns the reorganized subfolder structure for the BOSS idlspec2d run2d version @@ -1583,17 +1582,15 @@ def spcoaddfolder(self, filetype, **kwargs): if (not run2d): return '' - elif (('v5' in run2d) or (str(run2d) in ['26','103','104']) or - ('v6_0' in run2d) or ('v6_1' in run2d)): + if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or + ('v6_0' in run2d) or ('v6_1' in run2d)): return '' - else: - if filetype.lower() in ['spall_coadd','spall-lite_coadd','spallline_coadd']: - return('summary') - elif filetype.lower() in ['speclite_coadd','specfull_coadd', - 'spallfield_coadd','spalllinefield_coadd']: - return(coaddname) - else: - return('fields') + if filetype.lower() in ['spall_coadd','spall-lite_coadd','spallline_coadd']: + return 'summary' + if filetype.lower() in ['speclite_coadd','specfull_coadd', + 'spallfield_coadd','spalllinefield_coadd']: + return coaddname + return('fields') def spcoaddgrp(self, filetype, **kwargs): @@ -1616,11 +1613,10 @@ def spcoaddgrp(self, filetype, **kwargs): if (not run2d): return '' - elif (('v5' in run2d) or (str(run2d) in ['26','103','104']) or - ('v6_0' in run2d) or ('v6_1' in run2d)): + if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or + ('v6_0' in run2d) or ('v6_1' in run2d)): return '' - else: - return(coaddname) + return coaddname def sptypefolder(self, filetype, **kwargs): ''' Returns the reorganized subfolder structure for the BOSS idlspec2d run2d version @@ -1638,31 +1634,26 @@ def sptypefolder(self, filetype, **kwargs): run2d = kwargs.get('run2d', None) - if (not run2d): + if (not run2d) or ('v5' in run2d) or (str(run2d) in ['26','103','104']): return '' - elif ('v5' in run2d) or (str(run2d) in ['26','103','104']): - return '' - elif ('v6_0' in run2d) or ('v6_1' in run2d): + if ('v6_0' in run2d) or ('v6_1' in run2d): if filetype.lower() in ['speclite_epoch','specfull_epoch', 'spallfield_epoch','spalllinefield_epoch']: - return('epoch/spectra') - else: - return 'epoch' - else: - if filetype.lower() in ['fieldlist_epoch','spall_epoch', - 'spall-lite_epoch','spallline_epoch']: - return('summary/epoch') - elif filetype.lower() in ['speclite_epoch','specfull_epoch', - 'spallfield_epoch','spalllinefield_epoch']: - return('spectra/epoch') - elif filetype.lower() in ['conflist','fieldlist','spall', - 'spall-lite','spallline']: - return('summary/daily') - elif filetype.lower() in ['speclite','specfull', - 'spallfield','spalllinefield']: - return('daily') - else: - return('fields') + return 'epoch/spectra' + return 'epoch' + if filetype.lower() in ['fieldlist_epoch','spall_epoch', + 'spall-lite_epoch','spallline_epoch']: + return 'summary/epoch' + if filetype.lower() in ['speclite_epoch','specfull_epoch', + 'spallfield_epoch','spalllinefield_epoch']: + return 'spectra/epoch' + if filetype.lower() in ['conflist','fieldlist','spall', + 'spall-lite','spallline']: + return 'summary/daily' + if filetype.lower() in ['speclite','specfull', + 'spallfield','spalllinefield']: + return 'daily' + return 'fields' def spcoaddobs(self, filetype, **kwargs): ''' Returns the formatted observatory flag for custom coadds for the BOSS idlspec2d @@ -1684,17 +1675,13 @@ def spcoaddobs(self, filetype, **kwargs): obs = kwargs.get('obs', None) run2d = kwargs.get('run2d', None) - if not obs: + if not obs or obs == '': return '' - - elif obs == '': + if (('v6_0' in run2d) or ('v6_1' in run2d)) and obs.lower() == 'apo': return '' - elif (('v6_0' in run2d) or ('v6_1' in run2d)) and obs.lower() == 'apo': - return '' - elif obs == '*': + if obs == '*': return obs - else: - return '_{}'.format(obs.lower()) + return '_{}'.format(obs.lower()) def epochflag(self, filetype, **kwargs): ''' Returns the flag for epoch coadds for the BOSS idlspec2d @@ -1712,11 +1699,11 @@ def epochflag(self, filetype, **kwargs): ''' run2d = kwargs.get('run2d', None) - + if ('v5' in run2d) or (str(run2d) in ['26','103','104']): + return '' if (('v6_0' in run2d) or ('v6_1' in run2d)): return '' - else: - return '-epoch' + return '-epoch' def fieldgrp(self, filetype, **kwargs): ''' Returns the fieldid group for the BOSS idlspec2d run2d version @@ -1743,14 +1730,12 @@ def fieldgrp(self, filetype, **kwargs): if ('v5' in run2d) or (str(run2d) in ['26','103','104']): return '' - elif ('v6_0' in run2d) or ('v6_1' in run2d): + if ('v6_0' in run2d) or ('v6_1' in run2d): return '' - else: - fieldid = str(fieldid) - if fieldid.isnumeric(): - return '{:0>3d}XXX'.format(int(fieldid) // 1000) - else: - return(fieldid) + fieldid = str(fieldid) + if fieldid.isnumeric(): + return '{:0>3d}XXX'.format(int(fieldid) // 1000) + return fieldid class AccessError(Exception): pass From d39f355f076fe9229f8f8dd3bb718bd2e2ad6aea Mon Sep 17 00:00:00 2001 From: Sean-Morrison Date: Tue, 5 Nov 2024 15:48:18 -0600 Subject: [PATCH 04/14] Update path.py --- python/sdss_access/path/path.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/python/sdss_access/path/path.py b/python/sdss_access/path/path.py index 9c23329..b2265bd 100644 --- a/python/sdss_access/path/path.py +++ b/python/sdss_access/path/path.py @@ -1677,7 +1677,8 @@ def spcoaddobs(self, filetype, **kwargs): if not obs or obs == '': return '' - if (('v6_0' in run2d) or ('v6_1' in run2d)) and obs.lower() == 'apo': + if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or + (('v6_0' in run2d) or ('v6_1' in run2d)) and obs.lower()) == 'apo': return '' if obs == '*': return obs @@ -1699,9 +1700,8 @@ def epochflag(self, filetype, **kwargs): ''' run2d = kwargs.get('run2d', None) - if ('v5' in run2d) or (str(run2d) in ['26','103','104']): - return '' - if (('v6_0' in run2d) or ('v6_1' in run2d)): + if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or + ('v6_0' in run2d) or ('v6_1' in run2d)): return '' return '-epoch' @@ -1728,9 +1728,8 @@ def fieldgrp(self, filetype, **kwargs): if (not fieldid): return '' - if ('v5' in run2d) or (str(run2d) in ['26','103','104']): - return '' - if ('v6_0' in run2d) or ('v6_1' in run2d): + if (('v5' in run2d) or (str(run2d) in ['26','103','104']) of + ('v6_0' in run2d) or ('v6_1' in run2d)): return '' fieldid = str(fieldid) if fieldid.isnumeric(): From 19b4b0a657279a74f6be1661a88af1d695bc1aab Mon Sep 17 00:00:00 2001 From: Brian Cherinka Date: Wed, 6 Nov 2024 10:29:46 -0500 Subject: [PATCH 05/14] Update python/sdss_access/path/path.py --- python/sdss_access/path/path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/sdss_access/path/path.py b/python/sdss_access/path/path.py index b2265bd..87cea01 100644 --- a/python/sdss_access/path/path.py +++ b/python/sdss_access/path/path.py @@ -1590,7 +1590,7 @@ def spcoaddfolder(self, filetype, **kwargs): if filetype.lower() in ['speclite_coadd','specfull_coadd', 'spallfield_coadd','spalllinefield_coadd']: return coaddname - return('fields') + return 'fields' def spcoaddgrp(self, filetype, **kwargs): From c91b7817761adbdec242a187125fda1f85f0100e Mon Sep 17 00:00:00 2001 From: Sean-Morrison Date: Wed, 6 Nov 2024 10:00:54 -0600 Subject: [PATCH 06/14] Update path.py --- python/sdss_access/path/path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/sdss_access/path/path.py b/python/sdss_access/path/path.py index 87cea01..8eda9be 100644 --- a/python/sdss_access/path/path.py +++ b/python/sdss_access/path/path.py @@ -1728,7 +1728,7 @@ def fieldgrp(self, filetype, **kwargs): if (not fieldid): return '' - if (('v5' in run2d) or (str(run2d) in ['26','103','104']) of + if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or ('v6_0' in run2d) or ('v6_1' in run2d)): return '' fieldid = str(fieldid) From 3493054ab98e178c7a6fdbca2b6e4290e3a5da0e Mon Sep 17 00:00:00 2001 From: Sean-Morrison Date: Wed, 6 Nov 2024 10:08:24 -0600 Subject: [PATCH 07/14] Update test_sdss5.py --- tests/path/test_sdss5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/path/test_sdss5.py b/tests/path/test_sdss5.py index 2a46b5b..af11882 100644 --- a/tests/path/test_sdss5.py +++ b/tests/path/test_sdss5.py @@ -77,7 +77,7 @@ def test_apogee_paths(self, path, name, special, keys, exp): 'fieldgrp','spcoaddobs','epochflag','spcoaddgrp']) def test_special_function(self, path, name, special, keys, exp): assert special in path.templates[name] - full = path.full(name, **keys) + full = os.path.normpath(path.full(name, **keys)) assert exp in full @pytest.mark.parametrize('name, keys', [('specLite', ['fieldid', 'catalogid', 'run2d', 'mjd']), From 35a4db733f4c08116c3ccc7bb79aa59a10dedf54 Mon Sep 17 00:00:00 2001 From: Sean-Morrison Date: Wed, 6 Nov 2024 10:12:53 -0600 Subject: [PATCH 08/14] Update path.py --- python/sdss_access/path/path.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/sdss_access/path/path.py b/python/sdss_access/path/path.py index 8eda9be..0ff4285 100644 --- a/python/sdss_access/path/path.py +++ b/python/sdss_access/path/path.py @@ -1640,7 +1640,9 @@ def sptypefolder(self, filetype, **kwargs): if filetype.lower() in ['speclite_epoch','specfull_epoch', 'spallfield_epoch','spalllinefield_epoch']: return 'epoch/spectra' - return 'epoch' + if 'epoch' in filetype.lower(): + return 'epoch' + return '' if filetype.lower() in ['fieldlist_epoch','spall_epoch', 'spall-lite_epoch','spallline_epoch']: return 'summary/epoch' From ed28f917a5a73c71f92fa3629db96630d694a1ee Mon Sep 17 00:00:00 2001 From: Sean-Morrison Date: Wed, 6 Nov 2024 10:33:45 -0600 Subject: [PATCH 09/14] Update path.py --- python/sdss_access/path/path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/sdss_access/path/path.py b/python/sdss_access/path/path.py index 0ff4285..83e67d1 100644 --- a/python/sdss_access/path/path.py +++ b/python/sdss_access/path/path.py @@ -706,7 +706,7 @@ def full(self, filetype, **kwargs): if not skip_tag_check: template = re.sub(r'tags/(v?[0-9._]+)', r'\1', template, count=1) - return self._check_compression(template) + return self._check_compression(os.path.normpath(template)) @staticmethod def check_modules(template, permanent=None): From e9dafa44e9d1665968fa316555537515f91e0ec3 Mon Sep 17 00:00:00 2001 From: Sean-Morrison Date: Wed, 6 Nov 2024 10:33:47 -0600 Subject: [PATCH 10/14] Update conftest.py --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 9eb1585..442df87 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -80,7 +80,7 @@ def expdata(datapath): # full source file location source = os.path.join(base, sas_module, location) # full final file location - destination = os.path.join(os.getenv('SAS_BASE_DIR'), sas_module, location) + destination = os.path.normpath(os.path.join(os.getenv('SAS_BASE_DIR'), sas_module, location)) # combined dict result = {'name': datapath['name'], 'params': datapath['params'], 'base': base, 'sas_module': sas_module, 'location': location, 'source': source, From 3a3974cf7e438e2f8a5f596f75a0a846a44b3e26 Mon Sep 17 00:00:00 2001 From: Sean-Morrison Date: Wed, 6 Nov 2024 10:36:08 -0600 Subject: [PATCH 11/14] Update test_sdss5.py --- tests/path/test_sdss5.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/path/test_sdss5.py b/tests/path/test_sdss5.py index af11882..65201e9 100644 --- a/tests/path/test_sdss5.py +++ b/tests/path/test_sdss5.py @@ -68,9 +68,9 @@ def test_apogee_paths(self, path, name, special, keys, exp): ('spFullsky','@spcoaddobs',{'run2d':'v6_2_0','mjd': '59630','coadd':'allepoch','obs':'apo'}, 'v6_2_0/fields/allepoch/allepoch_apo/spFullsky-allepoch_apo-59630.fits'), ('spAll_epoch','@epochflag',{'run2d':'v6_2_0'}, - 'v6_2_0/summary/epoch/spAll-v6_2_0-epsilon-epoch.fits.gz'), + 'v6_2_0/summary/epoch/spAll-v6_2_0-epoch.fits.gz'), ('spAll_coadd','@spcoaddgrp',{'run2d':'v6_2_0','coadd':'allepoch'}, - 'v6_2_0/summary/test/spAll-v6_2_0-epsilon-allepoch.fits.gz')], + 'v6_2_0/summary/test/spAll-v6_2_0-allepoch.fits.gz')], ids=['configgrp', 'apgprefix-apo', 'apgprefix-lco', 'apgprefix-ins', 'isplate-v6_0_4','pad_fieldid-5','pad_fieldid-6', 'frame-pad', 'frame-nopadp', 'pad_fieldid-*','spcoaddfolder', 'sptypefolder', From fcd1e03af65b07b48a0ecb0dc5a8a24ba4c5c8e2 Mon Sep 17 00:00:00 2001 From: Sean-Morrison Date: Wed, 6 Nov 2024 10:38:33 -0600 Subject: [PATCH 12/14] Update test_sdss5.py --- tests/path/test_sdss5.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/path/test_sdss5.py b/tests/path/test_sdss5.py index 65201e9..91fa252 100644 --- a/tests/path/test_sdss5.py +++ b/tests/path/test_sdss5.py @@ -68,9 +68,9 @@ def test_apogee_paths(self, path, name, special, keys, exp): ('spFullsky','@spcoaddobs',{'run2d':'v6_2_0','mjd': '59630','coadd':'allepoch','obs':'apo'}, 'v6_2_0/fields/allepoch/allepoch_apo/spFullsky-allepoch_apo-59630.fits'), ('spAll_epoch','@epochflag',{'run2d':'v6_2_0'}, - 'v6_2_0/summary/epoch/spAll-v6_2_0-epoch.fits.gz'), + 'v6_2_0/summary/epoch/spAll-v6_2_0-epoch.fits'), ('spAll_coadd','@spcoaddgrp',{'run2d':'v6_2_0','coadd':'allepoch'}, - 'v6_2_0/summary/test/spAll-v6_2_0-allepoch.fits.gz')], + 'v6_2_0/summary/test/spAll-v6_2_0-allepoch.fits')], ids=['configgrp', 'apgprefix-apo', 'apgprefix-lco', 'apgprefix-ins', 'isplate-v6_0_4','pad_fieldid-5','pad_fieldid-6', 'frame-pad', 'frame-nopadp', 'pad_fieldid-*','spcoaddfolder', 'sptypefolder', From 460582f93e841c5182e2caaf16bc80d3a10c682a Mon Sep 17 00:00:00 2001 From: Sean-Morrison Date: Wed, 6 Nov 2024 10:40:25 -0600 Subject: [PATCH 13/14] Update test_sdss5.py --- tests/path/test_sdss5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/path/test_sdss5.py b/tests/path/test_sdss5.py index 91fa252..16d2347 100644 --- a/tests/path/test_sdss5.py +++ b/tests/path/test_sdss5.py @@ -70,7 +70,7 @@ def test_apogee_paths(self, path, name, special, keys, exp): ('spAll_epoch','@epochflag',{'run2d':'v6_2_0'}, 'v6_2_0/summary/epoch/spAll-v6_2_0-epoch.fits'), ('spAll_coadd','@spcoaddgrp',{'run2d':'v6_2_0','coadd':'allepoch'}, - 'v6_2_0/summary/test/spAll-v6_2_0-allepoch.fits')], + 'v6_2_0/summary/allepoch/spAll-v6_2_0-allepoch.fits')], ids=['configgrp', 'apgprefix-apo', 'apgprefix-lco', 'apgprefix-ins', 'isplate-v6_0_4','pad_fieldid-5','pad_fieldid-6', 'frame-pad', 'frame-nopadp', 'pad_fieldid-*','spcoaddfolder', 'sptypefolder', From 6cea8e0dcef411e7f441e820b777de72fd554d65 Mon Sep 17 00:00:00 2001 From: Sean-Morrison Date: Wed, 6 Nov 2024 12:17:43 -0600 Subject: [PATCH 14/14] restore tilegrp --- python/sdss_access/path/path.py | 36 ++++++++++++++++++++++++++++----- tests/path/test_sdss5.py | 7 +++++-- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/python/sdss_access/path/path.py b/python/sdss_access/path/path.py index 83e67d1..3a60cc7 100644 --- a/python/sdss_access/path/path.py +++ b/python/sdss_access/path/path.py @@ -325,15 +325,15 @@ def extract(self, name, example): # handle special functions; perform a drop in replacement if re.match('@spectrodir[|]', template): template = re.sub('@spectrodir[|]', os.environ['BOSS_SPECTRO_REDUX'], template) - elif re.search('@platedir[|]', template): + if re.search('@platedir[|]', template): template = re.sub('@platedir[|]', r'(.*)/{plateid:0>6}', template) - elif re.search('@definitiondir[|]', template): + if re.search('@definitiondir[|]', template): template = re.sub('@definitiondir[|]', '{designid:0>6}', template) - elif re.search('@apgprefix[|]', template): + if re.search('@apgprefix[|]', template): template = re.sub('@apgprefix[|]', '{prefix}', template) - elif re.search('@healpixgrp[|]', template): + if re.search('@healpixgrp[|]', template): template = re.sub('@healpixgrp[|]', '{healpixgrp}', template) - elif re.search('@configgrp[|]', template): + if re.search('@configgrp[|]', template): template = re.sub('@configgrp[|]', '{configgrp}', template) if re.search('@isplate[|]', template): template = re.sub('@isplate[|]', '{isplate}', template) @@ -359,6 +359,8 @@ def extract(self, name, example): template = re.sub('@cat_id_groups[|]', '{cat_id_groups}', template) if re.search('@sdss_id_groups[|]', template): template = re.sub('@sdss_id_groups[|]', '{sdss_id_groups}', template) + if re.search('@tilegrp[|]', template): + template = re.sub('@tilegrp[|]', '{tilegrp}', template) # check if template has any brackets haskwargs = re.search('[{}]', template) @@ -1738,5 +1740,29 @@ def fieldgrp(self, filetype, **kwargs): return '{:0>3d}XXX'.format(int(fieldid) // 1000) return fieldid + def tilegrp(self, filetype, **kwargs): + ''' Returns LVM tile id group subdirectory + + Parameters + ---------- + filetype : str + File type parameter. + tileid : int or str + LVM Tile ID number. Will be converted to int internally. + + Returns + ------- + tileidgrp : str + Tile ID group directory in the format ``NNNNXX``. + + ''' + + tileid = kwargs.get('tileid', None) + if not tileid: + return '0000XX' + elif '*' in str(tileid): + return '{0}XX'.format(tileid) + return '{:0>4d}XX'.format(int(tileid) // 1000) + class AccessError(Exception): pass diff --git a/tests/path/test_sdss5.py b/tests/path/test_sdss5.py index 16d2347..baf05bd 100644 --- a/tests/path/test_sdss5.py +++ b/tests/path/test_sdss5.py @@ -70,11 +70,14 @@ def test_apogee_paths(self, path, name, special, keys, exp): ('spAll_epoch','@epochflag',{'run2d':'v6_2_0'}, 'v6_2_0/summary/epoch/spAll-v6_2_0-epoch.fits'), ('spAll_coadd','@spcoaddgrp',{'run2d':'v6_2_0','coadd':'allepoch'}, - 'v6_2_0/summary/allepoch/spAll-v6_2_0-allepoch.fits')], + 'v6_2_0/summary/allepoch/spAll-v6_2_0-allepoch.fits'), + ('lvm_frame', '@tilegrp', {'drpver': 'master', 'mjd': 60235, + 'tileid': 1055360, 'kind': 'CFrame', 'expnum': 6817}, + '1055XX/1055360/60235/lvmCFrame-00006817.fits')], ids=['configgrp', 'apgprefix-apo', 'apgprefix-lco', 'apgprefix-ins', 'isplate-v6_0_4','pad_fieldid-5','pad_fieldid-6', 'frame-pad', 'frame-nopadp', 'pad_fieldid-*','spcoaddfolder', 'sptypefolder', - 'fieldgrp','spcoaddobs','epochflag','spcoaddgrp']) + 'fieldgrp','spcoaddobs','epochflag','spcoaddgrp','lvm-tileid']) def test_special_function(self, path, name, special, keys, exp): assert special in path.templates[name] full = os.path.normpath(path.full(name, **keys))