update boss paths for v6_2+

python/sdss_access/path/path.py

@@ -325,20 +325,32 @@ def extract(self, name, example):
         # handle special functions; perform a drop in replacement
         if re.match('@spectrodir[|]', template):
             template = re.sub('@spectrodir[|]', os.environ['BOSS_SPECTRO_REDUX'], template)
-        elif re.search('@platedir[|]', template):
+        if re.search('@platedir[|]', template):
             template = re.sub('@platedir[|]', r'(.*)/{plateid:0>6}', template)
-        elif re.search('@definitiondir[|]', template):
+        if re.search('@definitiondir[|]', template):
             template = re.sub('@definitiondir[|]', '{designid:0>6}', template)
-        elif re.search('@apgprefix[|]', template):
+        if re.search('@apgprefix[|]', template):
             template = re.sub('@apgprefix[|]', '{prefix}', template)
-        elif re.search('@healpixgrp[|]', template):
+        if re.search('@healpixgrp[|]', template):
             template = re.sub('@healpixgrp[|]', '{healpixgrp}', template)
-        elif re.search('@configgrp[|]', template):
+        if re.search('@configgrp[|]', template):
             template = re.sub('@configgrp[|]', '{configgrp}', template)
-        elif re.search('@isplate[|]', template):
+        if re.search('@isplate[|]', template):
             template = re.sub('@isplate[|]', '{isplate}', template)
-        elif re.search('@pad_fieldid[|]', template):
+        if re.search('@pad_fieldid[|]', template):
            template = re.sub('@pad_fieldid[|]', '{fieldid}', template)
+        if re.search('@spcoaddfolder[|]',template):
+           template = re.sub('@spcoaddfolder[|]', '{spcoaddfolder}', template)
+        if re.search('@sptypefolder[|]',template):
+           template = re.sub('@sptypefolder[|]', '{sptypefolder}', template)
+        if re.search('@epochflag[|]', template):
+           template = re.sub('@epochflag[|]', '{epochflag}', template)
+        if re.search('@spcoaddobs[|]',template):
+           template = re.sub('@spcoaddobs[|]', '{obs}', template)
+        if re.search('@spcoaddgrp[|]',template):
+           template = re.sub('@spcoaddgrp[|]', '{spcoaddgrp}', template)
+        if re.search('@fieldgrp[|]',template):
+           template = re.sub('@fieldgrp[|]', '{fieldgrp}', template)
         if re.search('@plateid6[|]', template):
             template = re.sub('@plateid6[|]', '{plateid:0>6}', template)
         if re.search('@component_default[|]', template):
@@ -347,6 +359,8 @@ def extract(self, name, example):
             template = re.sub('@cat_id_groups[|]', '{cat_id_groups}', template)
         if re.search('@sdss_id_groups[|]', template):
             template = re.sub('@sdss_id_groups[|]', '{sdss_id_groups}', template)
+        if re.search('@tilegrp[|]', template):
+            template = re.sub('@tilegrp[|]', '{tilegrp}', template)
 
         # check if template has any brackets
         haskwargs = re.search('[{}]', template)
@@ -386,12 +400,28 @@ def extract(self, name, example):
                             drval = re.match('^DR[1-9][0-9]', value).group(0)
                             otherval = value.split(drval)[-1]
                             pdict = {keys[0]: drval, keys[1]: otherval}
+                        elif keys == ['fieldid','isplate']:
+                            # for {fieldid}{isplate} as isplate is calculated automatically
+                            if value.endswith('p') and value[:-1].isdigit():
+                                value = value[:-1]
+                            pdict = {keys[0]:value}
+                        elif keys == ['run2d','epochflag']:
+                            # for {run2d}{epochflag} as epochflag is calculated automatically
+                            pdict = {keys[0]:value.replace('-epoch','')}
+                        elif keys == ['coadd', 'obs']:
+                            value = value.split('_')
+                            if len(value) == 1:
+                                value.append('')
+                            pdict = {keys[0]: value[0], keys[1]: value[1]}
                         elif keys[0] in ['rc', 'br', 'filter', 'camrow']:
                             # for {camrow}{camcol}, {filter}{camcol}, {br}{id}, etc
                             pdict = {keys[0]: value[0], keys[1]: value[1:]}
                         else:
                             raise ValueError('This case has not yet been accounted for.')
                         path_dict.update(pdict)
+                    elif keys[0] in ['sptypefolder','fieldgrp','spcoaddfolder','spcoaddgrp']:
+                        # supress the keys since they are automatically calculated
+                        continue
                     else:
                         path_dict[keys[0]] = value
         return path_dict
@@ -678,7 +708,7 @@ def full(self, filetype, **kwargs):
         if not skip_tag_check:
             template = re.sub(r'tags/(v?[0-9._]+)', r'\1', template, count=1)
 
-        return self._check_compression(template)
+        return self._check_compression(os.path.normpath(template))
 
     @staticmethod
     def check_modules(template, permanent=None):
@@ -1531,8 +1561,184 @@ def pad_fieldid(self, filetype, **kwargs):
             return str(fieldid)
         if fieldid.isnumeric():
             return str(fieldid).zfill(6)
-        else:
-            return fieldid
+        return fieldid
+
+    def spcoaddfolder(self, filetype, **kwargs):
+        ''' Returns the reorganized subfolder structure for the BOSS idlspec2d run2d version
+
+        Parameters
+        ---------
+        filetype : str
+            File type parameter
+        run2d : str
+            BOSS idlspec2d run2d version
+        coadd : str
+            Name of the custom coadd schema
+        Returns
+        -------
+        sptypefolder : str
+        '''
+
+        run2d = kwargs.get('run2d', None)
+        coaddname = kwargs.get('coadd', None)
+
+        if (not run2d):
+            return ''
+        if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or
+            ('v6_0' in run2d) or ('v6_1' in run2d)):
+            return ''
+        if filetype.lower() in ['spall_coadd','spall-lite_coadd','spallline_coadd']:
+            return 'summary'
+        if filetype.lower() in ['speclite_coadd','specfull_coadd',
+                                'spallfield_coadd','spalllinefield_coadd']:
+            return coaddname
+        return 'fields'
+
+
+    def spcoaddgrp(self, filetype, **kwargs):
+        ''' Returns the coadd group (field group analog) subfolder structure for the BOSS idlspec2d run2d version
+
+        Parameters
+        ---------
+        filetype : str
+            File type parameter
+        run2d : str
+            BOSS idlspec2d run2d version
+        coadd : str
+            Name of the custom coadd schema
+        Returns
+        -------
+        spcoaddgrp : str
+        '''
+        run2d = kwargs.get('run2d', None)
+        coaddname = kwargs.get('coadd', None)
+
+        if (not run2d):
+            return ''
+        if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or
+            ('v6_0' in run2d) or ('v6_1' in run2d)):
+            return ''
+        return coaddname
+
+    def sptypefolder(self, filetype, **kwargs):
+        ''' Returns the reorganized subfolder structure for the BOSS idlspec2d run2d version
+
+        Parameters
+        ---------
+        filetype : str
+            File type parameter
+        run2d : str
+            BOSS idlspec2d run2d version
+        Returns
+        -------
+        sptypefolder : str
+        '''
+
+        run2d = kwargs.get('run2d', None)
+
+        if (not run2d) or ('v5' in run2d) or (str(run2d) in ['26','103','104']):
+            return ''
+        if ('v6_0' in run2d) or ('v6_1' in run2d):
+            if filetype.lower() in ['speclite_epoch','specfull_epoch',
+                                    'spallfield_epoch','spalllinefield_epoch']:
+                return 'epoch/spectra'
+            if 'epoch' in filetype.lower():
+                return 'epoch'
+            return ''
+        if filetype.lower() in ['fieldlist_epoch','spall_epoch',
+                                'spall-lite_epoch','spallline_epoch']:
+            return 'summary/epoch'
+        if filetype.lower() in ['speclite_epoch','specfull_epoch',
+                                'spallfield_epoch','spalllinefield_epoch']:
+            return 'spectra/epoch'
+        if filetype.lower() in ['conflist','fieldlist','spall',
+                                'spall-lite','spallline']:
+            return 'summary/daily'
+        if filetype.lower() in ['speclite','specfull',
+                                'spallfield','spalllinefield']:
+            return 'daily'
+        return 'fields'
+
+    def spcoaddobs(self, filetype, **kwargs):
+        ''' Returns the formatted observatory flag for custom coadds for the BOSS idlspec2d
+
+        Parameters
+        ----------
+        filetype : str
+            File type parameter
+        run2d : str
+            BOSS idlspec2d run2d version
+        obs : str
+            Observatory of observations: LCO, APO, ''(for merged)
+
+        Returns
+        -------
+        obs : str
+        '''
+
+        obs = kwargs.get('obs', None)
+        run2d = kwargs.get('run2d', None)
+
+        if not obs or obs == '':
+            return ''
+        if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or
+            (('v6_0' in run2d) or ('v6_1' in run2d)) and obs.lower()) == 'apo':
+            return ''
+        if obs == '*':
+            return obs
+        return '_{}'.format(obs.lower())
+
+    def epochflag(self, filetype, **kwargs):
+        ''' Returns the flag for epoch coadds for the BOSS idlspec2d
+
+        Parameters
+        ----------
+        filetype : str
+            File type parameter
+        run2d : str
+            BOSS idlspec2d run2d version
+
+        Returns
+        -------
+        epochflag : str
+        '''
+
+        run2d = kwargs.get('run2d', None)
+        if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or
+            ('v6_0' in run2d) or ('v6_1' in run2d)):
+            return ''
+        return '-epoch'
+
+    def fieldgrp(self, filetype, **kwargs):
+        ''' Returns the fieldid group for the BOSS idlspec2d run2d version
+
+        Parameters
+        ---------
+        filetype : str
+            File type parameter
+        run2d : str
+            BOSS idlspec2d run2d version
+        fieldid : str or int
+            Field ID number. Will be converted to str internally.
+
+        Returns
+        -------
+        fieldgrp : str
+        '''
+
+        fieldid = kwargs.get('fieldid', None)
+        run2d = kwargs.get('run2d', None)
+
+        if (not fieldid):
+            return ''
+
+        if (('v5' in run2d) or (str(run2d) in ['26','103','104']) or
+            ('v6_0' in run2d) or ('v6_1' in run2d)):
+            return ''
+        fieldid = str(fieldid)
+        if fieldid.isnumeric():
+            return '{:0>3d}XXX'.format(int(fieldid) // 1000)
+        return fieldid
 
     def tilegrp(self, filetype, **kwargs):
         ''' Returns LVM tile id group subdirectory

tests/conftest.py

@@ -80,7 +80,7 @@ def expdata(datapath):
     # full source file location
     source = os.path.join(base, sas_module, location)
     # full final file location
-    destination = os.path.join(os.getenv('SAS_BASE_DIR'), sas_module, location)
+    destination = os.path.normpath(os.path.join(os.getenv('SAS_BASE_DIR'), sas_module, location))
     # combined dict
     result = {'name': datapath['name'], 'params': datapath['params'], 'base': base,
               'sas_module': sas_module, 'location': location, 'source': source,

tests/path/test_sdss5.py

@@ -59,15 +59,28 @@ def test_apogee_paths(self, path, name, special, keys, exp):
                                 'v6_0_4/1234p/spFrame-b1-00005432.fits.gz'),
                               ('spField', '@pad_fieldid', {'run2d': 'v6_1_1', 'mjd': '59630', 'fieldid': '*'},
                                 'v6_1_1/*/spField-*-59630.fits'),
+                              ('spCOADDS','@spcoaddfolder',{'run2d':'v6_2_0','survey':'BHM','coadd':'allepoch','phase':'SDSSV'},
+                                'v6_2_0/fields/SDSSV_BHM_COADDS.par'),
+                              ('spField','@sptypefolder',{'run2d':'v6_2_0','mjd': '59630', 'fieldid': '123456'},
+                                'v6_2_0/fields/123XXX/123456/spField-123456-59630.fits'),
+                              ('spField','@fieldgrp',{'run2d':'v6_2_0','mjd': '59630', 'fieldid': '123456'},
+                                'v6_2_0/fields/123XXX/123456/spField-123456-59630.fits'),
+                              ('spFullsky','@spcoaddobs',{'run2d':'v6_2_0','mjd': '59630','coadd':'allepoch','obs':'apo'},
+                                'v6_2_0/fields/allepoch/allepoch_apo/spFullsky-allepoch_apo-59630.fits'),
+                              ('spAll_epoch','@epochflag',{'run2d':'v6_2_0'},
+                                'v6_2_0/summary/epoch/spAll-v6_2_0-epoch.fits'),
+                              ('spAll_coadd','@spcoaddgrp',{'run2d':'v6_2_0','coadd':'allepoch'},
+                                'v6_2_0/summary/allepoch/spAll-v6_2_0-allepoch.fits'),
                               ('lvm_frame', '@tilegrp', {'drpver': 'master', 'mjd': 60235,
                                                          'tileid': 1055360, 'kind': 'CFrame', 'expnum': 6817},
                                '1055XX/1055360/60235/lvmCFrame-00006817.fits')],
                              ids=['configgrp', 'apgprefix-apo', 'apgprefix-lco', 'apgprefix-ins',
-                                  'isplate-v6_0_4','pad_fieldid-5','pad_fieldid-6', 'frame-pad', 'frame-nopadp',
-                                  'pad_fieldid-*', 'lvm-tileid'])
+                                  'isplate-v6_0_4','pad_fieldid-5','pad_fieldid-6', 'frame-pad',
+                                  'frame-nopadp', 'pad_fieldid-*','spcoaddfolder', 'sptypefolder',
+                                  'fieldgrp','spcoaddobs','epochflag','spcoaddgrp','lvm-tileid'])
     def test_special_function(self, path, name, special, keys, exp):
         assert special in path.templates[name]
-        full = path.full(name, **keys)
+        full = os.path.normpath(path.full(name, **keys))
         assert exp in full
 
     @pytest.mark.parametrize('name, keys', [('specLite', ['fieldid', 'catalogid', 'run2d', 'mjd']),