Adding routes for accessing mocs

.github/workflows/build.yml

@@ -32,8 +32,8 @@ jobs:
 
     - name: Install dependencies
       run: |
-        poetry install
-      if: steps.cache.outputs.cache-hit != 'true' 
+        poetry install  # --no-interaction --no-ansi -vvv  # add this for local debugs with "act"
+      if: steps.cache.outputs.cache-hit != 'true'
 
     - name: Test with pytest
       run: |

pyproject.toml

@@ -30,7 +30,7 @@ valis = "valis.cli:main"
 
 [tool.poetry.dependencies]
 python = "^3.8"
-sdss-tree = "^3.1.4"
+sdss-tree = "^3.1.6"
 sdss-access = "^2.0.5"
 sdsstools = "^0.4.14"
 Sphinx = {version="^3.0.0", optional=true}

python/valis/main.py

@@ -11,17 +11,21 @@
 # Modified By: Brian Cherinka
 
 
-from __future__ import print_function, division, absolute_import
-from fastapi import FastAPI, Depends
-from fastapi.openapi.utils import get_openapi
-from fastapi.middleware.cors import CORSMiddleware
+from __future__ import absolute_import, division, print_function
+
+import os
 from typing import Dict
 
+from fastapi import Depends, FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.openapi.utils import get_openapi
+from fastapi.staticfiles import StaticFiles
+
 import valis
-from valis.settings import settings
-from valis.routes import access, envs, files, auth, info, maskbits, target
-from valis.routes.base import release
+from valis.routes import access, auth, envs, files, info, maskbits, mocs, target
 from valis.routes.auth import set_auth
+from valis.routes.base import release
+from valis.settings import settings
 
 
 tags_metadata = [
@@ -65,6 +69,10 @@
         "name": "maskbits",
         "description": "Work with SDSS maskbits",
     },
+    {
+        "name": "mocs",
+        "description": "Access SDSS surveys MOCs",
+    },
 ]
 
 # create the application
@@ -77,6 +85,7 @@
 app.add_middleware(CORSMiddleware, allow_origin_regex="^https://.*\.sdss\.(org|utah\.edu)$",
                    allow_credentials=True, allow_methods=['*'], allow_headers=['*'])
 
+app.mount("/static/mocs", StaticFiles(directory=os.getenv("SDSS_HIPS"), html=True, follow_symlink=True), name="static")
 
 @app.get("/", summary='Hello World route', response_model=Dict[str, str])
 def hello(release = Depends(release)):
@@ -89,6 +98,7 @@ def hello(release = Depends(release)):
 app.include_router(auth.router, prefix='/auth', tags=['auth'])
 app.include_router(target.router, prefix='/target', tags=['target'])
 app.include_router(maskbits.router, prefix='/maskbits', tags=['maskbits'])
+app.include_router(mocs.router, prefix='/mocs', tags=['mocs'])
 
 
 def custom_openapi():

python/valis/routes/base.py

@@ -44,4 +44,13 @@ class Base:
     """ Base class for all API routes"""
     release: str = Depends(release)
     tree: Tree = Depends(get_tree)
-    path: Path = Depends(get_access)
+    path: Path = Depends(get_access)
+
+    def check_path_name(self, name: str):
+        names = self.path.lookup_names()
+        if name not in names:
+            raise HTTPException(status_code=422, detail=f'path name {name} not in release.')
+
+    def check_path_exists(self, path: str):
+        if not self.path.exists('', full=path):
+            raise HTTPException(status_code=422, detail=f'path {path} does not exist on disk.')

python/valis/routes/mocs.py

@@ -0,0 +1,79 @@
+
+# !/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+
+import orjson
+import pathlib
+from typing import List, Dict
+from pydantic import BaseModel, Field
+from fastapi import APIRouter, HTTPException, Query
+from fastapi_utils.cbv import cbv
+from fastapi.responses import FileResponse, RedirectResponse
+from valis.routes.base import Base
+from valis.routes.files import ORJSONResponseCustom
+
+from sdss_access.path import Path
+
+def read_json(path):
+    with open(path, 'r') as f:
+        lines = f.readlines()
+        first = lines[0]
+        mocorder = int(first.split('\n')[0].split('#MOCORDER ')[-1])
+        data = orjson.loads("\n".join(lines[1:]))
+        return {'order': mocorder, 'moc': data}
+
+
+class MocModel(BaseModel):
+    """ Model representing the output Moc.json file from Hipsgen-cat """
+    order: int = Field(..., description='the depth of the MOC')
+    moc: Dict[str, List[int]] = Field(..., description='the MOC data')
+
+
+router = APIRouter()
+@cbv(router)
+class Mocs(Base):
+    """ Endpoints for interacting with SDSS MOCs """
+    name: str = 'sdss_moc'
+
+    def check_path_name(self, path, name: str):
+        """ temp function until sort out directory org for """
+        names = path.lookup_names()
+        if name not in names:
+            raise HTTPException(status_code=422, detail=f'path name {name} not in release.')
+
+    def check_path_exists(self, spath, path: str):
+        """ temp function until sort out directory org for """
+        if not spath.exists('', full=path):
+            raise HTTPException(status_code=422, detail=f'path {path} does not exist on disk.')
+
+    @router.get('/preview', summary='Preview an individual survey MOC', response_class=RedirectResponse)
+    async def get_moc(self, survey: str):
+        """ Preview an individual survey MOC """
+        return f'/static/mocs/{self.release.lower()}/{survey}/'
+
+    @router.get('/json', summary='Get the MOC file in JSON format')
+    async def get_json(self, survey: str = Query(..., description='The SDSS survey name', example='manga')) -> MocModel:
+        """ Get the MOC file in JSON format """
+        # temporarily affixing the access path to sdss5 sandbox until
+        # we decide on real org for DRs, etc
+        spath = Path(release='sdss5')
+
+        self.check_path_name(spath, self.name)
+        path = spath.full(self.name, release=self.release.lower(), survey=survey, ext='json')
+        self.check_path_exists(spath, path)
+        return ORJSONResponseCustom(content=read_json(path), option=orjson.OPT_SERIALIZE_NUMPY)
+
+    @router.get('/fits', summary='Download the MOC file in FITs format')
+    async def get_fits(self, survey: str = Query(..., description='The SDSS survey name', example='manga')):
+        """ Download the MOC file in FITs format """
+        # temporarily affixing the access path to sdss5 sandbox
+        # we decide on real org for DRs, etc
+        spath = Path(release='sdss5')
+
+        self.check_path_name(spath, self.name)
+        path = spath.full(self.name, release=self.release.lower(), survey=survey.lower(), ext='fits')
+        self.check_path_exists(spath, path)
+        pp = pathlib.Path(path)
+        name = f'{survey.lower()}_{pp.name}'
+        return FileResponse(path, filename=name, media_type='application/fits')

tests/conftest.py

@@ -66,20 +66,26 @@ def create_fits(name = 'testfile.fits'):
 def setup_sas(monkeypatch, tmp_path, mocker):
     """ fixtures that sets up temp SAS and a TEST_REDUX directory """
     sas_dir = tmp_path / 'sas'
+
+    # create dirs for test
     test_dir = sas_dir / 'dr17' / 'test/spectro/redux'
     test_dir.mkdir(parents=True)
     monkeypatch.setenv("SAS_BASE_DIR", str(sas_dir))
     monkeypatch.setenv("TEST_REDUX", str(test_dir))
 
+    # create dirs for mocs
+    moc_dir = sas_dir / 'sdsswork/sandbox/hips'
+    moc_dir.mkdir(parents=True)
+    monkeypatch.setenv("SDSS_HIPS", str(moc_dir))
+
     mocker.patch('valis.routes.base.Tree', new=MockTree)
     mocker.patch('valis.routes.base.Path', new=MockPath)
     #mocker.patch('datamodel.validate.check.Tree', new=MockTree)
 
 
 @pytest.fixture()
 def testfile(setup_sas):
-        # write out the file to the designated path
-
+    # write out the file to the designated path
     name = 'testfile_A.fits'
     hdu = create_fits(name)
 
@@ -92,6 +98,17 @@ def testfile(setup_sas):
     return path
 
 
+@pytest.fixture()
+def testmoc(setup_sas):
+    moc = os.getenv("SDSS_HIPS", "")
+    path = pathlib.Path(moc) / 'dr17/manga/Moc.json'
+    if not path.exists():
+       path.parent.mkdir(parents=True, exist_ok=True)
+
+    with open(path, 'w') as f:
+        f.write("#MOCORDER 10\n")
+        f.write("""{"9":[224407,224413,664253,664290,664292]}\n""")
+
 class MockTree(Tree):
     """ mock out the Tree class to insert test file """
 
@@ -105,6 +122,7 @@ def _create_environment(self, cfg=None, sections=None):
     def _create_paths(self, cfg=None):
         paths = super(MockTree, self)._create_paths(cfg=cfg)
         paths.update({'test': '$TEST_REDUX/{ver}/testfile_{id}.fits'})
+        paths.update({'sdss_moc': '$SDSS_HIPS/{release}/{survey}/Moc.{ext}'})
         return paths
 
 class MockPath(Path):

tests/test_mocs.py

@@ -0,0 +1,15 @@
+# encoding: utf-8
+#
+
+
+def get_data(response):
+    assert response.status_code == 200
+    return response.json()
+
+
+def test_moc_json(client, testmoc):
+    response = client.get("/mocs/json?survey=manga&release=DR17")
+    data = get_data(response)
+
+    assert data['order'] == 10
+    assert data['moc']['9'] == [224407,224413,664253,664290,664292]