Merge pull request #786 from open-rmf/fix/split-cache-and-static

Support offline doc generation (#784)

.prettierignore

@@ -0,0 +1,2 @@
+packages/api-server/api_server/static/
+

packages/api-server/api_server/app.py

@@ -7,6 +7,11 @@
 import schedule
 from fastapi import Depends
 from fastapi.middleware.cors import CORSMiddleware
+from fastapi.openapi.docs import (
+    get_redoc_html,
+    get_swagger_ui_html,
+    get_swagger_ui_oauth2_redirect_html,
+)
 from fastapi.staticfiles import StaticFiles
 from tortoise import Tortoise
 
@@ -27,7 +32,7 @@
     User,
 )
 from .models import tortoise_models as ttm
-from .repositories import StaticFilesRepository, TaskRepository
+from .repositories import TaskRepository
 from .rmf_io import HealthWatchdog, RmfBookKeeper, rmf_events
 from .types import is_coroutine
 
@@ -46,7 +51,12 @@ async def on_sio_connect(sid: str, _environ: dict, auth: Optional[dict] = None):
         return False
 
 
-app = FastIO(title="RMF API Server", socketio_connect=on_sio_connect)
+app = FastIO(
+    title="RMF API Server",
+    socketio_connect=on_sio_connect,
+    docs_url=None,
+    redoc_url=None,
+)
 
 
 app.add_middleware(
@@ -56,22 +66,25 @@ async def on_sio_connect(sid: str, _environ: dict, auth: Optional[dict] = None):
     allow_methods=["*"],
     allow_headers=["*"],
 )
-os.makedirs(app_config.static_directory, exist_ok=True)
+
 app.mount(
     "/static",
-    StaticFiles(directory=app_config.static_directory),
+    StaticFiles(
+        directory=os.path.join(os.path.dirname(os.path.abspath(__file__)), "static")
+    ),
     name="static",
 )
 
+os.makedirs(app_config.cache_directory, exist_ok=True)
+app.mount(
+    "/cache",
+    StaticFiles(directory=app_config.cache_directory),
+    name="cache",
+)
+
 # will be called in reverse order on app shutdown
 shutdown_cbs: List[Union[Coroutine[Any, Any, Any], Callable[[], None]]] = []
 
-static_files_repo = StaticFilesRepository(
-    f"{app_config.public_url.geturl()}/static",
-    app_config.static_directory,
-    logger.getChild("static_files"),
-)
-
 rmf_bookkeeper = RmfBookKeeper(rmf_events, logger=logger.getChild("BookKeeper"))
 
 app.include_router(routes.main_router)
@@ -202,6 +215,33 @@ async def on_shutdown():
     logger.info("shutdown app")
 
 
+@app.get("/docs", include_in_schema=False)
+async def custom_swagger_ui_html():
+    openapi_url = app.openapi_url if app.openapi_url is not None else "/openapi.json"
+    return get_swagger_ui_html(
+        openapi_url=openapi_url,
+        title=app.title + " - Swagger UI",
+        oauth2_redirect_url=app.swagger_ui_oauth2_redirect_url,
+        swagger_js_url="/static/swagger-ui-bundle.js",
+        swagger_css_url="/static/swagger-ui.css",
+    )
+
+
+@app.get(app.swagger_ui_oauth2_redirect_url, include_in_schema=False)
+async def swagger_ui_redirect():
+    return get_swagger_ui_oauth2_redirect_html()
+
+
+@app.get("/redoc", include_in_schema=False)
+async def redoc_html():
+    openapi_url = app.openapi_url if app.openapi_url is not None else "/openapi.json"
+    return get_redoc_html(
+        openapi_url=openapi_url,
+        title=app.title + " - ReDoc",
+        redoc_js_url="/static/redoc.standalone.js",
+    )
+
+
 async def _spin_scheduler():
     while True:
         schedule.run_pending()

packages/api-server/api_server/app_config.py

@@ -13,7 +13,7 @@ class AppConfig:
     port: int
     db_url: str
     public_url: urllib.parse.ParseResult
-    static_directory: str
+    cache_directory: str
     log_level: str
     builtin_admin: str
     jwt_public_key: Optional[str]

packages/api-server/api_server/default_config.py

@@ -10,7 +10,7 @@
     # When being a proxy, this must be the url that rmf-server is mounted on.
     # E.g. https://example.com/rmf/api/v1
     "public_url": "http://localhost:8000",
-    "static_directory": "static",  # The directory where static files should be stored.
+    "cache_directory": "run/cache",  # The directory where cached files should be stored.
     "log_level": "WARNING",  # https://docs.python.org/3.8/library/logging.html#levels
     # a user that is automatically given admin privileges, note that this does not guarantee that the user exists in the identity provider.
     "builtin_admin": "admin",

packages/api-server/api_server/fast_io/__init__.py

@@ -154,6 +154,8 @@ def __init__(
         **kwargs,
     ):
         super().__init__(*args, **kwargs)
+        if self.swagger_ui_oauth2_redirect_url is None:
+            self.swagger_ui_oauth2_redirect_url = "docs/oauth2-redirect"
         self.sio = socketio.AsyncServer(
             async_mode="asgi", cors_allowed_origins=[], serializer=FastIOPacket
         )

packages/api-server/api_server/gateway.py

@@ -28,18 +28,18 @@
 
 from .logger import logger as base_logger
 from .models import BuildingMap, DispenserState, DoorState, IngestorState, LiftState
-from .repositories import StaticFilesRepository, static_files_repo
+from .repositories import CachedFilesRepository, cached_files_repo
 from .rmf_io import rmf_events
 from .ros import ros_node
 
 
 def process_building_map(
     rmf_building_map: RmfBuildingMap,
-    static_files: StaticFilesRepository,
+    cached_files: CachedFilesRepository,
 ) -> BuildingMap:
     """
     1. Converts a `BuildingMap` message to an ordered dict.
-    2. Saves the images into `{static_directory}/{map_name}/`.
+    2. Saves the images into `{cache_directory}/{map_name}/`.
     3. Change the `AffineImage` `data` field to the url of the image.
     """
     processed_map = message_to_ordereddict(rmf_building_map)
@@ -53,15 +53,15 @@ def process_building_map(
             sha1_hash.update(image.data)
             fingerprint = base64.b32encode(sha1_hash.digest()).lower().decode()
             relpath = f"{rmf_building_map.name}/{level.name}-{image.name}.{fingerprint}.{image.encoding}"  # pylint: disable=line-too-long
-            urlpath = static_files.add_file(image.data, relpath)
+            urlpath = cached_files.add_file(image.data, relpath)
             processed_map["levels"][i]["images"][j]["data"] = urlpath
     return BuildingMap(**processed_map)
 
 
 class RmfGateway:
     def __init__(
         self,
-        static_files: StaticFilesRepository,
+        cached_files: CachedFilesRepository,
         *,
         logger: Optional[logging.Logger] = None,
     ):
@@ -74,7 +74,7 @@ def __init__(
         self._submit_task_srv = ros_node().create_client(RmfSubmitTask, "submit_task")
         self._cancel_task_srv = ros_node().create_client(RmfCancelTask, "cancel_task")
 
-        self.static_files = static_files
+        self.cached_files = cached_files
         self.logger = logger or base_logger.getChild(self.__class__.__name__)
         self._subscriptions: List[Subscription] = []
 
@@ -135,7 +135,7 @@ def convert_lift_state(lift_state: RmfLiftState):
             RmfBuildingMap,
             "map",
             lambda msg: rmf_events.building_map.on_next(
-                process_building_map(msg, self.static_files)
+                process_building_map(msg, self.cached_files)
             ),
             rclpy.qos.QoSProfile(
                 history=rclpy.qos.HistoryPolicy.KEEP_ALL,
@@ -191,5 +191,5 @@ def startup():
     Must be called after the ros node is created and before spinning the it.
     """
     global _rmf_gateway
-    _rmf_gateway = RmfGateway(static_files_repo)
+    _rmf_gateway = RmfGateway(cached_files_repo)
     return _rmf_gateway

packages/api-server/api_server/repositories/__init__.py

@@ -1,5 +1,5 @@
 from .alerts import AlertRepository, alert_repo_dep
+from .cached_files import CachedFilesRepository, cached_files_repo
 from .fleets import FleetRepository, fleet_repo_dep
 from .rmf import RmfRepository, rmf_repo_dep
-from .static_files import StaticFilesRepository, static_files_repo
 from .tasks import TaskRepository, task_repo_dep

packages/api-server/api_server/repositories/static_files.py → packages/api-server/api_server/repositories/cached_files.py

@@ -6,18 +6,18 @@
 from api_server.logger import logger as _logger
 
 
-class StaticFilesRepository:
+class CachedFilesRepository:
     def __init__(
         self,
         base_url: str,
         directory: str,
         logger: Optional[logging.Logger] = None,
     ):
         """
-        :param base_url: base url that static files are served from. When running behind a proxy,
+        :param base_url: base url that cached files are served from. When running behind a proxy,
         this should be the url of the proxy.
         :param directory: location to write the files to.
-        This should be the same directory that a static files server is serving from.
+        This should be the same directory that the cached files server is serving from.
         """
         self.base_url = base_url
         self.directory = directory
@@ -40,8 +40,9 @@ def add_file(self, data: bytes, path: str) -> str:
         return urlpath
 
 
-static_files_repo = StaticFilesRepository(
-    f"{app_config.public_url.geturl()}/static",
-    app_config.static_directory,
-    _logger.getChild("static_files"),
+os.makedirs(app_config.cache_directory, exist_ok=True)
+cached_files_repo = CachedFilesRepository(
+    f"{app_config.public_url.geturl()}/cache",
+    app_config.cache_directory,
+    _logger.getChild("cached_files"),
 )

packages/api-server/api_server/repositories/test_static_files.py → packages/api-server/api_server/repositories/test_cached_files.py

@@ -2,21 +2,21 @@
 import unittest
 from os.path import dirname
 
-from .static_files import StaticFilesRepository
+from .cached_files import CachedFilesRepository
 
 
-class TestStaticFilesRepository(unittest.TestCase):
+class TestCachedFilesRepository(unittest.TestCase):
     def setUp(self):
         self.artifact_dir = f"{dirname(dirname(dirname(__file__)))}/test_artifacts"
-        self.repo = StaticFilesRepository("/static", self.artifact_dir)
+        self.repo = CachedFilesRepository("/cache", self.artifact_dir)
 
     def test_add_file(self):
         """
         test file is saved in the correct location and it returns the correct url
         """
-        target_path = "TestStaticFilesRepository/test_add_file.txt"
+        target_path = "TestCachedFilesRepository/test_add_file.txt"
         url_path = self.repo.add_file(b"hello", target_path)
-        self.assertEqual(url_path, f"/static/{target_path}")
+        self.assertEqual(url_path, f"/cache/{target_path}")
         saved_path = f"{self.artifact_dir}/{target_path}"
         self.assertTrue(os.path.exists(saved_path))
         with open(saved_path, "br") as f:

packages/api-server/package.json

@@ -6,8 +6,8 @@
   "scripts": {
     "prepack": "../../scripts/pipenv run python setup.py bdist_wheel",
     "restart": "RMF_API_SERVER_CONFIG=sqlite_local_config.py ../../scripts/pipenv run python -m api_server",
-    "start": "rm -rf run && mkdir -p run && RMF_API_SERVER_CONFIG=sqlite_local_config.py ../../scripts/pipenv run python -m api_server",
-    "start:psql": "rm -rf run && mkdir -p run && RMF_API_SERVER_CONFIG=psql_local_config.py ../../scripts/pipenv run python -m api_server",
+    "start": "rm -rf run && mkdir -p run/cache && RMF_API_SERVER_CONFIG=sqlite_local_config.py ../../scripts/pipenv run python -m api_server",
+    "start:psql": "rm -rf run && mkdir -p run/cache && RMF_API_SERVER_CONFIG=psql_local_config.py ../../scripts/pipenv run python -m api_server",
     "test": "../../scripts/pipenv run python scripts/test.py",
     "test:cov": "../../scripts/pipenv run python -m coverage run scripts/test.py",
     "test:report": "../../scripts/pipenv run python -m coverage html && xdg-open htmlcov/index.html",

packages/api-server/psql_local_config.py

@@ -1,3 +1,11 @@
 from sqlite_local_config import config
 
-config.update({"db_url": "postgres://postgres:postgres@127.0.0.1:5432"})
+here = dirname(__file__)
+run_dir = f"{here}/run"
+
+config.update(
+    {
+        "db_url": "postgres://postgres:postgres@127.0.0.1:5432",
+        "cache_directory": f"{run_dir}/cache",  # The directory where cached files should be stored.
+    }
+)

packages/api-server/setup.py

@@ -38,5 +38,8 @@
             "rmf_api_server=api_server.__main__:main",
         ],
     },
+    package_data={
+        "api_server.static": ["*.js", "*.css"],
+    },
     license="Apache License, Version 2.0",
 )

packages/api-server/sqlite_local_config.py

@@ -8,7 +8,7 @@
 config.update(
     {
         "db_url": f"sqlite://{run_dir}/db.sqlite3",
-        "static_directory": f"{run_dir}/static",  # The directory where static files should be stored.
+        "cache_directory": f"{run_dir}/cache",  # The directory where cached files should be stored.
         "ros_args": ["-p", "use_sim_time:=true"],
     }
 )