Feature/browser db

.github/workflows/test-ci.yml

@@ -50,6 +50,9 @@ jobs:
           python-version: ${{ env.PYTHON_VERSION }}
           cache: 'pip'
 
+      - name: Build browse-ocrd Docker image
+        run: make build-browse-ocrd-docker
+
       - name: Install dependencies using pip
         run: pip install -e ".[dev]"
 

.gitignore

@@ -2,5 +2,5 @@
 authorized_keys
 __pycache__/
 .python-version
-.pdm.toml
+.pdm-python
 .pdm.lock

Makefile

@@ -10,6 +10,11 @@ build:
 pull:
 	docker pull $(TAGNAME)
 
+
+build-browse-ocrd-docker:
+	docker build -t ocrd-browser:latest -f docker-browse-ocrd/Dockerfile docker-browse-ocrd
+
+
 define HELP
 cat <<"EOF"
 Targets:

docker-browse-ocrd/Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.7
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends libcairo2-dev libgtk-3-bin libgtk-3-dev libglib2.0-dev libgtksourceview-3.0-dev libgirepository1.0-dev gir1.2-webkit2-4.0 pkg-config cmake \
+    && pip3 install -U setuptools \
+    && pip3 install browse-ocrd
+
+ENV GDK_BACKEND broadway
+ENV BROADWAY_DISPLAY :5
+
+EXPOSE 8085
+
+COPY init.sh /init.sh
+
+RUN chmod +x /init.sh
+
+CMD ["/init.sh"]

docker-browse-ocrd/init.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+set -x
+nohup broadwayd :5 &
+browse-ocrd /data/mets.xml

docker-compose.yml

@@ -15,6 +15,7 @@ services:
     environment:
       MONITOR_PORT_LOG: ${MONITOR_PORT_LOG}
       CONTROLLER: "${CONTROLLER_HOST}:${CONTROLLER_PORT_SSH}"
+      DB_CONNECTION: "mongodb://${DB_ROOT_USER:-root}:${DB_ROOT_PASSWORD:-root_password}@ocrd-database:27017"
 
     ports:
       - ${MONITOR_PORT_WEB}:5000
@@ -37,5 +38,29 @@ services:
       # DOZZLE_USERNAME=
       # DOZZLE_PASSWORD=
 
+  ocrd-database:
+    image: "mongo:latest"
+    container_name: ocrd-database
+
+    environment:
+      MONGO_INITDB_ROOT_USERNAME: ${DB_ROOT_USER:-root}
+      MONGO_INITDB_ROOT_PASSWORD: ${DB_ROOT_PASSWORD:-root_password}
+
+    volumes:
+      - db-volume:/data/db
+
+
+  mongo-express:
+    image: mongo-express:latest
+    depends_on: 
+      - ocrd-database
+    ports:
+      - 8081:8081
+    environment:
+      ME_CONFIG_MONGODB_ADMINUSERNAME: ${DB_ROOT_USER:-root}
+      ME_CONFIG_MONGODB_ADMINPASSWORD: ${DB_ROOT_PASSWORD:-root_password}
+      ME_CONFIG_MONGODB_SERVER: ocrd-database
+
 volumes:
+  db-volume:
   shared:

init.sh

@@ -20,6 +20,7 @@ fi
 export OCRD_BROWSER__MODE=native
 export OCRD_BROWSER__WORKSPACE_DIR=/data
 export OCRD_BROWSER__PORT_RANGE="[9000,9100]"
+export OCRD_BROWSER__DB_CONNECTION_STRING=$DB_CONNECTION
 export OCRD_LOGVIEW__PORT=$MONITOR_PORT_LOG
 export OCRD_CONTROLLER__JOB_DIR=/run/lock/ocrd.jobs
 export OCRD_CONTROLLER__HOST=$CONTROLLER_HOST

ocrdbrowser/__init__.py

@@ -5,33 +5,23 @@
     OcrdBrowser,
     OcrdBrowserClient,
     OcrdBrowserFactory,
-    filter_owned,
-    in_other_workspaces,
-    in_same_workspace,
-    launch,
-    stop_all,
-    stop_owned_in_workspace,
 )
-from ._docker import DockerOcrdBrowserFactory
-from ._port import NoPortsAvailableError
-from ._subprocess import SubProcessOcrdBrowserFactory
 from ._client import HttpBrowserClient
+from ._docker import DockerOcrdBrowser, DockerOcrdBrowserFactory
+from ._port import NoPortsAvailableError
+from ._subprocess import SubProcessOcrdBrowser, SubProcessOcrdBrowserFactory
 
 __all__ = [
     "Channel",
     "ChannelClosed",
+    "DockerOcrdBrowser",
     "DockerOcrdBrowserFactory",
     "HttpBrowserClient",
     "NoPortsAvailableError",
     "OcrdBrowser",
     "OcrdBrowserClient",
     "OcrdBrowserFactory",
+    "SubProcessOcrdBrowser",
     "SubProcessOcrdBrowserFactory",
-    "filter_owned",
-    "launch",
-    "in_other_workspaces",
-    "in_same_workspace",
-    "stop_all",
-    "stop_owned_in_workspace",
     "workspace",
 ]

ocrdbrowser/_browser.py

@@ -6,6 +6,9 @@
 
 
 class OcrdBrowser(Protocol):
+    def process_id(self) -> str:
+        ...
+
     def address(self) -> str:
         ...
 
@@ -18,9 +21,6 @@ def workspace(self) -> str:
     def client(self) -> OcrdBrowserClient:
         ...
 
-    async def start(self) -> None:
-        ...
-
     async def stop(self) -> None:
         ...
 
@@ -46,67 +46,5 @@ def open_channel(self) -> AsyncContextManager[Channel]:
 
 
 class OcrdBrowserFactory(Protocol):
-    def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser:
+    async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser:
         ...
-
-
-BrowserProcesses = set[OcrdBrowser]
-
-
-async def launch(
-    workspace_path: str,
-    owner: str,
-    browser_factory: OcrdBrowserFactory,
-    running_browsers: BrowserProcesses | None = None,
-) -> OcrdBrowser:
-    running_browsers = running_browsers or set()
-    owned_processes = filter_owned(owner, running_browsers)
-    in_workspace = in_same_workspace(workspace_path, owned_processes)
-
-    if in_workspace:
-        return in_workspace.pop()
-
-    return await start_process(browser_factory, workspace_path, owner)
-
-
-def in_same_workspace(
-    workspace_path: str, browser_processes: BrowserProcesses
-) -> BrowserProcesses:
-    workspace_path = path.abspath(workspace_path)
-    return {
-        p for p in browser_processes if path.abspath(p.workspace()) == workspace_path
-    }
-
-
-def in_other_workspaces(
-    workspace_path: str, browser_processes: BrowserProcesses
-) -> BrowserProcesses:
-    workspace_path = path.abspath(workspace_path)
-    return {p for p in browser_processes if p.workspace() != workspace_path}
-
-
-def filter_owned(owner: str, running_processes: BrowserProcesses) -> BrowserProcesses:
-    return {p for p in running_processes if p.owner() == owner}
-
-
-async def stop_all(owned_processes: BrowserProcesses) -> None:
-    async with asyncio.TaskGroup() as group:
-        for p in owned_processes:
-            group.create_task(p.stop())
-
-
-async def stop_owned_in_workspace(
-    owner: str, workspace: str, browsers: set[OcrdBrowser]
-) -> set[OcrdBrowser]:
-    owned = filter_owned(owner, browsers)
-    in_workspace = in_same_workspace(workspace, owned)
-    await stop_all(in_workspace)
-    return in_workspace
-
-
-async def start_process(
-    process_factory: OcrdBrowserFactory, workspace_path: str, owner: str
-) -> OcrdBrowser:
-    process = process_factory(owner, workspace_path)
-    await process.start()
-    return process

ocrdbrowser/_client.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import logging
+
 from types import TracebackType
 from typing import AsyncContextManager, Type, cast
 
@@ -66,9 +68,14 @@ def __init__(self, address: str) -> None:
         self.address = address
 
     async def get(self, resource: str) -> bytes:
-        async with httpx.AsyncClient(base_url=self.address) as client:
-            response = await client.get(resource)
-            return response.content
+        try:
+            async with httpx.AsyncClient(base_url=self.address) as client:
+                response = await client.get(resource)
+                return response.content
+        except Exception as ex:
+            logging.error(f"Tried to connect to {self.address}")
+            logging.error(f"Requested resource {resource}")
+            raise ConnectionError from ex
 
     def open_channel(self) -> AsyncContextManager[Channel]:
         return WebSocketChannel(self.address + "/socket")

ocrdbrowser/_docker.py

@@ -6,7 +6,7 @@
 from typing import Any
 
 from ._browser import OcrdBrowser, OcrdBrowserClient
-from ._port import Port
+from ._port import NoPortsAvailableError
 from ._client import HttpBrowserClient
 
 _docker_run = "docker run --rm -d --name {} -v {}:/data -p {}:8085 ocrd-browser:latest"
@@ -19,51 +19,53 @@ async def _run_command(cmd: str, *args: Any) -> asyncio.subprocess.Process:
     return await asyncio.create_subprocess_shell(
         command,
         stdout=asyncio.subprocess.PIPE,
+        stderr=asyncio.subprocess.PIPE,
     )
 
 
 class DockerOcrdBrowser:
-    def __init__(self, host: str, port: Port, owner: str, workspace: str) -> None:
-        self._host = host
-        self._port = port
+    def __init__(
+        self, owner: str, workspace: str, address: str, process_id: str
+    ) -> None:
         self._owner = owner
-        self._workspace = path.abspath(workspace)
-        self.id: str | None = None
+        self._workspace = workspace
+        self._address = address
+        self._process_id: str = process_id
+
+    def process_id(self) -> str:
+        return self._process_id
 
     def address(self) -> str:
-        return f"{self._host}:{self._port}"
+        return self._address
 
     def workspace(self) -> str:
         return self._workspace
 
     def owner(self) -> str:
         return self._owner
 
-    async def start(self) -> None:
-        cmd = await _run_command(
-            _docker_run, self._container_name(), self._workspace, self._port.get()
-        )
-        self.id = str(cmd.stdout).strip()
-
     async def stop(self) -> None:
-        cmd = await _run_command(
-            _docker_stop, self._container_name(), self.workspace(), self._port.get()
-        )
+        cmd = await _run_command(_docker_stop, self._process_id)
 
         if cmd.returncode != 0:
             logging.info(
-                f"Stopping container {self.id} returned exit code {cmd.returncode}"
+                f"Stopping container {self.process_id} returned exit code {cmd.returncode}"
             )
 
-        self._port.release()
-        self.id = None
-
     def client(self) -> OcrdBrowserClient:
         return HttpBrowserClient(self.address())
 
-    def _container_name(self) -> str:
-        workspace = path.basename(self.workspace())
-        return f"ocrd-browser-{self.owner()}-{workspace}"
+
+def _container_name(owner: str, workspace: str) -> str:
+    workspace = path.basename(workspace)
+    return f"ocrd-browser-{owner}-{workspace}"
+
+
+async def log_from_stream(stream: asyncio.StreamReader | None) -> None:
+    if not stream:
+        return
+
+    logging.info(await stream.read())
 
 
 class DockerOcrdBrowserFactory:
@@ -72,16 +74,51 @@ def __init__(self, host: str, available_ports: set[int]) -> None:
         self._ports = available_ports
         self._containers: list[DockerOcrdBrowser] = []
 
-    def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser:
-        container = DockerOcrdBrowser(
-            self._host, Port(self._ports), owner, workspace_path
-        )
-        self._containers.append(container)
-        return container
+    async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser:
+        abs_workspace = path.abspath(workspace_path)
+
+        container: DockerOcrdBrowser | None = None
+        for port in self._ports:
+            cmd = await _run_command(
+                _docker_run,
+                _container_name(owner, abs_workspace),
+                abs_workspace,
+                port,
+            )
+
+            return_code = await self.wait_for(cmd)
+            if return_code != 0:
+                continue
+
+            container = DockerOcrdBrowser(
+                owner,
+                abs_workspace,
+                f"{self._host}:{port}",
+                await self._read_container_id(cmd),
+            )
+            self._containers.append(container)
+            return container
+
+        raise NoPortsAvailableError()
+
+    async def wait_for(self, cmd: asyncio.subprocess.Process) -> int:
+        return_code = await cmd.wait()
+        await log_from_stream(cmd.stderr)
+
+        return return_code
+
+    async def _read_container_id(self, cmd: asyncio.subprocess.Process) -> str:
+        stdout = cmd.stdout
+        container_id = ""
+        if stdout:
+            container_id = str(await stdout.read()).strip()
+
+        return container_id
 
     async def stop_all(self) -> None:
-        running_ids = [c.id for c in self._containers if c.id]
+        running_ids = [c.process_id() for c in self._containers]
         if running_ids:
-            await _run_command(_docker_kill, " ".join(running_ids))
+            cmd = await _run_command(_docker_kill, " ".join(running_ids))
+            await cmd.wait()
 
         self._containers = []